max/nix-super
1
0
Fork 0
mirror of https://github.com/privatevoid-net/nix-super.git synced 2024-11-22 14:06:16 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/master' into large-path-warning

Browse source
This commit is contained in:
Eelco Dolstra 2024-06-03 15:32:27 +02:00
commit 7f5b57d18f
435 changed files with 6086 additions and 2767 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.clang-format
View file

@ -15,7 +15,7 @@ SpaceAfterCStyleCast: true
SpaceAfterTemplateKeyword: false
AccessModifierOffset: -4
AlignAfterOpenBracket: AlwaysBreak
AlignEscapedNewlines: DontAlign
AlignEscapedNewlines: Left
ColumnLimit: 120
BreakStringLiterals: false
BitFieldColonSpacing: None
@ -30,3 +30,5 @@ BreakBeforeBinaryOperators: NonAssignment
AlwaysBreakBeforeMultilineStrings: true
IndentPPDirectives: AfterHash
PPIndentWidth: 2
BinPackArguments: false
BinPackParameters: false

11
.github/CODEOWNERS vendored
View file

@ -11,7 +11,16 @@
.github/CODEOWNERS @edolstra
# Documentation of built-in functions
src/libexpr/primops.cc @roberth
src/libexpr/primops.cc @roberth @fricklerhandwerk
# Documentation of settings
src/libexpr/eval-settings.hh @fricklerhandwerk
src/libstore/globals.hh @fricklerhandwerk
# Documentation
doc/manual @fricklerhandwerk
maintainers/*.md @fricklerhandwerk
src/**/*.md @fricklerhandwerk
# Libstore layer
/src/libstore @thufschmitt @ericson2314

2
.github/labeler.yml vendored
View file

@ -13,7 +13,7 @@
"documentation":
- changed-files:
- any-glob-to-any-file: "doc/manual/*"
- any-glob-to-any-file: "doc/manual/**/*"
- any-glob-to-any-file: "src/nix/**/*.md"
"store":

2
.github/workflows/backport.yml vendored
View file

@ -21,7 +21,7 @@ jobs:
fetch-depth: 0
- name: Create backport PRs
# should be kept in sync with `version`
uses: zeebe-io/backport-action@v2.5.0
uses: zeebe-io/backport-action@v3.0.2
with:
# Config README: https://github.com/zeebe-io/backport-action#backport-action
github_token: ${{ secrets.GITHUB_TOKEN }}

14
.github/workflows/ci.yml vendored
View file

@ -20,12 +20,12 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: cachix/install-nix-action@v26
- uses: cachix/install-nix-action@V27
with:
# The sandbox would otherwise be disabled by default on Darwin
extra_nix_config: "sandbox = true"
- run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
- uses: cachix/cachix-action@v14
- uses: cachix/cachix-action@v15
if: needs.check_secrets.outputs.cachix == 'true'
with:
name: '${{ env.CACHIX_NAME }}'
@ -62,10 +62,10 @@ jobs:
with:
fetch-depth: 0
- run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
- uses: cachix/install-nix-action@v26
- uses: cachix/install-nix-action@V27
with:
install_url: https://releases.nixos.org/nix/nix-2.20.3/install
- uses: cachix/cachix-action@v14
- uses: cachix/cachix-action@v15
with:
name: '${{ env.CACHIX_NAME }}'
signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
@ -84,7 +84,7 @@ jobs:
steps:
- uses: actions/checkout@v4
- run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
- uses: cachix/install-nix-action@v26
- uses: cachix/install-nix-action@V27
with:
install_url: '${{needs.installer.outputs.installerURL}}'
install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
@ -114,12 +114,12 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: cachix/install-nix-action@v26
- uses: cachix/install-nix-action@V27
with:
install_url: https://releases.nixos.org/nix/nix-2.20.3/install
- run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
- run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV
- uses: cachix/cachix-action@v14
- uses: cachix/cachix-action@v15
if: needs.check_secrets.outputs.cachix == 'true'
with:
name: '${{ env.CACHIX_NAME }}'

2
.gitignore vendored
View file

@ -92,7 +92,7 @@ perl/Makefile.config
# /tests/functional/
/tests/functional/test-tmp
/tests/functional/common/vars-and-functions.sh
/tests/functional/common/subst-vars.sh
/tests/functional/result*
/tests/functional/restricted-innocent
/tests/functional/shell

2
.shellcheckrc Normal file
View file

@ -0,0 +1,2 @@
external-sources=true
source-path=SCRIPTDIR

42
CITATION.cff Normal file
View file

@ -0,0 +1,42 @@
cff-version: 1.2.0
title: Nix
message: >-
If you use this software, please cite it using the
metadata from this file.
type: software
authors:
- given-names: Eelco
family-names: Dolstra
email: edolstra@gmail.com
- name: The Nix contributors
website: 'https://github.com/NixOS/nix'
references:
- title: The Purely Functional Software Deployment Model
authors:
- family-names: Dolstra
given-names: Eelco
year: 2006
type: thesis
thesis-type: PhD thesis
isbn: 90-393-4130-3
url: https://dspace.library.uu.nl/handle/1874/7540
database-provider: Utrecht University Repository
institution:
name: Utrecht University
keywords:
- configuration management
- software deployment
- purely functional
- component-based software engineering
repository-code: 'https://github.com/NixOS/nix'
url: 'https://nixos.org/'
abstract: >-
Nix, a purely functional package manager, is a powerful
package manager for Linux and other Unix systems that
makes package management reliable and reproducible.
keywords:
- reproducibility
- open-source
- c++
- functional
license: LGPL-2.1

2
Makefile
View file

@ -98,7 +98,7 @@ ifdef HOST_WINDOWS
GLOBAL_LDFLAGS += -Wl,--export-all-symbols
endif
GLOBAL_CXXFLAGS += -g -Wall -Wimplicit-fallthrough -include $(buildprefix)config.h -std=c++2a -I src
GLOBAL_CXXFLAGS += -g -Wall -Wdeprecated-copy -Wignored-qualifiers -Wimplicit-fallthrough -include $(buildprefix)config.h -std=c++2a -I src
# Include the main lib, causing rules to be defined

186
build/hydra.nix Normal file
View file

@ -0,0 +1,186 @@
{ inputs
, binaryTarball
, forAllCrossSystems
, forAllSystems
, lib
, linux64BitSystems
, nixpkgsFor
, self
}:
let
inherit (inputs) nixpkgs nixpkgs-regression;
inherit (lib) fileset;
installScriptFor = tarballs:
nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix {
inherit tarballs;
};
testNixVersions = pkgs: client: daemon:
pkgs.callPackage ../package.nix {
pname =
"nix-tests"
+ lib.optionalString
(lib.versionAtLeast daemon.version "2.4pre20211005" &&
lib.versionAtLeast client.version "2.4pre20211005")
"-${client.version}-against-${daemon.version}";
inherit fileset;
test-client = client;
test-daemon = daemon;
doBuild = false;
};
in
{
# Binary package for various platforms.
build = forAllSystems (system: self.packages.${system}.nix);
shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation);
buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static);
buildCross = forAllCrossSystems (crossSystem:
lib.genAttrs [ "x86_64-linux" ] (system: self.packages.${system}."nix-${crossSystem}"));
buildNoGc = forAllSystems (system:
self.packages.${system}.nix.override { enableGC = false; }
);
buildNoTests = forAllSystems (system:
self.packages.${system}.nix.override {
doCheck = false;
doInstallCheck = false;
installUnitTests = false;
}
);
# Toggles some settings for better coverage. Windows needs these
# library combinations, and Debian build Nix with GNU readline too.
buildReadlineNoMarkdown = forAllSystems (system:
self.packages.${system}.nix.override {
enableMarkdown = false;
readlineFlavor = "readline";
}
);
# Perl bindings for various platforms.
perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings);
# Binary tarball for various platforms, containing a Nix store
# with the closure of 'nix' package, and the second half of
# the installation script.
binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native);
binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system:
forAllCrossSystems (crossSystem:
binaryTarball
self.packages.${system}."nix-${crossSystem}"
nixpkgsFor.${system}.cross.${crossSystem}));
# The first half of the installation script. This is uploaded
# to https://nixos.org/nix/install. It downloads the binary
# tarball for the user's system and calls the second half of the
# installation script.
installerScript = installScriptFor [
# Native
self.hydraJobs.binaryTarball."x86_64-linux"
self.hydraJobs.binaryTarball."i686-linux"
self.hydraJobs.binaryTarball."aarch64-linux"
self.hydraJobs.binaryTarball."x86_64-darwin"
self.hydraJobs.binaryTarball."aarch64-darwin"
# Cross
self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
];
installerScriptForGHA = installScriptFor [
# Native
self.hydraJobs.binaryTarball."x86_64-linux"
self.hydraJobs.binaryTarball."x86_64-darwin"
# Cross
self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
];
# docker image with Nix inside
dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);
# Line coverage analysis.
coverage = nixpkgsFor.x86_64-linux.native.nix.override {
pname = "nix-coverage";
withCoverageChecks = true;
};
# API docs for Nix's unstable internal C++ interfaces.
internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ../package.nix {
inherit fileset;
doBuild = false;
enableInternalAPIDocs = true;
};
# API docs for Nix's C bindings.
external-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ../package.nix {
inherit fileset;
doBuild = false;
enableExternalAPIDocs = true;
};
# System tests.
tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor; } // {
# Make sure that nix-env still produces the exact same result
# on a particular version of Nixpkgs.
evalNixpkgs =
let
inherit (nixpkgsFor.x86_64-linux.native) runCommand nix;
in
runCommand "eval-nixos" { buildInputs = [ nix ]; }
''
type -p nix-env
# Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593.
(
set -x
time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages
[[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]]
)
mkdir $out
'';
nixpkgsLibTests =
forAllSystems (system:
import (nixpkgs + "/lib/tests/release.nix")
{
pkgs = nixpkgsFor.${system}.native;
nixVersions = [ self.packages.${system}.nix ];
}
);
};
metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" {
pkgs = nixpkgsFor.x86_64-linux.native;
nixpkgs = nixpkgs-regression;
};
installTests = forAllSystems (system:
let pkgs = nixpkgsFor.${system}.native; in
pkgs.runCommand "install-tests"
{
againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
againstCurrentUnstable =
# FIXME: temporarily disable this on macOS because of #3605.
if system == "x86_64-linux"
then testNixVersions pkgs pkgs.nix pkgs.nixUnstable
else null;
# Disabled because the latest stable version doesn't handle
# `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
# againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
} "touch $out");
installerTests = import ../tests/installer {
binaryTarballs = self.hydraJobs.binaryTarball;
inherit nixpkgsFor;
};
}

2
configure.ac
View file

@ -313,7 +313,7 @@ case "$host_os" in
]))
if test "x$enable_seccomp_sandboxing" != "xno"; then
PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp],
[CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"])
[CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS" CFLAGS="$LIBSECCOMP_CFLAGS $CFLAGS"])
have_seccomp=1
AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.])
AC_COMPILE_IFELSE([

8
doc/external-api/README.md
View file

@ -27,7 +27,7 @@ appreciated.
The following examples, for simplicity, don't include error handling. See the
[Handling errors](@ref errors) section for more information.
# Embedding the Nix Evaluator
# Embedding the Nix Evaluator{#nix_evaluator_example}
In this example we programmatically start the Nix language evaluator with a
dummy store (that has no store paths and cannot be written to), and evaluate the
@ -46,9 +46,9 @@ Nix expression `builtins.nixVersion`.
// NOTE: This example lacks all error handling. Production code must check for
// errors, as some return values will be undefined.
void my_get_string_cb(const char * start, unsigned int n, char ** user_data)
void my_get_string_cb(const char * start, unsigned int n, void * user_data)
{
*user_data = strdup(start);
*((char **) user_data) = strdup(start);
}
int main()
@ -63,7 +63,7 @@ int main()
nix_value_force(NULL, state, value);
char * version;
nix_get_string(NULL, value, my_get_string_cb, version);
nix_get_string(NULL, value, my_get_string_cb, &version);
printf("Nix version: %s\n", version);
free(version);

2
doc/manual/book.toml
View file

@ -6,8 +6,6 @@ additional-css = ["custom.css"]
additional-js = ["redirects.js"]
edit-url-template = "https://github.com/NixOS/nix/tree/master/doc/manual/{path}"
git-repository-url = "https://github.com/NixOS/nix"
fold.enable = true
fold.level = 1
[preprocessor.anchors]
renderers = ["html"]

10
doc/manual/redirects.js
View file

@ -285,15 +285,15 @@ const redirects = {
"ch-basic-package-mgmt": "package-management/basic-package-mgmt.html",
"ssec-binary-cache-substituter": "package-management/binary-cache-substituter.html",
"sec-channels": "command-ref/nix-channel.html",
"ssec-copy-closure": "package-management/copy-closure.html",
"ssec-copy-closure": "command-ref/nix-copy-closure.html",
"sec-garbage-collection": "package-management/garbage-collection.html",
"ssec-gc-roots": "package-management/garbage-collector-roots.html",
"chap-package-management": "package-management/index.html",
"sec-profiles": "package-management/profiles.html",
"ssec-s3-substituter": "package-management/s3-substituter.html",
"ssec-s3-substituter-anonymous-reads": "package-management/s3-substituter.html#anonymous-reads-to-your-s3-compatible-binary-cache",
"ssec-s3-substituter-authenticated-reads": "package-management/s3-substituter.html#authenticated-reads-to-your-s3-binary-cache",
"ssec-s3-substituter-authenticated-writes": "package-management/s3-substituter.html#authenticated-writes-to-your-s3-compatible-binary-cache",
"ssec-s3-substituter": "store/types/s3-substituter.html",
"ssec-s3-substituter-anonymous-reads": "store/types/s3-substituter.html#anonymous-reads-to-your-s3-compatible-binary-cache",
"ssec-s3-substituter-authenticated-reads": "store/types/s3-substituter.html#authenticated-reads-to-your-s3-binary-cache",
"ssec-s3-substituter-authenticated-writes": "store/types/s3-substituter.html#authenticated-writes-to-your-s3-compatible-binary-cache",
"sec-sharing-packages": "package-management/sharing-packages.html",
"ssec-ssh-substituter": "package-management/ssh-substituter.html",
"chap-quick-start": "quick-start.html",

6
doc/manual/rl-next/consistent-nix-build.md Normal file
View file

@ -0,0 +1,6 @@
---
synopsis: Show all FOD errors with `nix build --keep-going`
---
`nix build --keep-going` now behaves consistently with `nix-build --keep-going`. This means
that if e.g. multiple FODs fail to build, all hash mismatches are displayed.

12
doc/manual/rl-next/derivation-json-change.md Normal file
View file

@ -0,0 +1,12 @@
---
synopsis: Modify `nix derivation {add,show}` JSON format
issues: 9866
prs: 10722
---
The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@contributing/cli-guideline#returning-future-proof-json).
In particular, the hash algorithm and content addressing method of content-addresed derivation outputs is now separated into two fields `hashAlgo` and `method`,
rather than one field with an arcane `:`-separated format.
This JSON format is only used by the experimental `nix derivation` family of commands, at this time.
Future revisions are expected as the JSON format is still not entirely in compliance even after these changes.

28
doc/manual/rl-next/fix-silent-unknown-options.md Normal file
View file

@ -0,0 +1,28 @@
---
synopsis: Warn on unknown settings anywhere in the command line
prs: 10701
---
All `nix` commands will now properly warn when an unknown option is specified anywhere in the command line.
Before:
```console
$ nix-instantiate --option foobar baz --expr '{}'
warning: unknown setting 'foobar'
$ nix-instantiate '{}' --option foobar baz --expr
$ nix eval --expr '{}' --option foobar baz
{ }
```
After:
```console
$ nix-instantiate --option foobar baz --expr '{}'
warning: unknown setting 'foobar'
$ nix-instantiate '{}' --option foobar baz --expr
warning: unknown setting 'foobar'
$ nix eval --expr '{}' --option foobar baz
warning: unknown setting 'foobar'
{ }
```

12
doc/manual/rl-next/nix-env-shell.md Normal file
View file

@ -0,0 +1,12 @@
---
synopsis: "`nix env shell` is the new `nix shell`, and `nix shell` remains an accepted alias"
issues: 10504
prs: 10807
---
This is part of an effort to bring more structure to the CLI subcommands.
`nix env` will be about the process environment.
Future commands may include `nix env run` and `nix env print-env`.
It is also somewhat analogous to the [planned](https://github.com/NixOS/nix/issues/10504) `nix dev shell` (currently `nix develop`), which is less about environment variables, and more about running a development shell, which is a more powerful command, but also requires more setup.

18
doc/manual/rl-next/print-value-in-installable-flake-error.md Normal file
View file

@ -0,0 +1,18 @@
---
synopsis: New-cli flake commands that expect derivations now print the failing value and its type
prs: 10778
---
In errors like `flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path`, the message now includes the failing value and type.
Before:
```
error: flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path
````
After:
```
error: expected flake output attribute 'nixosConfigurations.yuki.config' to be a derivation or path but found a set: { appstream = «thunk»; assertions = «thunk»; boot = { bcache = «thunk»; binfmt = «thunk»; binfmtMiscRegistrations = «thunk»; blacklistedKernelModules = «thunk»; bootMount = «thunk»; bootspec = «thunk»; cleanTmpDir = «thunk»; consoleLogLevel = «thunk»; «43 attributes elided» }; «48 attributes elided» }
```

11
doc/manual/rl-next/store-object-info.md Normal file
View file

@ -0,0 +1,11 @@
---
synopsis: Store object info JSON format now uses `null` rather than omitting fields.
prs: 9995
---
The [store object info JSON format](@docroot@/protocols/json/store-object-info.md), used for e.g. `nix path-info`, no longer omits fields to indicate absent information, but instead includes the fields with a `null` value.
For example, `"ca": null` is used to to indicate a store object that isn't content-addressed rather than omitting the `ca` field entirely.
This makes records of this sort more self-describing, and easier to consume programmatically.
We will follow this design principle going forward;
the [JSON guidelines](@docroot@/contributing/json-guideline.md) in the contributing section have been updated accordingly.

5
doc/manual/src/SUMMARY.md.in
View file

@ -18,7 +18,9 @@
- [Uninstalling Nix](installation/uninstall.md)
- [Nix Store](store/index.md)
- [File System Object](store/file-system-object.md)
- [Content-Addressing File System Objects](store/file-system-object/content-address.md)
- [Store Object](store/store-object.md)
- [Content-Addressing Store Objects](store/store-object/content-address.md)
- [Store Path](store/store-path.md)
- [Store Types](store/types/index.md)
{{#include ./store/types/SUMMARY.md}}
@ -41,9 +43,7 @@
- [Advanced Topics](advanced-topics/index.md)
- [Sharing Packages Between Machines](package-management/sharing-packages.md)
- [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md)
- [Copying Closures via SSH](package-management/copy-closure.md)
- [Serving a Nix store via SSH](package-management/ssh-substituter.md)
- [Serving a Nix store via S3](package-management/s3-substituter.md)
- [Remote Builds](advanced-topics/distributed-builds.md)
- [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md)
- [Verifying Build Reproducibility](advanced-topics/diff-hook.md)
@ -121,6 +121,7 @@
- [Documentation](contributing/documentation.md)
- [Experimental Features](contributing/experimental-features.md)
- [CLI guideline](contributing/cli-guideline.md)
- [JSON guideline](contributing/json-guideline.md)
- [C++ style guide](contributing/cxx.md)
- [Releases](release-notes/index.md)
{{#include ./SUMMARY-rl-next.md}}

2
doc/manual/src/_redirects
View file

@ -39,3 +39,5 @@
/json/* /protocols/json/:splat 301!
/release-notes/release-notes /release-notes 301!
/package-management/copy-closure /command-ref/nix-copy-closure 301!

2
doc/manual/src/command-ref/nix-collect-garbage.md
View file

@ -74,4 +74,4 @@ $ nix-collect-garbage -d
```
[profiles]: @docroot@/command-ref/files/profiles.md
[store objects]: @docroot@/glossary.md#gloss-store-object
[store objects]: @docroot@/store/store-object.md

104
doc/manual/src/command-ref/nix-copy-closure.md
View file

@ -1,6 +1,6 @@
# Name
`nix-copy-closure` - copy a closure to or from a remote machine via SSH
`nix-copy-closure` - copy store objects to or from a remote machine via SSH
# Synopsis
@ -10,82 +10,82 @@
[`--include-outputs`]
[`--use-substitutes` | `-s`]
[`-v`]
_user@machine_ _paths_
[_user_@]_machine_[:_port_] _paths_
# Description
`nix-copy-closure` gives you an easy and efficient way to exchange
software between machines. Given one or more Nix store _paths_ on the
local machine, `nix-copy-closure` computes the closure of those paths
(i.e. all their dependencies in the Nix store), and copies all paths
in the closure to the remote machine via the `ssh` (Secure Shell)
command. With the `--from` option, the direction is reversed: the
closure of _paths_ on a remote machine is copied to the Nix store on
the local machine.
Given _paths_ from one machine, `nix-copy-closure` computes the [closure](@docroot@/glossary.md#gloss-closure) of those paths (i.e. all their dependencies in the Nix store), and copies [store objects](@docroot@/glossary.md#gloss-store-object) in that closure to another machine via SSH.
It doesn’t copy store objects that are already present on the other machine.
This command is efficient because it only sends the store paths
that are missing on the target machine.
> **Note**
>
> While the Nix store to use on the local machine can be specified on the command line with the [`--store`](@docroot@/command-ref/conf-file.md#conf-store) option, the Nix store to be accessed on the remote machine can only be [configured statically](@docroot@/command-ref/conf-file.md#configuration-file) on that remote machine.
Since `nix-copy-closure` calls `ssh`, you may be asked to type in the
appropriate password or passphrase. In fact, you may be asked _twice_
because `nix-copy-closure` currently connects twice to the remote
machine, first to get the set of paths missing on the target machine,
and second to send the dump of those paths. When using public key
authentication, you can avoid typing the passphrase with `ssh-agent`.
Since `nix-copy-closure` calls `ssh`, you may need to authenticate with the remote machine.
In fact, you may be asked for authentication _twice_ because `nix-copy-closure` currently connects twice to the remote machine: first to get the set of paths missing on the target machine, and second to send the dump of those paths.
When using public key authentication, you can avoid typing the passphrase with `ssh-agent`.
# Options
- `--to`\
Copy the closure of _paths_ from the local Nix store to the Nix
store on _machine_. This is the default.
- `--to`
- `--from`\
Copy the closure of _paths_ from the Nix store on _machine_ to the
local Nix store.
Copy the closure of _paths_ from a Nix store accessible from the local machine to the Nix store on the remote _machine_.
This is the default behavior.
- `--from`
Copy the closure of _paths_ from the Nix store on the remote _machine_ to the local machine's specified Nix store.
- `--gzip`
- `--gzip`\
Enable compression of the SSH connection.
- `--include-outputs`\
- `--include-outputs`
Also copy the outputs of [store derivation]s included in the closure.
[store derivation]: @docroot@/glossary.md#gloss-store-derivation
- `--use-substitutes` / `-s`\
Attempt to download missing paths on the target machine using Nix’s
substitute mechanism. Any paths that cannot be substituted on the
target are still copied normally from the source. This is useful,
for instance, if the connection between the source and target
machine is slow, but the connection between the target machine and
`nixos.org` (the default binary cache server) is
fast.
- `--use-substitutes` / `-s`
- `-v`\
Show verbose output.
Attempt to download missing store objects on the target from [substituters](@docroot@/command-ref/conf-file.md#conf-substituters).
Any store objects that cannot be substituted on the target are still copied normally from the source.
This is useful, for instance, if the connection between the source and target machine is slow, but the connection between the target machine and `cache.nixos.org` (the default binary cache server) is fast.
{{#include ./opt-common.md}}
# Environment variables
- `NIX_SSHOPTS`\
Additional options to be passed to `ssh` on the command
line.
- `NIX_SSHOPTS`
Additional options to be passed to `ssh` on the command line.
{{#include ./env-common.md}}
# Examples
Copy Firefox with all its dependencies to a remote machine:
> **Example**
>
> Copy GNU Hello with all its dependencies to a remote machine:
>
> ```shell-session
> $ storePath="$(nix-build '<nixpkgs>' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link)"
> $ nix-copy-closure --to alice@itchy.example.org "$storePath"
> copying 5 paths...
> copying path '/nix/store/nrwkk6ak3rgkrxbqhsscb01jpzmslf2r-xgcc-13.2.0-libgcc' to 'ssh://alice@itchy.example.org'...
> copying path '/nix/store/gm61h1y42pqyl6178g90x8zm22n6pyy5-libunistring-1.1' to 'ssh://alice@itchy.example.org'...
> copying path '/nix/store/ddfzjdykw67s20c35i7a6624by3iz5jv-libidn2-2.3.7' to 'ssh://alice@itchy.example.org'...
> copying path '/nix/store/apab5i73dqa09wx0q27b6fbhd1r18ihl-glibc-2.39-31' to 'ssh://alice@itchy.example.org'...
> copying path '/nix/store/g1n2vryg06amvcc1avb2mcq36faly0mh-hello-2.12.1' to 'ssh://alice@itchy.example.org'...
> ```
```console
$ nix-copy-closure --to alice@itchy.labs $(type -tP firefox)
```
Copy Subversion from a remote machine and then install it into a user
environment:
```console
$ nix-copy-closure --from alice@itchy.labs \
/nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
$ nix-env --install /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
```
> **Example**
>
> Copy GNU Hello from a remote machine using a known store path, and run it:
>
> ```shell-session
> $ storePath="$(nix-instantiate --eval '<nixpkgs>' -I nixpkgs=channel:nixpkgs-unstable -A hello.outPath | tr -d '"')"
> $ nix-copy-closure --from alice@itchy.example.org "$storePath"
> $ "$storePath"/bin/hello
> Hello, world!
> ```

94
doc/manual/src/command-ref/nix-env.md
View file

@ -47,39 +47,83 @@ These pages can be viewed offline:
Example: `nix-env --help --install`
# Package sources
`nix-env` can obtain packages from multiple sources:
- An attribute set of derivations from:
- The [default Nix expression](@docroot@/command-ref/files/default-nix-expression.md) (by default)
- A Nix file, specified via `--file`
- A [profile](@docroot@/command-ref/files/profiles.md), specified via `--from-profile`
- A Nix expression that is a function which takes default expression as argument, specified via `--from-expression`
- A [store path](@docroot@/store/store-path.md)
# Selectors
Several commands, such as `nix-env --query ` and `nix-env --install `, take a list of
arguments that specify the packages on which to operate. These are
extended regular expressions that must match the entire name of the
package. (For details on regular expressions, see **regex**(7).) The match is
case-sensitive. The regular expression can optionally be followed by a
dash and a version number; if omitted, any version of the package will
match. Here are some examples:
Several operations, such as [`nix-env --query`](./nix-env/query.md) and [`nix-env --install`](./nix-env/install.md), take a list of *arguments* that specify the packages on which to operate.
- `firefox`\
Matches the package name `firefox` and any version.
Packages are identified based on a `name` part and a `version` part of a [symbolic derivation name](@docroot@/language/derivations.md#attr-names):
- `firefox-32.0`\
Matches the package name `firefox` and version `32.0`.
- `name`: Everything up to but not including the first dash (`-`) that is *not* followed by a letter.
- `version`: The rest, excluding the separating dash.
- `gtk\\+`\
Matches the package name `gtk+`. The `+` character must be escaped
using a backslash to prevent it from being interpreted as a
quantifier, and the backslash must be escaped in turn with another
backslash to ensure that the shell passes it on.
> **Example**
>
> `nix-env` parses the symbolic derivation name `apache-httpd-2.0.48` as:
>
> ```json
> {
> "name": "apache-httpd",
> "version": "2.0.48"
> }
> ```
- `.\*`\
Matches any package name. This is the default for most commands.
> **Example**
>
> `nix-env` parses the symbolic derivation name `firefox.*` as:
>
> ```json
> {
> "name": "firefox.*",
> "version": ""
> }
> ```
- `'.*zip.*'`\
Matches any package name containing the string `zip`. Note the dots:
`'*zip*'` does not work, because in a regular expression, the
character `*` is interpreted as a quantifier.
The `name` parts of the *arguments* to `nix-env` are treated as extended regular expressions and matched against the `name` parts of derivation names in the package source.
The match is case-sensitive.
The regular expression can optionally be followed by a dash (`-`) and a version number; if omitted, any version of the package will match.
For details on regular expressions, see [**regex**(7)](https://linux.die.net/man/7/regex).
- `'.*(firefox|chromium).*'`\
Matches any package name containing the strings `firefox` or
`chromium`.
> **Example**
>
> Common patterns for finding package names with `nix-env`:
>
> - `firefox`
>
> Matches the package name `firefox` and any version.
>
> - `firefox-32.0`
>
> Matches the package name `firefox` and version `32.0`.
>
> - `gtk\\+`
>
> Matches the package name `gtk+`.
> The `+` character must be escaped using a backslash (`\`) to prevent it from being interpreted as a quantifier, and the backslash must be escaped in turn with another backslash to ensure that the shell passes it on.
>
> - `.\*`
>
> Matches any package name.
> This is the default for most commands.
>
> - `'.*zip.*'`
>
> Matches any package name containing the string `zip`.
> Note the dots: `'*zip*'` does not work, because in a regular expression, the character `*` is interpreted as a quantifier.
>
> - `'.*(firefox|chromium).*'`
>
> Matches any package name containing the strings `firefox` or `chromium`.
# Files

2
doc/manual/src/command-ref/nix-env/delete-generations.md
View file

@ -49,7 +49,7 @@ Periodically deleting old generations is important to make garbage collection
effective.
The is because profiles are also garbage collection roots — any [store object] reachable from a profile is "alive" and ineligible for deletion.
[store object]: @docroot@/glossary.md#gloss-store-object
[store object]: @docroot@/store/store-object.md
{{#include ./opt-common.md}}

5
doc/manual/src/command-ref/nix-env/install.md
View file

@ -14,14 +14,13 @@
# Description
The install operation creates a new user environment.
The `--install` operation creates a new user environment.
It is based on the current generation of the active [profile](@docroot@/command-ref/files/profiles.md), to which a set of [store paths] described by *args* is added.
[store paths]: @docroot@/glossary.md#gloss-store-path
[store paths]: @docroot@/store/store-path.md
The arguments *args* map to store paths in a number of possible ways:
- By default, *args* is a set of [derivation] names denoting derivations in the [default Nix expression].
These are [realised], and the resulting output paths are installed.
Currently installed derivations with a name equal to the name of a derivation being added are removed unless the option `--preserve-installed` is specified.

11
doc/manual/src/command-ref/nix-hash.md
View file

@ -20,16 +20,21 @@ an example.
The hash is computed over a *serialisation* of each path: a dump of
the file system tree rooted at the path. This allows directories and
symlinks to be hashed as well as regular files. The dump is in the
*NAR format* produced by [`nix-store
*[Nix Archive (NAR)][Nix Archive] format* produced by [`nix-store
--dump`](@docroot@/command-ref/nix-store/dump.md). Thus, `nix-hash path`
yields the same cryptographic hash as `nix-store --dump path |
md5sum`.
[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive
# Options
- `--flat`\
Print the cryptographic hash of the contents of each regular file
*path*. That is, do not compute the hash over the dump of *path*.
Print the cryptographic hash of the contents of each regular file *path*.
That is, instead of computing
the hash of the [Nix Archive (NAR)](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) of *path*,
just [directly hash]((@docroot@/store/file-system-object/content-address.md#serial-flat) *path* as is.
This requires *path* to resolve to a regular file rather than directory.
The result is identical to that produced by the GNU commands
`md5sum` and `sha1sum`.

9
doc/manual/src/command-ref/nix-store/dump.md
View file

@ -1,6 +1,6 @@
# Name
`nix-store --dump` - write a single path to a Nix Archive
`nix-store --dump` - write a single path to a [Nix Archive]
## Synopsis
@ -8,7 +8,7 @@
## Description
The operation `--dump` produces a NAR (Nix ARchive) file containing the
The operation `--dump` produces a [Nix archive](@docroot@/glossary.md#gloss-nar) (NAR) file containing the
contents of the file system tree rooted at *path*. The archive is
written to standard output.
@ -30,8 +30,9 @@ NAR archives support filenames of unlimited length and 64-bit file
sizes. They can contain regular files, directories, and symbolic links,
but not other types of files (such as device nodes).
A Nix archive can be unpacked using `nix-store
--restore`.
A Nix archive can be unpacked using [`nix-store --restore`](@docroot@/command-ref/nix-store/restore.md).
[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive
{{#include ./opt-common.md}}

56
doc/manual/src/command-ref/nix-store/export.md
View file

@ -1,6 +1,6 @@
# Name
`nix-store --export` - export store paths to a Nix Archive
`nix-store --export` - export store paths to a [Nix Archive]
## Synopsis
@ -8,16 +8,22 @@
## Description
The operation `--export` writes a serialisation of the specified store
paths to standard output in a format that can be imported into another
Nix store with `nix-store --import`. This is like `nix-store
--dump`, except that the NAR archive produced by that command doesn’t
contain the necessary meta-information to allow it to be imported into
another Nix store (namely, the set of references of the path).
The operation `--export` writes a serialisation of the given [store objects](@docroot@/glossary.md#gloss-store-object) to standard output in a format that can be imported into another [Nix store](@docroot@/store/index.md) with [`nix-store --import`](./import.md).
This command does not produce a *closure* of the specified paths, so if
a store path references other store paths that are missing in the target
Nix store, the import will fail.
> **Warning**
>
> This command *does not* produce a [closure](@docroot@/glossary.md#gloss-closure) of the specified store paths.
> Trying to import a store object that refers to store paths not available in the target Nix store will fail.
>
> Use [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) to obtain the closure of a store path.
This command is different from [`nix-store --dump`](./dump.md), which produces a [Nix archive](@docroot@/glossary.md#gloss-nar) that *does not* contain the set of [references](@docroot@/glossary.md#gloss-reference) of a given store path.
> **Note**
>
> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md).
[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive
{{#include ./opt-common.md}}
@ -27,15 +33,21 @@ Nix store, the import will fail.
# Examples
To copy a whole closure, do something
like:
```console
$ nix-store --export $(nix-store --query --requisites paths) > out
```
To import the whole closure again, run:
```console
$ nix-store --import < out
```
> **Example**
>
> Deploy GNU Hello to an airgapped machine via USB stick.
>
> Write the closure to the block device on a machine with internet connection:
>
> ```shell-session
> [alice@itchy]$ storePath=$(nix-build '<nixpkgs>' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link)
> [alice@itchy]$ nix-store --export $(nix-store --query --requisites $storePath) | sudo dd of=/dev/usb
> ```
>
> Read the closure from the block device on the machine without internet connection:
>
> ```shell-session
> [bob@scratchy]$ hello=$(sudo dd if=/dev/usb | nix-store --import | tail -1)
> [bob@scratchy]$ $hello/bin/hello
> Hello, world!
> ```

34
doc/manual/src/command-ref/nix-store/import.md
View file

@ -1,6 +1,8 @@
# Name
`nix-store --import` - import Nix Archive into the store
`nix-store --import` - import [Nix Archive] into the store
[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive
# Synopsis
@ -8,14 +10,34 @@
# Description
The operation `--import` reads a serialisation of a set of store paths
produced by `nix-store --export` from standard input and adds those
store paths to the Nix store. Paths that already exist in the Nix store
are ignored. If a path refers to another path that doesn’t exist in the
Nix store, the import fails.
The operation `--import` reads a serialisation of a set of [store objects](@docroot@/glossary.md#gloss-store-object) produced by [`nix-store --export`](./export.md) from standard input, and adds those store objects to the specified [Nix store](@docroot@/store/index.md).
Paths that already exist in the target Nix store are ignored.
If a path [refers](@docroot@/glossary.md#gloss-reference) to another path that doesn’t exist in the target Nix store, the import fails.
> **Note**
>
> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md).
{{#include ./opt-common.md}}
{{#include ../opt-common.md}}
{{#include ../env-common.md}}
# Examples
> **Example**
>
> Given a closure of GNU Hello as a file:
>
> ```shell-session
> $ storePath="$(nix-build '<nixpkgs>' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link)"
> $ nix-store --export $(nix-store --query --requisites $storePath) > hello.closure
> ```
>
> Import the closure into a [remote SSH store](@docroot@/store/types/ssh-store.md) using the [`--store`](@docroot@/command-ref/conf-file.md#conf-store) option:
>
> ```console
> $ nix-store --import --store ssh://alice@itchy.example.org < hello.closure
> ```

3
doc/manual/src/command-ref/nix-store/optimise.md
View file

@ -12,7 +12,7 @@ The operation `--optimise` reduces Nix store disk space usage by finding
identical files in the store and hard-linking them to each other. It
typically reduces the size of the store by something like 25-35%. Only
regular files and symlinks are hard-linked in this manner. Files are
considered identical when they have the same NAR archive serialisation:
considered identical when they have the same [Nix Archive (NAR)][Nix Archive] serialisation:
that is, regular files must have the same contents and permission
(executable or non-executable), and symlinks must have the same
contents.
@ -38,3 +38,4 @@ hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1'
there are 114486 files with equal contents out of 215894 files in total
```
[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive

4
doc/manual/src/command-ref/nix-store/realise.md
View file

@ -25,11 +25,11 @@ Each of *paths* is processed as follows:
If no substitutes are available and no store derivation is given, realisation fails.
[store paths]: @docroot@/glossary.md#gloss-store-path
[store paths]: @docroot@/store/store-path.md
[valid]: @docroot@/glossary.md#gloss-validity
[store derivation]: @docroot@/glossary.md#gloss-store-derivation
[output paths]: @docroot@/glossary.md#gloss-output-path
[store objects]: @docroot@/glossary.md#gloss-store-object
[store objects]: @docroot@/store/store-object.md
[closure]: @docroot@/glossary.md#gloss-closure
[substituters]: @docroot@/command-ref/conf-file.md#conf-substituters
[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations

4
doc/manual/src/command-ref/nix-store/restore.md
View file

@ -8,9 +8,11 @@
## Description
The operation `--restore` unpacks a NAR archive to *path*, which must
The operation `--restore` unpacks a [Nix Archive (NAR)][Nix Archive] to *path*, which must
not already exist. The archive is read from standard input.
[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive
{{#include ./opt-common.md}}
{{#include ../opt-common.md}}

82
doc/manual/src/contributing/cli-guideline.md
View file

@ -389,88 +389,6 @@ colors, no emojis and using ASCII instead of Unicode symbols). The same should
happen when TTY is not detected on STDERR. We should not display progress /
status section, but only print warnings and errors.
## Returning future proof JSON
The schema of JSON output should allow for backwards compatible extension. This section explains how to achieve this.
Two definitions are helpful here, because while JSON only defines one "key-value"
object type, we use it to cover two use cases:
- **dictionary**: a map from names to value that all have the same type. In
C++ this would be a `std::map` with string keys.
- **record**: a fixed set of attributes each with their own type. In C++, this
would be represented by a `struct`.
It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type.
This leads to the following guidelines:
- The top-level (root) value must be a record.
Otherwise, one can not change the structure of a command's output.
- The value of a dictionary item must be a record.
Otherwise, the item type can not be extended.
- List items should be records.
Otherwise, one can not change the structure of the list items.
If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. If the order of the items needs to be preserved, return a list of records.
- Streaming JSON should return records.
An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. These JSON values can be considered top-level values or list items, and they must be records.
### Examples
This is bad, because all keys must be assumed to be store types:
```json
{
"local": { ... },
"remote": { ... },
"http": { ... }
}
```
This is good, because the it is extensible at the root, and is somewhat self-documenting:
```json
{
"storeTypes": { "local": { ... }, ... },
"pluginSupport": true
}
```
While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information.
For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing.
The following representation is bad because it is not extensible:
```json
{ "outputs": [ "out" "bin" ] }
```
However, simply converting everything to records is not enough, because the order of outputs must be preserved:
```json
{ "outputs": { "bin": {}, "out": {} } }
```
The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work.
While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries.
This representation is extensible and preserves the ordering:
```json
{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] }
```
## Dialog with the user
CLIs don't always make it clear when an action has taken place. For every

2
doc/manual/src/contributing/documentation.md
View file

@ -147,7 +147,7 @@ Please observe these guidelines to ease reviews:
```
A [store object] contains a [file system object] and [references] to other store objects.
[store object]: @docroot@/glossary.md#gloss-store-object
[store object]: @docroot@/store/store-object.md
[file system object]: @docroot@/architecture/file-system-object.md
[references]: @docroot@/glossary.md#gloss-reference
```

128
doc/manual/src/contributing/json-guideline.md Normal file
View file

@ -0,0 +1,128 @@
# JSON guideline
Nix consumes and produces JSON in a variety of contexts.
These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another.
## Extensibility
The schema of JSON input and output should allow for backwards compatible extension.
This section explains how to achieve this.
Two definitions are helpful here, because while JSON only defines one "key-value" object type, we use it to cover two use cases:
- **dictionary**: a map from names to value that all have the same type.
In C++ this would be a `std::map` with string keys.
- **record**: a fixed set of attributes each with their own type.
In C++, this would be represented by a `struct`.
It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes.
For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type, and dictionary items with a colliding name can not be represented anymore.
This leads to the following guidelines:
- The top-level (root) value must be a record.
Otherwise, one can not change the structure of a command's output.
- The value of a dictionary item must be a record.
Otherwise, the item type can not be extended.
- List items should be records.
Otherwise, one can not change the structure of the list items.
If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead.
If the order of the items needs to be preserved, return a list of records.
- Streaming JSON should return records.
An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value.
These JSON values can be considered top-level values or list items, and they must be records.
### Examples
This is bad, because all keys must be assumed to be store types:
```json
{
"local": { ... },
"remote": { ... },
"http": { ... }
}
```
This is good, because the it is extensible at the root, and is somewhat self-documenting:
```json
{
"storeTypes": { "local": { ... }, ... },
"pluginSupport": true
}
```
While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information.
For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing.
The following representation is bad because it is not extensible:
```json
{ "outputs": [ "out" "bin" ] }
```
However, simply converting everything to records is not enough, because the order of outputs must be preserved:
```json
{ "outputs": { "bin": {}, "out": {} } }
```
The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work.
While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries.
This representation is extensible and preserves the ordering:
```json
{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] }
```
## Self-describing values
As described in the previous section, it's crucial that schemas can be extended with with new fields without breaking compatibility.
However, that should *not* mean we use the presence/absence of fields to indicate optional information *within* a version of the schema.
Instead, always include the field, and use `null` to indicate the "nothing" case.
### Examples
Here are two JSON objects:
```json
{
"foo": {}
}
```
```json
{
"foo": {},
"bar": {}
}
```
Since they differ in which fields they contain, they should *not* both be valid values of the same schema.
At most, they can match two different schemas where the second (with `foo` and `bar`) is considered a newer version of the first (with just `foo`).
Within each version, all fields are mandatory (always `foo`, and always `foo` and `bar`).
Only *between* each version, `bar` gets added as a new mandatory field.
Here are another two JSON objects:
```json
{ "foo": null }
```
```json
{ "foo": { "bar": 1 } }
```
Since they both contain a `foo` field, they could be valid values of the same schema.
The schema would have `foo` has an optional field, which is either `null` or an object where `bar` is an integer.

8
doc/manual/src/contributing/testing.md
View file

@ -60,7 +60,7 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks.
> ```
The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`.
Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`.
Given an interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`.
Data for unit tests is stored in a `data` subdir of the directory for each unit test executable.
For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`.
@ -162,14 +162,14 @@ ran test tests/functional/${testName}.sh... [PASS]
or without `make`:
```shell-session
$ ./mk/run-test.sh tests/functional/${testName}.sh tests/functional/init.sh
$ ./mk/run-test.sh tests/functional/${testName}.sh
ran test tests/functional/${testName}.sh... [PASS]
```
To see the complete output, one can also run:
```shell-session
$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh
$ ./mk/debug-test.sh tests/functional/${testName}.sh
+(${testName}.sh:1) foo
output from foo
+(${testName}.sh:2) bar
@ -204,7 +204,7 @@ edit it like so:
Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point:
```shell-session
$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh
$ ./mk/debug-test.sh tests/functional/${testName}.sh
...
+ gdb blash blub
GNU gdb (GDB) 12.1

28
doc/manual/src/glossary.md
View file

@ -1,5 +1,24 @@
# Glossary
- [content address]{#gloss-content-address}
A
[*content address*](https://en.wikipedia.org/wiki/Content-addressable_storage)
is a secure way to reference immutable data.
The reference is calculated directly from the content of the data being referenced, which means the reference is
[*tamper proof*](https://en.wikipedia.org/wiki/Tamperproofing)
--- variations of the data should always calculate to distinct content addresses.
For how Nix uses content addresses, see:
- [Content-Addressing File System Objects](@docroot@/store/file-system-object/content-address.md)
- [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md)
- [content-addressed derivation](#gloss-content-addressed-derivation)
Software Heritage's writing on [*Intrinsic and Extrinsic identifiers*](https://www.softwareheritage.org/2020/07/09/intrinsic-vs-extrinsic-identifiers) is also a good introduction to the value of content-addressing over other referencing schemes.
Besides content addressing, the Nix store also uses [input addressing](#gloss-input-addressed-store-object).
- [derivation]{#gloss-derivation}
A description of a build task. The result of a derivation is a
@ -118,9 +137,12 @@
- [content-addressed store object]{#gloss-content-addressed-store-object}
A [store object] whose [store path] is determined by its contents.
A [store object] which is [content-addressed](#gloss-content-address),
i.e. whose [store path] is determined by its contents.
This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation).
See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details.
- [substitute]{#gloss-substitute}
A substitute is a command invocation stored in the [Nix database] that
@ -266,13 +288,15 @@
See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details.
- [NAR]{#gloss-nar}
- [Nix Archive (NAR)]{#gloss-nar}
A *N*ix *AR*chive. This is a serialisation of a path in the Nix
store. It can contain regular files, directories and symbolic
links. NARs are generated and unpacked using `nix-store --dump`
and `nix-store --restore`.
See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details.
- [`∅`]{#gloss-emtpy-set}
The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile.

45
doc/manual/src/language/advanced-attributes.md
View file

@ -197,33 +197,40 @@ Derivations can declare some infrequently used optional attributes.
`outputHashAlgo` can only be `null` when `outputHash` follows the SRI format.
The `outputHashMode` attribute determines how the hash is computed.
It must be one of the following two values:
It must be one of the following values:
- `"flat"`\
The output must be a non-executable regular file. If it isn’t,
the build fails. The hash is simply computed over the contents
of that file (so it’s equal to what Unix commands like
`sha256sum` or `sha1sum` produce).
- [`"flat"`](@docroot@/store/store-object/content-address.md#method-flat)
This is the default.
- `"recursive"` or `"nar"`\
The hash is computed over the [NAR archive](@docroot@/glossary.md#gloss-nar) dump of the output
(i.e., the result of [`nix-store --dump`](@docroot@/command-ref/nix-store/dump.md)). In
this case, the output can be anything, including a directory
tree.
- [`"recursive"` or `"nar"`](@docroot@/store/store-object/content-address.md#method-nix-archive)
`"recursive"` is the traditional way of indicating this,
and is supported since 2005 (virtually the entire history of Nix).
`"nar"` is more clear, and consistent with other parts of Nix (such as the CLI),
however support for it is only added in Nix version 2.21.
> **Compatibility**
>
> `"recursive"` is the traditional way of indicating this,
> and is supported since 2005 (virtually the entire history of Nix).
> `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI),
> however support for it is only added in Nix version 2.21.
- [`"text"`](@docroot@/store/store-object/content-address.md#method-text)
> **Warning**
>
> The use of this method for derivation outputs is part of the [`dynamic-derivations`][xp-feature-dynamic-derivations] experimental feature.
- [`"git"`](@docroot@/store/store-object/content-address.md#method-git)
> **Warning**
>
> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature.
- [`__contentAddressed`]{#adv-attr-__contentAddressed}
> **Warning**
> This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md).
>
> To use this attribute, you must enable the
> [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature.
> [`ca-derivations`][xp-feature-ca-derivations] experimental feature.
> For example, in [nix.conf](../command-ref/conf-file.md) you could add:
>
> ```
@ -303,7 +310,7 @@ Derivations can declare some infrequently used optional attributes.
[`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites),
the following attributes are available:
- `maxSize` defines the maximum size of the resulting [store object](@docroot@/glossary.md#gloss-store-object).
- `maxSize` defines the maximum size of the resulting [store object](@docroot@/store/store-object.md).
- `maxClosureSize` defines the maximum size of the output's closure.
- `ignoreSelfRefs` controls whether self-references should be considered when
checking for allowed references/requisites.
@ -355,3 +362,7 @@ Derivations can declare some infrequently used optional attributes.
```
ensures that the derivation can only be built on a machine with the `kvm` feature.
[xp-feature-ca-derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations
[xp-feature-dynamic-derivations]: @docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations
[xp-feature-git-hashing]: @docroot@/contributing/experimental-features.md#xp-feature-git-hashing

59
doc/manual/src/language/constructs.md
View file

@ -414,12 +414,62 @@ Does evaluate to `"inner"`.
## Comments
Comments can be single-line, started with a `#` character, or
inline/multi-line, enclosed within `/* ... */`.
- Inline comments start with `#` and run until the end of the line.
`#` comments last until the end of the line.
> **Example**
>
> ```nix
> # A number
> 2 # Equals 1 + 1
> ```
>
> ```console
> 2
> ```
`/*` comments run until the next occurrence of `*/`; this cannot be escaped.
- Block comments start with `/*` and run until the next occurrence of `*/`.
> **Example**
>
> ```nix
> /*
> Block comments
> can span multiple lines.
> */ "hello"
> ```
>
> ```console
> "hello"
> ```
This means that block comments cannot be nested.
> **Example**
>
> ```nix
> /* /* nope */ */ 1
> ```
>
> ```console
> error: syntax error, unexpected '*'
>
> at «string»:1:15:
>
> 1| /* /* nope */ *
> | ^
> ```
Consider escaping nested comments and unescaping them in post-processing.
> **Example**
>
> ```nix
> /* /* nested *\/ */ 1
> ```
>
> ```console
> 1
> ```
## Scoping rules
@ -432,6 +482,5 @@ Nix is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science
* secondary scope --- implicitly-bound variables
* [`with`](#with-expressions)
Primary scope takes precedence over secondary scope.
See [`with`](#with-expressions) for a detailed example.

4
doc/manual/src/language/derivations.md
View file

@ -17,7 +17,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
A symbolic name for the derivation.
It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path).
[store path]: @docroot@/glossary.md#gloss-store-path
[store path]: @docroot@/store/store-path.md
> **Example**
>
@ -141,7 +141,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
By default, a derivation produces a single output called `out`.
However, derivations can produce multiple outputs.
This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately.
This allows the associated [store objects](@docroot@/store/store-object.md) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately.
> **Example**
>

4
doc/manual/src/language/import-from-derivation.md
View file

@ -2,9 +2,9 @@
The value of a Nix expression can depend on the contents of a [store object].
[store object]: @docroot@/glossary.md#gloss-store-object
[store object]: @docroot@/store/store-object.md
Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD):
Passing an expression `expr` that evaluates to a [store path](@docroot@/store/store-path.md) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD):
- [`import`](./builtins.md#builtins-import)` expr`
- [`builtins.readFile`](./builtins.md#builtins-readFile)` expr`

4
doc/manual/src/language/operators.md
View file

@ -128,7 +128,7 @@ The result is a string.
> The file or directory at *path* must exist and is copied to the [store].
> The path appears in the result as the corresponding [store path].
[store path]: @docroot@/glossary.md#gloss-store-path
[store path]: @docroot@/store/store-path.md
[store]: @docroot@/glossary.md#gloss-store
[String and path concatenation]: #string-and-path-concatenation
@ -141,7 +141,7 @@ The result is a string.
Update [attribute set] *attrset1* with names and values from *attrset2*.
The returned attribute set will have of all the attributes in *attrset1* and *attrset2*.
The returned attribute set will have all of the attributes in *attrset1* and *attrset2*.
If an attribute name is present in both, the attribute value from the latter is taken.
[Update]: #update

4
doc/manual/src/language/string-interpolation.md
View file

@ -107,9 +107,9 @@ An expression that is interpolated must evaluate to one of the following:
A string interpolates to itself.
A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](@docroot@/glossary.md#gloss-store-object).
A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](@docroot@/store/store-object.md).
[store path]: @docroot@/glossary.md#gloss-store-path
[store path]: @docroot@/store/store-path.md
> **Example**
>

2
doc/manual/src/language/values.md
View file

@ -124,7 +124,7 @@
For example, assume you used a file path in an interpolated string during a `nix repl` session.
Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. Use `:r` to reset the repl as needed.
[store path]: @docroot@/glossary.md#gloss-store-path
[store path]: @docroot@/store/store-path.md
Path literals can also include [string interpolation], besides being [interpolated into other expressions].

34
doc/manual/src/package-management/copy-closure.md
View file

@ -1,34 +0,0 @@
# Copying Closures via SSH
The command `nix-copy-closure` copies a Nix store path along with all
its dependencies to or from another machine via the SSH protocol. It
doesn’t copy store paths that are already present on the target machine.
For example, the following command copies Firefox with all its
dependencies:
$ nix-copy-closure --to alice@itchy.example.org $(type -p firefox)
See the [manpage for `nix-copy-closure`](../command-ref/nix-copy-closure.md) for details.
With `nix-store
--export` and `nix-store --import` you can write the closure of a store
path (that is, the path and all its dependencies) to a file, and then
unpack that file into another Nix store. For example,
$ nix-store --export $(nix-store --query --requisites $(type -p firefox)) > firefox.closure
writes the closure of Firefox to a file. You can then copy this file to
another machine and install the closure:
$ nix-store --import < firefox.closure
Any store paths in the closure that are already present in the target
store are ignored. It is also possible to pipe the export into another
command, e.g. to copy and install a closure directly to/on another
machine:
$ nix-store --export $(nix-store --query --requisites $(type -p firefox)) | bzip2 | \
ssh alice@itchy.example.org "bunzip2 | nix-store --import"
However, `nix-copy-closure` is generally more efficient because it only
copies paths that are not already present in the target Nix store.

27
doc/manual/src/protocols/json/derivation.md
View file

@ -18,10 +18,30 @@ is a JSON object with the following fields:
Information about the output paths of the derivation.
This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields:
* `path`: The output path.
* `path`:
The output path, if it is known in advanced.
Otherwise, `null`.
* `method`:
For an output which will be [content addresed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen.
Valid method strings are:
- [`flat`](@docroot@/store/store-object/content-address.md#method-flat)
- [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
- [`text`](@docroot@/store/store-object/content-address.md#method-text)
- [`git`](@docroot@/store/store-object/content-address.md#method-git)
Otherwise, `null`.
* `hashAlgo`:
For fixed-output derivations, the hashing algorithm (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a NAR hash rather than a flat file hash.
For an output which will be [content addresed], the name of the hash algorithm used.
Valid algorithm strings are:
- `md5`
- `sha1`
- `sha256`
- `sha512`
* `hash`:
For fixed-output derivations, the expected content hash in base-16.
@ -32,7 +52,8 @@ is a JSON object with the following fields:
> "outputs": {
> "out": {
> "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source",
> "hashAlgo": "r:sha256",
> "method": "nar",
> "hashAlgo": "sha256",
> "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62"
> }
> }

24
doc/manual/src/protocols/json/store-object-info.md
View file

@ -24,40 +24,44 @@ Info about a [store object].
An array of [store paths][store path], possibly including this one.
* `ca` (optional):
* `ca`:
Content address of this store object's file system object, used to compute its store path.
If the store object is [content-addressed],
this is the content address of this store object's file system object, used to compute its store path.
Otherwise (i.e. if it is [input-addressed]), this is `null`.
[store path]: @docroot@/glossary.md#gloss-store-path
[store path]: @docroot@/store/store-path.md
[file system object]: @docroot@/store/file-system-object.md
[Nix Archive]: @docroot@/glossary.md#gloss-nar
[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive
## Impure fields
These are not intrinsic properties of the store object.
In other words, the same store object residing in different store could have different values for these properties.
* `deriver` (optional):
* `deriver`:
The path to the [derivation] from which this store object is produced.
If known, the path to the [derivation] from which this store object was produced.
Otherwise `null`.
[derivation]: @docroot@/glossary.md#gloss-store-derivation
* `registrationTime` (optional):
When this derivation was added to the store.
If known, when this derivation was added to the store.
Otherwise `null`.
* `ultimate` (optional):
* `ultimate`:
Whether this store object is trusted because we built it ourselves, rather than substituted a build product from elsewhere.
* `signatures` (optional):
* `signatures`:
Signatures claiming that this store object is what it claims to be.
Not relevant for [content-addressed] store objects,
but useful for [input-addressed] store objects.
[content-addressed]: @docroot@/glossary.md#gloss-content-addressed-store-object
[content-addressed]: @docroot@/store/store-object/content-address.md
[input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object
### `.narinfo` extra fields

3
doc/manual/src/protocols/nix-archive.md
View file

@ -1,9 +1,10 @@
# Nix Archive (NAR) format
This is the complete specification of the Nix Archive format.
This is the complete specification of the [Nix Archive] format.
The Nix Archive format closely follows the abstract specification of a [file system object] tree,
because it is designed to serialize exactly that data structure.
[Nix Archive]: @docroot@/store/file-system-object/content-address.md#nix-archive
[file system object]: @docroot@/store/file-system-object.md
The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings.

31
doc/manual/src/protocols/store-path.md
View file

@ -1,12 +1,14 @@
# Complete Store Path Calculation
This is the complete specification for how store paths are calculated.
This is the complete specification for how [store path]s are calculated.
The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), but must deviate for a few things such as hash functions which we treat as bidirectional for specification purposes.
Regular users do *not* need to know this information --- store paths can be treated as black boxes computed from the properties of the store objects they refer to.
But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful.
[store path](@docroot@/store/store-path.md)
## Store path proper
```ebnf
@ -34,18 +36,23 @@ where
- `type` = one of:
- ```ebnf
| "text" ( ":" store-path )*
| "text" { ":" store-path }
```
for encoded derivations written to the store.
This is for the
["Text"](@docroot@/store/store-object/content-address.md#method-text)
method of content addressing store objects.
The optional trailing store paths are the references of the store object.
- ```ebnf
| "source" ( ":" store-path )*
| "source" { ":" store-path } [ ":self" ]
```
For paths copied to the store and hashed via a [Nix Archive (NAR)] and [SHA-256][sha-256].
Just like in the text case, we can have the store objects referenced by their paths.
This is for the
["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive)
method of content addressing store objects,
if the hash algorithm is [SHA-256].
Just like in the "Text" case, we can have the store objects referenced by their paths.
Additionally, we can have an optional `:self` label to denote self reference.
- ```ebnf
@ -53,8 +60,12 @@ where
```
For either the outputs built from derivations,
paths copied to the store hashed that area single file hashed directly, or the via a hash algorithm other than [SHA-256][sha-256].
(in that case "source" is used; this is only necessary for compatibility).
or content-addressed store objects that are not using one of the two above cases.
To be explicit about the latter, that is currently these methods:
- ["Flat"](@docroot@/store/store-object/content-address.md#method-flat)
- ["Git"](@docroot@/store/store-object/content-address.md#method-git)
- ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) if the hash algorithm is not [SHA-256].
`id` is the name of the output (usually, "out").
For content-addressed store objects, `id`, is always "out".
@ -113,8 +124,8 @@ where
Note that `id` = `"out"`, regardless of the name part of the store path.
Also note that NAR + SHA-256 must not use this case, and instead must use the `type` = `"source:" ...` case.
[Nix Archive (NAR)]: @docroot@/glossary.md#gloss-NAR
[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256
[Nix Archive (NAR)]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive
[SHA-256]: https://en.m.wikipedia.org/wiki/SHA-256
### Historical Note

4
doc/manual/src/protocols/tarball-fetcher.md
View file

@ -22,7 +22,7 @@ Link: <flakeref>; rel="immutable"
*flakeref* must be a tarball flakeref. It can contain the tarball flake attributes
`narHash`, `rev`, `revCount` and `lastModified`. If `narHash` is included, its
value must be the NAR hash of the unpacked tarball (as computed via
value must be the [NAR hash][Nix Archive] of the unpacked tarball (as computed via
`nix hash path`). Nix checks the contents of the returned tarball
against the `narHash` attribute. The `rev` and `revCount` attributes
are useful when the tarball flake is a mirror of a fetcher type that
@ -40,3 +40,5 @@ Link: <https://example.org/hello/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz
For tarball flakes, the value of the `lastModified` flake attribute is
defined as the timestamp of the newest file inside the tarball.
[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive

85
doc/manual/src/store/file-system-object/content-address.md Normal file
View file

@ -0,0 +1,85 @@
# Content-Addressing File System Objects
For many operations, Nix needs to calculate [a content addresses](@docroot@/glossary.md#gloss-content-address) of [a file system object][file system object].
Usually this is needed as part of
[content addressing store objects](../store-object/content-address.md),
since store objects always have a root file system object.
But some command-line utilities also just work on "raw" file system objects, not part of any store object.
Every content addressing scheme Nix uses ultimately involves feeding data into a [hash function](https://en.wikipedia.org/wiki/Hash_function), and getting back an opaque fixed-size digest which is deemed a content address.
The various *methods* of content addressing thus differ in how abstract data (in this case, a file system object and its descendents) are fed into the hash function.
## Serialising File System Objects { #serial }
The simplest method is to serialise the entire file system object tree into a single binary string, and then hash that binary string, yielding the content address.
In this section we describe the currently-supported methods of serialising file system objects.
### Flat { #serial-flat }
A single file object can just be hashed by its contents.
This is not enough information to encode the fact that the file system object is a file,
but if we *already* know that the FSO is a single non-executable file by other means, it is sufficient.
Because the hashed data is just the raw file, as is, this choice is good for compatibility with other systems.
For example, Unix commands like `sha256sum` or `sha1sum` will produce hashes for single files that match this.
### Nix Archive (NAR) { #serial-nix-archive }
For the other cases of [file system objects][file system object], especially directories with arbitrary descendents, we need a more complex serialisation format.
Examples of such serialisations are the ZIP and TAR file formats.
However, for our purposes these formats have two problems:
- They do not have a canonical serialisation, meaning that given an FSO, there can
be many different serialisations.
For instance, TAR files can have variable amounts of padding between archive members;
and some archive formats leave the order of directory entries undefined.
This would be bad because we use serialisation to compute cryptographic hashes over file system objects, and for those hashes to be useful as a content address or for integrity checking, uniqueness is crucial.
Otherwise, correct hashes would report false mismatches, and the store would fail to find the content.
- They store more information than we have in our notion of FSOs, such as time stamps.
This can cause FSOs that Nix should consider equal to hash to different values on different machines, just because the dates differ.
- As a practical consideration, the TAR format is the only truly universal format in the Unix environment.
It has many problems, such as an inability to deal with long file names and files larger than 2^33 bytes.
Current implementations such as GNU Tar work around these limitations in various ways.
For these reasons, Nix has its very own archive format—the Nix Archive (NAR) format,
which is carefully designed to avoid the problems described above.
The exact specification of the Nix Archive format is in `protocols/nix-archive.md`
## Content addressing File System Objects beyond a single serialisation pass
Serialising the entire tree and then hashing that binary string is not the only option for content addressing, however.
Another technique is that of a [Merkle graph](https://en.wikipedia.org/wiki/Merkle_tree), where previously computed hashes are included in subsequent byte strings to be hashed.
In particular, the Merkle graphs can match the original graph structure of file system objects:
we can first hash (serialised) child file system objects, and then hash parent objects using the hashes of their children in the serialisation (to be hashed) of the parent file system objects.
Currently, there is one such Merkle DAG content addressing method supported.
### Git ([experimental][xp-feature-git-hashing]) { #git }
> **Warning**
>
> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature.
Git's file system model is very close to Nix's, and so Git's content addressing method is a pretty good fit.
Just as with regular Git, files and symlinks are hashed as git "blobs", and directories are hashed as git "trees".
However, one difference between Nix's and Git's file system model needs special treatment.
Plain files, executable files, and symlinks are not differentiated as distinctly addressable objects, but by their context: by the directory entry that refers to them.
That means so long as the root object is a directory, there is no problem:
every non-directory object is owned by a parent directory, and the entry that refers to it provides the missing information.
However, if the root object is not a directory, then we have no way of knowing which one of an executable file, non-executable file, or symlink it is supposed to be.
In response to this, we have decided to treat a bare file as non-executable file.
This is similar to do what we do with [flat serialisation](#serial-flat), which also lacks this information.
To avoid an address collision, attempts to hash a bare executable file or symlink will result in an error (just as would happen for flat serialisation also).
Thus, Git can encode some, but not all of Nix's "File System Objects", and this sort of content-addressing is likewise partial.
In the future, we may support a Git-like hash for such file system objects, or we may adopt another Merkle DAG format which is capable of representing all Nix file system objects.
[file system object]: ../file-system-object.md
[store object]: ../store-object.md
[xp-feature-git-hashing]: @docroot@/contributing/experimental-features.md#xp-feature-git-hashing

95
doc/manual/src/store/store-object/content-address.md Normal file
View file

@ -0,0 +1,95 @@
# Content-Addressing Store Objects
Just [like][fso-ca] [File System Objects][File System Object],
[Store Objects][Store Object] can also be [content-addressed](@docroot@/glossary.md#gloss-content-addressed),
unless they are [input-addressed](@docroot@/glossary.md#gloss-input-addressed-store-object).
For store objects, the content address we produce will take the form of a [Store Path] rather than regular hash.
In particular, the content-addressing scheme will ensure that the digest of the store path is solely computed from the
- file system object graph (the root one and its children, if it has any)
- references
- [store directory](../store-path.md#store-directory)
- name
of the store object, and not any other information, which would not be an intrinsic property of that store object.
For the full specification of the algorithms involved, see the [specification of store path digests][sp-spec].
[File System Object]: ../file-system-object.md
[Store Object]: ../store-object.md
[Store Path]: ../store-path.md
## Content addressing each part of a store object
### File System Objects
With all currently supported store object content addressing methods, the file system object is always [content-addressed][fso-ca] first, and then that hash is incorporated into content address computation for the store object.
### References
With all currently supported store object content addressing methods,
other objects are referred to by their regular (string-encoded-) [store paths][Store Path].
Self-references however cannot be referred to by their path, because we are in the midst of describing how to compute that path!
> The alternative would require finding as hash function fixed point, i.e. the solution to an equation in the form
> ```
> digest = hash(..... || digest || ....)
> ```
> which is computationally infeasible.
> As far as we know, this is equivalent to finding a hash collision.
Instead we just have a "has self reference" boolean, which will end up affecting the digest.
### Name and Store Directory
These two items affect the digest in a way that is standard for store path digest computations and not specific to content-addressing.
Consult the [specification of store path digests][sp-spec] for further details.
## Content addressing Methods
For historical reasons, we don't support all features in all combinations.
Each currently supported method of content addressing chooses a single method of file system object hashing, and may offer some restrictions on references.
The names and store directories are unrestricted however.
### Flat { #method-flat }
This uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing.
References are not supported: store objects with flat hashing *and* references can not be created.
### Text { #method-text }
This also uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing.
References to other store objects are supported, but self references are not.
This is the only store-object content-addressing method that is not named identically with a corresponding file system object method.
It is somewhat obscure, mainly used for "drv files"
(derivations serialized as store objects in their ["ATerm" file format](@docroot@/protocols/derivation-aterm.md)).
Prefer another method if possible.
### Nix Archive { #method-nix-archive }
This uses the corresponding [Nix Archive](../file-system-object/content-address.md#serial-nix-archive) method of file system object content addressing.
References (to other store objects and self references alike) are supported so long as the hash algorithm is SHA-256, but not (neither kind) otherwise.
### Git { #method-git }
> **Warning**
>
> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature.
This uses the corresponding [Git](../file-system-object/content-address.md#serial-git) method of file system object content addressing.
References are not supported.
Only SHA-1 is supported at this time.
If [SHA-256-based Git](https://git-scm.com/docs/hash-function-transition)
becomes more widespread, this restriction will be revisited.
[fso-ca]: ../file-system-object/content-address.md
[sp-spec]: @docroot@/protocols/store-path.md
[xp-feature-git-hashing]: @docroot@/contributing/experimental-features.md#xp-feature-git-hashing

10
doc/manual/src/store/store-path.md
View file

@ -1,5 +1,11 @@
# Store Path
> **Example**
>
> `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1`
>
> A rendered store path
Nix implements references to [store objects](./index.md#store-object) as *store paths*.
Think of a store path as an [opaque], [unique identifier]:
@ -37,6 +43,10 @@ A store path is rendered to a file system path as the concatenation of
> store directory digest name
> ```
Exactly how the digest is calculated depends on the type of store path.
Store path digests are *supposed* to be opaque, and so for most operations, it is not necessary to know the details.
That said, the manual has a full [specification of store path digests](@docroot@/protocols/store-path.md).
## Store Directory
Every [Nix store](./index.md) has a store directory.

182
flake.nix
View file

@ -104,28 +104,6 @@
cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
});
installScriptFor = tarballs:
nixpkgsFor.x86_64-linux.native.callPackage ./scripts/installer.nix {
inherit tarballs;
};
testNixVersions = pkgs: client: daemon:
pkgs.callPackage ./package.nix {
pname =
"nix-tests"
+ lib.optionalString
(lib.versionAtLeast daemon.version "2.4pre20211005" &&
lib.versionAtLeast client.version "2.4pre20211005")
"-${client.version}-against-${daemon.version}";
inherit fileset;
test-client = client;
test-daemon = daemon;
doBuild = false;
};
binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix {
inherit nix;
};
@ -232,156 +210,17 @@
# 'nix.perl-bindings' packages.
overlays.default = overlayFor (p: p.stdenv);
hydraJobs = {
# Binary package for various platforms.
build = forAllSystems (system: self.packages.${system}.nix);
shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation);
buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static);
buildCross = forAllCrossSystems (crossSystem:
lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}"));
buildNoGc = forAllSystems (system:
self.packages.${system}.nix.override { enableGC = false; }
);
buildNoTests = forAllSystems (system:
self.packages.${system}.nix.override {
doCheck = false;
doInstallCheck = false;
installUnitTests = false;
}
);
# Toggles some settings for better coverage. Windows needs these
# library combinations, and Debian build Nix with GNU readline too.
buildReadlineNoMarkdown = forAllSystems (system:
self.packages.${system}.nix.override {
enableMarkdown = false;
readlineFlavor = "readline";
}
);
# Perl bindings for various platforms.
perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings);
# Binary tarball for various platforms, containing a Nix store
# with the closure of 'nix' package, and the second half of
# the installation script.
binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native);
binaryTarballCross = lib.genAttrs ["x86_64-linux"] (system:
forAllCrossSystems (crossSystem:
hydraJobs = import ./build/hydra.nix {
inherit
inputs
binaryTarball
self.packages.${system}."nix-${crossSystem}"
nixpkgsFor.${system}.cross.${crossSystem}));
# The first half of the installation script. This is uploaded
# to https://nixos.org/nix/install. It downloads the binary
# tarball for the user's system and calls the second half of the
# installation script.
installerScript = installScriptFor [
# Native
self.hydraJobs.binaryTarball."x86_64-linux"
self.hydraJobs.binaryTarball."i686-linux"
self.hydraJobs.binaryTarball."aarch64-linux"
self.hydraJobs.binaryTarball."x86_64-darwin"
self.hydraJobs.binaryTarball."aarch64-darwin"
# Cross
self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
];
installerScriptForGHA = installScriptFor [
# Native
self.hydraJobs.binaryTarball."x86_64-linux"
self.hydraJobs.binaryTarball."x86_64-darwin"
# Cross
self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
];
# docker image with Nix inside
dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);
# Line coverage analysis.
coverage = nixpkgsFor.x86_64-linux.native.nix.override {
pname = "nix-coverage";
withCoverageChecks = true;
};
# API docs for Nix's unstable internal C++ interfaces.
internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix {
inherit fileset;
doBuild = false;
enableInternalAPIDocs = true;
};
# API docs for Nix's C bindings.
external-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix {
inherit fileset;
doBuild = false;
enableExternalAPIDocs = true;
};
# System tests.
tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // {
# Make sure that nix-env still produces the exact same result
# on a particular version of Nixpkgs.
evalNixpkgs =
let
inherit (nixpkgsFor.x86_64-linux.native) runCommand nix;
in
runCommand "eval-nixos" { buildInputs = [ nix ]; }
''
type -p nix-env
# Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593.
(
set -x
time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages
[[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]]
)
mkdir $out
'';
nixpkgsLibTests =
forAllSystems (system:
import (nixpkgs + "/lib/tests/release.nix")
{ pkgs = nixpkgsFor.${system}.native;
nixVersions = [ self.packages.${system}.nix ];
}
);
};
metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" {
pkgs = nixpkgsFor.x86_64-linux.native;
nixpkgs = nixpkgs-regression;
};
installTests = forAllSystems (system:
let pkgs = nixpkgsFor.${system}.native; in
pkgs.runCommand "install-tests" {
againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
againstCurrentUnstable =
# FIXME: temporarily disable this on macOS because of #3605.
if system == "x86_64-linux"
then testNixVersions pkgs pkgs.nix pkgs.nixUnstable
else null;
# Disabled because the latest stable version doesn't handle
# `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
# againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
} "touch $out");
installerTests = import ./tests/installer {
binaryTarballs = self.hydraJobs.binaryTarball;
inherit nixpkgsFor;
};
forAllCrossSystems
forAllSystems
lib
linux64BitSystems
nixpkgsFor
self
;
};
checks = forAllSystems (system: {
@ -393,6 +232,7 @@
in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } ''
LANG=C.UTF-8 ${pkgs.changelog-d-nix}/bin/changelog-d ${./doc/manual/rl-next} >$out
'';
repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { };
} // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
dockerImage = self.hydraJobs.dockerImage.${system};
} // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) {

334
maintainers/flake-module.nix
View file

@ -10,14 +10,12 @@
# https://flake.parts/options/pre-commit-hooks-nix.html#options
pre-commit.settings = {
hooks = {
clang-format.enable = true;
# TODO: nixfmt, https://github.com/NixOS/nixfmt/issues/153
};
clang-format = {
enable = true;
excludes = [
# We don't want to format test data
# ''tests/(?!nixos/).*\.nix''
''^tests/.*''
''^tests/unit/[^/]*/data/.*$''
# Don't format vendored code
''^src/toml11/.*''
@ -127,8 +125,8 @@
''^src/libfetchers/registry\.hh$''
''^src/libfetchers/tarball\.cc$''
''^src/libfetchers/tarball\.hh$''
''^src/libfetchers/unix/git\.cc$''
''^src/libfetchers/unix/mercurial\.cc$''
''^src/libfetchers/git\.cc$''
''^src/libfetchers/mercurial\.cc$''
''^src/libmain/common-args\.cc$''
''^src/libmain/common-args\.hh$''
''^src/libmain/loggers\.cc$''
@ -225,29 +223,29 @@
''^src/libstore/store-api\.cc$''
''^src/libstore/store-api\.hh$''
''^src/libstore/store-dir-config\.hh$''
''^src/libstore/unix/build/derivation-goal\.cc$''
''^src/libstore/unix/build/derivation-goal\.hh$''
''^src/libstore/unix/build/drv-output-substitution-goal\.cc$''
''^src/libstore/unix/build/drv-output-substitution-goal\.hh$''
''^src/libstore/unix/build/entry-points\.cc$''
''^src/libstore/unix/build/goal\.cc$''
''^src/libstore/unix/build/goal\.hh$''
''^src/libstore/build/derivation-goal\.cc$''
''^src/libstore/build/derivation-goal\.hh$''
''^src/libstore/build/drv-output-substitution-goal\.cc$''
''^src/libstore/build/drv-output-substitution-goal\.hh$''
''^src/libstore/build/entry-points\.cc$''
''^src/libstore/build/goal\.cc$''
''^src/libstore/build/goal\.hh$''
''^src/libstore/unix/build/hook-instance\.cc$''
''^src/libstore/unix/build/local-derivation-goal\.cc$''
''^src/libstore/unix/build/local-derivation-goal\.hh$''
''^src/libstore/unix/build/substitution-goal\.cc$''
''^src/libstore/unix/build/substitution-goal\.hh$''
''^src/libstore/unix/build/worker\.cc$''
''^src/libstore/unix/build/worker\.hh$''
''^src/libstore/unix/builtins/fetchurl\.cc$''
''^src/libstore/unix/builtins/unpack-channel\.cc$''
''^src/libstore/build/substitution-goal\.cc$''
''^src/libstore/build/substitution-goal\.hh$''
''^src/libstore/build/worker\.cc$''
''^src/libstore/build/worker\.hh$''
''^src/libstore/builtins/fetchurl\.cc$''
''^src/libstore/builtins/unpack-channel\.cc$''
''^src/libstore/gc\.cc$''
''^src/libstore/unix/local-overlay-store\.cc$''
''^src/libstore/unix/local-overlay-store\.hh$''
''^src/libstore/local-overlay-store\.cc$''
''^src/libstore/local-overlay-store\.hh$''
''^src/libstore/local-store\.cc$''
''^src/libstore/local-store\.hh$''
''^src/libstore/unix/lock\.cc$''
''^src/libstore/unix/lock\.hh$''
''^src/libstore/unix/user-lock\.cc$''
''^src/libstore/unix/user-lock\.hh$''
''^src/libstore/optimise-store\.cc$''
''^src/libstore/unix/pathlocks\.cc$''
''^src/libstore/posix-fs-canonicalise\.cc$''
@ -426,9 +424,295 @@
''^src/nix/upgrade-nix\.cc$''
''^src/nix/verify\.cc$''
''^src/nix/why-depends\.cc$''
''^tests/functional/plugins/plugintest\.cc''
''^tests/functional/test-libstoreconsumer/main\.cc''
''^tests/nixos/ca-fd-leak/sender\.c''
''^tests/nixos/ca-fd-leak/smuggler\.c''
''^tests/unit/libexpr-support/tests/libexpr\.hh''
''^tests/unit/libexpr-support/tests/value/context\.cc''
''^tests/unit/libexpr-support/tests/value/context\.hh''
''^tests/unit/libexpr/derived-path\.cc''
''^tests/unit/libexpr/error_traces\.cc''
''^tests/unit/libexpr/eval\.cc''
''^tests/unit/libexpr/flake/flakeref\.cc''
''^tests/unit/libexpr/flake/url-name\.cc''
''^tests/unit/libexpr/json\.cc''
''^tests/unit/libexpr/main\.cc''
''^tests/unit/libexpr/primops\.cc''
''^tests/unit/libexpr/search-path\.cc''
''^tests/unit/libexpr/trivial\.cc''
''^tests/unit/libexpr/value/context\.cc''
''^tests/unit/libexpr/value/print\.cc''
''^tests/unit/libfetchers/public-key\.cc''
''^tests/unit/libstore-support/tests/derived-path\.cc''
''^tests/unit/libstore-support/tests/derived-path\.hh''
''^tests/unit/libstore-support/tests/libstore\.hh''
''^tests/unit/libstore-support/tests/nix_api_store\.hh''
''^tests/unit/libstore-support/tests/outputs-spec\.cc''
''^tests/unit/libstore-support/tests/outputs-spec\.hh''
''^tests/unit/libstore-support/tests/path\.cc''
''^tests/unit/libstore-support/tests/path\.hh''
''^tests/unit/libstore-support/tests/protocol\.hh''
''^tests/unit/libstore/common-protocol\.cc''
''^tests/unit/libstore/content-address\.cc''
''^tests/unit/libstore/derivation\.cc''
''^tests/unit/libstore/derived-path\.cc''
''^tests/unit/libstore/downstream-placeholder\.cc''
''^tests/unit/libstore/machines\.cc''
''^tests/unit/libstore/nar-info-disk-cache\.cc''
''^tests/unit/libstore/nar-info\.cc''
''^tests/unit/libstore/outputs-spec\.cc''
''^tests/unit/libstore/path-info\.cc''
''^tests/unit/libstore/path\.cc''
''^tests/unit/libstore/serve-protocol\.cc''
''^tests/unit/libstore/worker-protocol\.cc''
''^tests/unit/libutil-support/tests/characterization\.hh''
''^tests/unit/libutil-support/tests/hash\.cc''
''^tests/unit/libutil-support/tests/hash\.hh''
''^tests/unit/libutil/args\.cc''
''^tests/unit/libutil/canon-path\.cc''
''^tests/unit/libutil/chunked-vector\.cc''
''^tests/unit/libutil/closure\.cc''
''^tests/unit/libutil/compression\.cc''
''^tests/unit/libutil/config\.cc''
''^tests/unit/libutil/file-content-address\.cc''
''^tests/unit/libutil/git\.cc''
''^tests/unit/libutil/hash\.cc''
''^tests/unit/libutil/hilite\.cc''
''^tests/unit/libutil/json-utils\.cc''
''^tests/unit/libutil/logging\.cc''
''^tests/unit/libutil/lru-cache\.cc''
''^tests/unit/libutil/pool\.cc''
''^tests/unit/libutil/references\.cc''
''^tests/unit/libutil/suggestions\.cc''
''^tests/unit/libutil/tests\.cc''
''^tests/unit/libutil/url\.cc''
''^tests/unit/libutil/xml-writer\.cc''
];
};
shellcheck = {
enable = true;
excludes = [
# We haven't linted these files yet
''^config/install-sh$''
''^misc/systemv/nix-daemon$''
''^misc/bash/completion\.sh$''
''^misc/fish/completion\.fish$''
''^misc/zsh/completion\.zsh$''
''^scripts/check-hydra-status\.sh$''
''^scripts/create-darwin-volume\.sh$''
''^scripts/install-darwin-multi-user\.sh$''
''^scripts/install-multi-user\.sh$''
''^scripts/install-nix-from-closure\.sh$''
''^scripts/install-systemd-multi-user\.sh$''
''^src/nix/get-env\.sh$''
''^tests/functional/bash-profile\.sh$''
''^tests/functional/binary-cache-build-remote\.sh$''
''^tests/functional/binary-cache\.sh$''
''^tests/functional/brotli\.sh$''
''^tests/functional/build-delete\.sh$''
''^tests/functional/build-dry\.sh$''
''^tests/functional/build\.sh$''
''^tests/functional/ca/build-cache\.sh$''
''^tests/functional/ca/build-dry\.sh$''
''^tests/functional/ca/build-with-garbage-path\.sh$''
''^tests/functional/ca/build\.sh$''
''^tests/functional/ca/common\.sh$''
''^tests/functional/ca/concurrent-builds\.sh$''
''^tests/functional/ca/derivation-json\.sh$''
''^tests/functional/ca/duplicate-realisation-in-closure\.sh$''
''^tests/functional/ca/eval-store\.sh$''
''^tests/functional/ca/gc\.sh$''
''^tests/functional/ca/import-derivation\.sh$''
''^tests/functional/ca/new-build-cmd\.sh$''
''^tests/functional/ca/nix-copy\.sh$''
''^tests/functional/ca/nix-run\.sh$''
''^tests/functional/ca/nix-shell\.sh$''
''^tests/functional/ca/post-hook\.sh$''
''^tests/functional/ca/recursive\.sh$''
''^tests/functional/ca/repl\.sh$''
''^tests/functional/ca/selfref-gc\.sh$''
''^tests/functional/ca/signatures\.sh$''
''^tests/functional/ca/substitute\.sh$''
''^tests/functional/ca/why-depends\.sh$''
''^tests/functional/case-hack\.sh$''
''^tests/functional/check-refs\.sh$''
''^tests/functional/check-reqs\.sh$''
''^tests/functional/check\.sh$''
''^tests/functional/chroot-store\.sh$''
''^tests/functional/common/vars-and-functions\.sh$''
''^tests/functional/completions\.sh$''
''^tests/functional/compression-levels\.sh$''
''^tests/functional/compute-levels\.sh$''
''^tests/functional/config\.sh$''
''^tests/functional/db-migration\.sh$''
''^tests/functional/debugger\.sh$''
''^tests/functional/dependencies\.builder0\.sh$''
''^tests/functional/dependencies\.sh$''
''^tests/functional/derivation-json\.sh$''
''^tests/functional/dump-db\.sh$''
''^tests/functional/dyn-drv/build-built-drv\.sh$''
''^tests/functional/dyn-drv/common\.sh$''
''^tests/functional/dyn-drv/dep-built-drv\.sh$''
''^tests/functional/dyn-drv/eval-outputOf\.sh$''
''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$''
''^tests/functional/dyn-drv/recursive-mod-json\.sh$''
''^tests/functional/dyn-drv/text-hashed-output\.sh$''
''^tests/functional/eval-store\.sh$''
''^tests/functional/eval\.sh$''
''^tests/functional/experimental-features\.sh$''
''^tests/functional/export-graph\.sh$''
''^tests/functional/export\.sh$''
''^tests/functional/extra-sandbox-profile\.sh$''
''^tests/functional/fetchClosure\.sh$''
''^tests/functional/fetchGit\.sh$''
''^tests/functional/fetchGitRefs\.sh$''
''^tests/functional/fetchGitSubmodules\.sh$''
''^tests/functional/fetchGitVerification\.sh$''
''^tests/functional/fetchMercurial\.sh$''
''^tests/functional/fetchPath\.sh$''
''^tests/functional/fetchTree-file\.sh$''
''^tests/functional/fetchurl\.sh$''
''^tests/functional/filter-source\.sh$''
''^tests/functional/fixed\.builder1\.sh$''
''^tests/functional/fixed\.builder2\.sh$''
''^tests/functional/fixed\.sh$''
''^tests/functional/flakes/absolute-attr-paths\.sh$''
''^tests/functional/flakes/absolute-paths\.sh$''
''^tests/functional/flakes/build-paths\.sh$''
''^tests/functional/flakes/bundle\.sh$''
''^tests/functional/flakes/check\.sh$''
''^tests/functional/flakes/circular\.sh$''
''^tests/functional/flakes/common\.sh$''
''^tests/functional/flakes/config\.sh$''
''^tests/functional/flakes/develop\.sh$''
''^tests/functional/flakes/flake-in-submodule\.sh$''
''^tests/functional/flakes/flakes\.sh$''
''^tests/functional/flakes/follow-paths\.sh$''
''^tests/functional/flakes/init\.sh$''
''^tests/functional/flakes/inputs\.sh$''
''^tests/functional/flakes/mercurial\.sh$''
''^tests/functional/flakes/prefetch\.sh$''
''^tests/functional/flakes/run\.sh$''
''^tests/functional/flakes/search-root\.sh$''
''^tests/functional/flakes/show\.sh$''
''^tests/functional/flakes/unlocked-override\.sh$''
''^tests/functional/fmt\.sh$''
''^tests/functional/fmt\.simple\.sh$''
''^tests/functional/function-trace\.sh$''
''^tests/functional/gc-auto\.sh$''
''^tests/functional/gc-concurrent\.builder\.sh$''
''^tests/functional/gc-concurrent\.sh$''
''^tests/functional/gc-concurrent2\.builder\.sh$''
''^tests/functional/gc-non-blocking\.sh$''
''^tests/functional/gc-runtime\.sh$''
''^tests/functional/gc\.sh$''
''^tests/functional/git-hashing/common\.sh$''
''^tests/functional/git-hashing/simple\.sh$''
''^tests/functional/hash-convert\.sh$''
''^tests/functional/hash-path\.sh$''
''^tests/functional/help\.sh$''
''^tests/functional/import-derivation\.sh$''
''^tests/functional/impure-derivations\.sh$''
''^tests/functional/impure-env\.sh$''
''^tests/functional/impure-eval\.sh$''
''^tests/functional/install-darwin\.sh$''
''^tests/functional/lang-test-infra\.sh$''
''^tests/functional/lang\.sh$''
''^tests/functional/lang/framework\.sh$''
''^tests/functional/legacy-ssh-store\.sh$''
''^tests/functional/linux-sandbox\.sh$''
''^tests/functional/local-overlay-store/add-lower-inner\.sh$''
''^tests/functional/local-overlay-store/add-lower\.sh$''
''^tests/functional/local-overlay-store/bad-uris\.sh$''
''^tests/functional/local-overlay-store/build-inner\.sh$''
''^tests/functional/local-overlay-store/build\.sh$''
''^tests/functional/local-overlay-store/check-post-init-inner\.sh$''
''^tests/functional/local-overlay-store/check-post-init\.sh$''
''^tests/functional/local-overlay-store/common\.sh$''
''^tests/functional/local-overlay-store/delete-duplicate-inner\.sh$''
''^tests/functional/local-overlay-store/delete-duplicate\.sh$''
''^tests/functional/local-overlay-store/delete-refs-inner\.sh$''
''^tests/functional/local-overlay-store/delete-refs\.sh$''
''^tests/functional/local-overlay-store/gc-inner\.sh$''
''^tests/functional/local-overlay-store/gc\.sh$''
''^tests/functional/local-overlay-store/optimise-inner\.sh$''
''^tests/functional/local-overlay-store/optimise\.sh$''
''^tests/functional/local-overlay-store/redundant-add-inner\.sh$''
''^tests/functional/local-overlay-store/redundant-add\.sh$''
''^tests/functional/local-overlay-store/remount\.sh$''
''^tests/functional/local-overlay-store/stale-file-handle-inner\.sh$''
''^tests/functional/local-overlay-store/stale-file-handle\.sh$''
''^tests/functional/local-overlay-store/verify-inner\.sh$''
''^tests/functional/local-overlay-store/verify\.sh$''
''^tests/functional/logging\.sh$''
''^tests/functional/misc\.sh$''
''^tests/functional/multiple-outputs\.sh$''
''^tests/functional/nar-access\.sh$''
''^tests/functional/nested-sandboxing\.sh$''
''^tests/functional/nested-sandboxing/command\.sh$''
''^tests/functional/nix-build\.sh$''
''^tests/functional/nix-channel\.sh$''
''^tests/functional/nix-collect-garbage-d\.sh$''
''^tests/functional/nix-copy-ssh-common\.sh$''
''^tests/functional/nix-copy-ssh-ng\.sh$''
''^tests/functional/nix-copy-ssh\.sh$''
''^tests/functional/nix-daemon-untrusting\.sh$''
''^tests/functional/nix-profile\.sh$''
''^tests/functional/nix-shell\.sh$''
''^tests/functional/nix_path\.sh$''
''^tests/functional/optimise-store\.sh$''
''^tests/functional/output-normalization\.sh$''
''^tests/functional/parallel\.builder\.sh$''
''^tests/functional/parallel\.sh$''
''^tests/functional/pass-as-file\.sh$''
''^tests/functional/path-from-hash-part\.sh$''
''^tests/functional/path-info\.sh$''
''^tests/functional/placeholders\.sh$''
''^tests/functional/plugins\.sh$''
''^tests/functional/post-hook\.sh$''
''^tests/functional/pure-eval\.sh$''
''^tests/functional/push-to-store-old\.sh$''
''^tests/functional/push-to-store\.sh$''
''^tests/functional/read-only-store\.sh$''
''^tests/functional/readfile-context\.sh$''
''^tests/functional/recursive\.sh$''
''^tests/functional/referrers\.sh$''
''^tests/functional/remote-store\.sh$''
''^tests/functional/repair\.sh$''
''^tests/functional/restricted\.sh$''
''^tests/functional/search\.sh$''
''^tests/functional/secure-drv-outputs\.sh$''
''^tests/functional/selfref-gc\.sh$''
''^tests/functional/shell\.sh$''
''^tests/functional/shell\.shebang\.sh$''
''^tests/functional/signing\.sh$''
''^tests/functional/simple\.builder\.sh$''
''^tests/functional/simple\.sh$''
''^tests/functional/ssh-relay\.sh$''
''^tests/functional/store-info\.sh$''
''^tests/functional/structured-attrs\.sh$''
''^tests/functional/substitute-with-invalid-ca\.sh$''
''^tests/functional/suggestions\.sh$''
''^tests/functional/supplementary-groups\.sh$''
''^tests/functional/tarball\.sh$''
''^tests/functional/test-infra\.sh$''
''^tests/functional/test-libstoreconsumer\.sh$''
''^tests/functional/timeout\.sh$''
''^tests/functional/toString-path\.sh$''
''^tests/functional/user-envs-migration\.sh$''
''^tests/functional/user-envs-test-case\.sh$''
''^tests/functional/user-envs\.builder\.sh$''
''^tests/functional/user-envs\.sh$''
''^tests/functional/why-depends\.sh$''
''^tests/functional/zstd\.sh$''
''^tests/unit/libutil/data/git/check-data\.sh$''
];
};
# TODO: nixfmt, https://github.com/NixOS/nixfmt/issues/153
};
};
};
# We'll be pulling from this in the main flake

10
mk/common-test.sh
View file

@ -1,19 +1,23 @@
# shellcheck shell=bash
# Remove overall test dir (at most one of the two should match) and
# remove file extension.
test_name=$(echo -n "$test" | sed \
test_name=$(echo -n "${test?must be defined by caller (test runner)}" | sed \
-e "s|^tests/unit/[^/]*/data/||" \
-e "s|^tests/functional/||" \
-e "s|\.sh$||" \
)
# shellcheck disable=SC2016
TESTS_ENVIRONMENT=(
"TEST_NAME=$test_name"
'NIX_REMOTE='
'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) '
)
: ${BASH:=/usr/bin/env bash}
read -r -a bash <<< "${BASH:-/usr/bin/env bash}"
run () {
cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1)
cd "$(dirname "$1")" && env "${TESTS_ENVIRONMENT[@]}" "${bash[@]}" -x -e -u -o pipefail "$(basename "$1")"
}

4
mk/debug-test.sh
View file

@ -3,12 +3,8 @@
set -eu -o pipefail
test=$1
init=${2-}
dir="$(dirname "${BASH_SOURCE[0]}")"
source "$dir/common-test.sh"
if [ -n "$init" ]; then
(run "$init" 2>/dev/null > /dev/null)
fi
run "$test"

7
mk/lib.mk
View file

@ -87,15 +87,14 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b
$(foreach script, $(bin-scripts), $(eval programs-list += $(script)))
$(foreach script, $(noinst-scripts), $(eval programs-list += $(script)))
$(foreach template, $(template-files), $(eval $(call instantiate-template,$(template))))
install_test_init=tests/functional/init.sh
$(foreach test, $(install-tests), \
$(eval $(call run-test,$(test),$(install_test_init))) \
$(eval $(call run-test,$(test))) \
$(eval installcheck: $(test).test))
$(foreach test-group, $(install-tests-groups), \
$(eval $(call run-test-group,$(test-group),$(install_test_init))) \
$(eval $(call run-test-group,$(test-group))) \
$(eval installcheck: $(test-group).test-group) \
$(foreach test, $($(test-group)-tests), \
$(eval $(call run-test,$(test),$(install_test_init))) \
$(eval $(call run-test,$(test))) \
$(eval $(test-group).test-group: $(test).test)))
# Compilation database.

9
mk/run-test.sh
View file

@ -8,7 +8,6 @@ yellow=""
normal=""
test=$1
init=${2-}
dir="$(dirname "${BASH_SOURCE[0]}")"
source "$dir/common-test.sh"
@ -22,20 +21,18 @@ if [ -t 1 ]; then
fi
run_test () {
if [ -n "$init" ]; then
(run "$init" 2>/dev/null > /dev/null)
fi
log="$(run "$test" 2>&1)" && status=0 || status=$?
}
run_test
if [ $status -eq 0 ]; then
if [[ "$status" = 0 ]]; then
echo "$post_run_msg [${green}PASS$normal]"
elif [ $status -eq 99 ]; then
elif [[ "$status" = 99 ]]; then
echo "$post_run_msg [${yellow}SKIP$normal]"
else
echo "$post_run_msg [${red}FAIL$normal]"
# shellcheck disable=SC2001
echo "$log" | sed 's/^/ /'
exit "$status"
fi

4
mk/tests.mk
View file

@ -12,8 +12,8 @@ endef
define run-test
$(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1 $2))
$(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1 $2))
$(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1))
$(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1))
endef

16
scripts/bigsur-nixbld-user-migration.sh
View file

@ -3,7 +3,7 @@
((NEW_NIX_FIRST_BUILD_UID=301))
id_available(){
dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null
dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null
}
change_nixbld_names_and_ids(){
@ -26,18 +26,18 @@ change_nixbld_names_and_ids(){
fi
done
if [[ $name == _* ]]; then
if [[ "$name" == _* ]]; then
echo " It looks like $name has already been renamed--skipping."
else
# first 3 are cleanup, it's OK if they aren't here
sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true
sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true
sudo dscl . delete "/Users/$name" dsAttrTypeNative:_writers_passwd &>/dev/null || true
sudo dscl . change "/Users/$name" NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true
# remove existing user from group
sudo dseditgroup -o edit -t user -d $name nixbld || true
sudo dscl . change /Users/$name UniqueID $uid $next_id
sudo dscl . change /Users/$name RecordName $name _$name
sudo dseditgroup -o edit -t user -d "$name" nixbld || true
sudo dscl . change "/Users/$name" UniqueID "$uid" "$next_id"
sudo dscl . change "/Users/$name" RecordName "$name" "_$name"
# add renamed user to group
sudo dseditgroup -o edit -t user -a _$name nixbld
sudo dseditgroup -o edit -t user -a "_$name" nixbld
echo " $name migrated to _$name (uid: $next_id)"
fi
done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2)

8
scripts/nix-profile-daemon.sh.in
View file

@ -1,4 +1,5 @@
# Only execute this file once per shell.
# This file is tested by tests/installer/default.nix.
if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi
__ETC_PROFILE_NIX_SOURCED=1
@ -9,11 +10,9 @@ else
NIX_LINK_NEW=$HOME/.local/state/nix/profile
fi
if [ -e "$NIX_LINK_NEW" ]; then
NIX_LINK="$NIX_LINK_NEW"
else
if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then
if [ -t 2 ] && [ -e "$NIX_LINK" ]; then
warning="\033[1;35mwarning:\033[0m"
printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2
printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2
if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then
printf " Since the profiles match, you can safely delete either of them.\n" 1>&2
else
@ -26,6 +25,7 @@ else
printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2
fi
fi
NIX_LINK="$NIX_LINK_NEW"
fi
export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK"

8
scripts/nix-profile.sh.in
View file

@ -1,3 +1,4 @@
# This file is tested by tests/installer/default.nix.
if [ -n "$HOME" ] && [ -n "$USER" ]; then
# Set up the per-user profile.
@ -9,11 +10,9 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
NIX_LINK_NEW="$HOME/.local/state/nix/profile"
fi
if [ -e "$NIX_LINK_NEW" ]; then
NIX_LINK="$NIX_LINK_NEW"
else
if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then
if [ -t 2 ] && [ -e "$NIX_LINK" ]; then
warning="\033[1;35mwarning:\033[0m"
printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2
printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2
if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then
printf " Since the profiles match, you can safely delete either of them.\n" 1>&2
else
@ -26,6 +25,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2
fi
fi
NIX_LINK="$NIX_LINK_NEW"
fi
# Set up environment.

22
src/build-remote/build-remote.cc
View file

@ -37,7 +37,7 @@ static std::string currentLoad;
static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
{
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true);
}
static bool allSupportedLocally(Store & store, const std::set<std::string>& requiredFeatures) {
@ -99,7 +99,7 @@ static int main_build_remote(int argc, char * * argv)
}
std::optional<StorePath> drvPath;
std::string storeUri;
StoreReference storeUri;
while (true) {
@ -135,7 +135,7 @@ static int main_build_remote(int argc, char * * argv)
Machine * bestMachine = nullptr;
uint64_t bestLoad = 0;
for (auto & m : machines) {
debug("considering building on remote machine '%s'", m.storeUri);
debug("considering building on remote machine '%s'", m.storeUri.render());
if (m.enabled &&
m.systemSupported(neededSystem) &&
@ -233,7 +233,7 @@ static int main_build_remote(int argc, char * * argv)
try {
Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri.render()));
sshStore = bestMachine->openStore();
sshStore->connect();
@ -242,7 +242,7 @@ static int main_build_remote(int argc, char * * argv)
} catch (std::exception & e) {
auto msg = chomp(drainFD(5, false));
printError("cannot build on '%s': %s%s",
bestMachine->storeUri, e.what(),
bestMachine->storeUri.render(), e.what(),
msg.empty() ? "" : ": " + msg);
bestMachine->enabled = false;
continue;
@ -257,15 +257,15 @@ connected:
assert(sshStore);
std::cerr << "# accept\n" << storeUri << "\n";
std::cerr << "# accept\n" << storeUri.render() << "\n";
auto inputs = readStrings<PathSet>(source);
auto wantedOutputs = readStrings<StringSet>(source);
AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri.render()) + ".upload-lock", true);
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri));
Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri.render()));
auto old = signal(SIGALRM, handleAlarm);
alarm(15 * 60);
@ -278,7 +278,7 @@ connected:
auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute;
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri.render()));
copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
}
@ -316,7 +316,7 @@ connected:
optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv);
auto & result = *optResult;
if (!result.success())
throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri.render(), result.errorMsg);
} else {
copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute);
auto res = sshStore->buildPathsWithResults({
@ -359,7 +359,7 @@ connected:
}
if (!missingPaths.empty()) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri.render()));
if (auto localStore = store.dynamic_pointer_cast<LocalStore>())
for (auto & path : missingPaths)
localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */

6
src/libcmd/common-eval-args.cc
View file

@ -20,7 +20,7 @@ MixEvalArgs::MixEvalArgs()
.description = "Pass the value *expr* as the argument *name* to Nix functions.",
.category = category,
.labels = {"name", "expr"},
.handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr(expr)}); }}
.handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }}
});
addFlag({
@ -28,7 +28,7 @@ MixEvalArgs::MixEvalArgs()
.description = "Pass the string *string* as the argument *name* to Nix functions.",
.category = category,
.labels = {"name", "string"},
.handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString(s)}); }},
.handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString{s}}); }},
});
addFlag({
@ -36,7 +36,7 @@ MixEvalArgs::MixEvalArgs()
.description = "Pass the contents of file *path* as the argument *name* to Nix functions.",
.category = category,
.labels = {"name", "path"},
.handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile(path)}); }},
.handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }},
.completer = completePath
});

2
src/libcmd/installable-attr-path.cc
View file

@ -75,6 +75,8 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
std::set<std::string> outputsToInstall;
for (auto & output : packageInfo.queryOutputs(false, true))
outputsToInstall.insert(output.first);
if (outputsToInstall.empty())
outputsToInstall.insert("out");
return OutputsSpec::Names { std::move(outputsToInstall) };
},
[&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec {

9
src/libcmd/installable-flake.cc
View file

@ -106,9 +106,14 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
fmt("while evaluating the flake output attribute '%s'", attrPath)))
{
return { *derivedPathWithInfo };
} else {
throw Error(
"expected flake output attribute '%s' to be a derivation or path but found %s: %s",
attrPath,
showType(v),
ValuePrinter(*this->state, v, errorPrintOptions)
);
}
else
throw Error("flake output attribute '%s' is not a derivation or path", attrPath);
}
auto drvPath = attr->forceDerivation();

38
src/libcmd/installables.cc
View file

@ -601,6 +601,37 @@ std::vector<BuiltPathWithResult> Installable::build(
return res;
}
static void throwBuildErrors(
std::vector<KeyedBuildResult> & buildResults,
const Store & store)
{
std::vector<KeyedBuildResult> failed;
for (auto & buildResult : buildResults) {
if (!buildResult.success()) {
failed.push_back(buildResult);
}
}
auto failedResult = failed.begin();
if (failedResult != failed.end()) {
if (failed.size() == 1) {
failedResult->rethrow();
} else {
StringSet failedPaths;
for (; failedResult != failed.end(); failedResult++) {
if (!failedResult->errorMsg.empty()) {
logError(ErrorInfo{
.level = lvlError,
.msg = failedResult->errorMsg,
});
}
failedPaths.insert(failedResult->path.to_string(store));
}
throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths)));
}
}
}
std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build2(
ref<Store> evalStore,
ref<Store> store,
@ -662,10 +693,9 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
if (settings.printMissing)
printMissing(store, pathsToBuild, lvlInfo);
for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) {
if (!buildResult.success())
buildResult.rethrow();
auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore);
throwBuildErrors(buildResults, *store);
for (auto & buildResult : buildResults) {
for (auto & aux : backmap[buildResult.path]) {
std::visit(overloaded {
[&](const DerivedPath::Built & bfd) {

26
src/libcmd/misc-store-flags.cc
View file

@ -81,9 +81,15 @@ Args::Flag fileIngestionMethod(FileIngestionMethod * method)
How to compute the hash of the input.
One of:
- `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
- `nar` (the default):
Serialises the input as a
[Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive)
and passes that to the hash function.
- `flat`: Assumes that the input is a single file and directly passes it to the hash function;
- `flat`:
Assumes that the input is a single file and
[directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat)
it to the hash function.
)",
.labels = {"file-ingestion-method"},
.handler = {[method](std::string s) {
@ -101,15 +107,23 @@ Args::Flag contentAddressMethod(ContentAddressMethod * method)
How to compute the content-address of the store object.
One of:
- `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
- [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
(the default):
Serialises the input as a
[Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive)
and passes that to the hash function.
- `flat`: Assumes that the input is a single file and directly passes it to the hash function;
- [`flat`](@docroot@/store/store-object/content-address.md#method-flat):
Assumes that the input is a single file and
[directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat)
it to the hash function.
- `text`: Like `flat`, but used for
- [`text`](@docroot@/store/store-object/content-address.md#method-text):
Like `flat`, but used for
[derivations](@docroot@/glossary.md#store-derivation) serialized in store object and
[`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile).
For advanced use-cases only;
for regular usage prefer `nar` and `flat.
for regular usage prefer `nar` and `flat`.
)",
.labels = {"content-address-method"},
.handler = {[method](std::string s) {

5
src/libcmd/network-proxy.cc
View file

@ -25,7 +25,10 @@ static StringSet getExcludingNoProxyVariables()
static const StringSet excludeVariables{"no_proxy", "NO_PROXY"};
StringSet variables;
std::set_difference(
networkProxyVariables.begin(), networkProxyVariables.end(), excludeVariables.begin(), excludeVariables.end(),
networkProxyVariables.begin(),
networkProxyVariables.end(),
excludeVariables.begin(),
excludeVariables.end(),
std::inserter(variables, variables.begin()));
return variables;
}

13
src/libcmd/repl.cc
View file

@ -137,12 +137,13 @@ void runNix(Path program, const Strings & args,
{
auto subprocessEnv = getEnv();
subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue();
//isInteractive avoid grabling interactive commands
runProgram2(RunOptions {
.program = settings.nixBinDir+ "/" + program,
.args = args,
.environment = subprocessEnv,
.input = input,
.isInteractive = true,
});
return;
@ -259,7 +260,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
try {
auto dir = std::string(cur, 0, slash);
auto prefix2 = std::string(cur, slash + 1);
for (auto & entry : readDirectory(dir == "" ? "/" : dir)) {
for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) {
auto name = entry.path().filename().string();
if (name[0] != '.' && hasPrefix(name, prefix2))
completions.insert(prev + entry.path().string());
@ -304,6 +305,8 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
// Quietly ignore evaluation errors.
} catch (BadURL & e) {
// Quietly ignore BadURL flake-related errors.
} catch (FileNotFound & e) {
// Quietly ignore non-existent file beeing `import`-ed.
}
}
@ -508,13 +511,9 @@ ProcessLineResult NixRepl::processLine(std::string line)
auto editor = args.front();
args.pop_front();
// avoid garbling the editor with the progress bar
logger->pause();
Finally resume([&]() { logger->resume(); });
// runProgram redirects stdout to a StringSink,
// using runProgram2 to allow editors to display their UI
runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args });
runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args , .isInteractive = true });
// Reload right after exiting the editor
state->resetFileCache();

11
src/libexpr-c/nix_api_expr.cc
View file

@ -65,6 +65,17 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, V
NIXC_CATCH_ERRS
}
nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, Value * fn, size_t nargs, Value ** args, Value * value)
{
if (context)
context->last_err_code = NIX_OK;
try {
state->state.callFunction(*(nix::Value *) fn, nargs, (nix::Value * *)args, *(nix::Value *) value, nix::noPos);
state->state.forceValue(*(nix::Value *) value, nix::noPos);
}
NIXC_CATCH_ERRS
}
nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value)
{
if (context)

69
src/libexpr-c/nix_api_expr.h
View file

@ -3,25 +3,7 @@
/** @defgroup libexpr libexpr
* @brief Bindings to the Nix language evaluator
*
* Example (without error handling):
* @code{.c}
* int main() {
* nix_libexpr_init(NULL);
*
* Store* store = nix_store_open(NULL, "dummy", NULL);
* EvalState* state = nix_state_create(NULL, NULL, store); // empty nix path
* Value *value = nix_alloc_value(NULL, state);
*
* nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value);
* nix_value_force(NULL, state, value);
* printf("nix version: %s\n", nix_get_string(NULL, value));
*
* nix_gc_decref(NULL, value);
* nix_state_free(state);
* nix_store_free(store);
* return 0;
* }
* @endcode
* See *[Embedding the Nix Evaluator](@ref nix_evaluator_example)* for an example.
* @{
*/
/** @file
@ -30,6 +12,7 @@
#include "nix_api_store.h"
#include "nix_api_util.h"
#include <stddef.h>
#ifdef __cplusplus
extern "C" {
@ -98,6 +81,46 @@ nix_err nix_expr_eval_from_string(
*/
nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value);
/**
* @brief Calls a Nix function with multiple arguments.
*
* Technically these are functions that return functions. It is common for Nix
* functions to be curried, so this function is useful for calling them.
*
* @param[out] context Optional, stores error information
* @param[in] state The state of the evaluation.
* @param[in] fn The Nix function to call.
* @param[in] nargs The number of arguments.
* @param[in] args The arguments to pass to the function.
* @param[out] value The result of the function call.
*
* @see nix_value_call For the single argument primitive.
* @see NIX_VALUE_CALL For a macro that wraps this function for convenience.
*/
nix_err nix_value_call_multi(
nix_c_context * context, EvalState * state, Value * fn, size_t nargs, Value ** args, Value * value);
/**
* @brief Calls a Nix function with multiple arguments.
*
* Technically these are functions that return functions. It is common for Nix
* functions to be curried, so this function is useful for calling them.
*
* @param[out] context Optional, stores error information
* @param[in] state The state of the evaluation.
* @param[out] value The result of the function call.
* @param[in] fn The Nix function to call.
* @param[in] args The arguments to pass to the function.
*
* @see nix_value_call_multi
*/
#define NIX_VALUE_CALL(context, state, value, fn, ...) \
do { \
Value * args_array[] = {__VA_ARGS__}; \
size_t nargs = sizeof(args_array) / sizeof(args_array[0]); \
nix_value_call_multi(context, state, fn, nargs, args_array, value); \
} while (0)
/**
* @brief Forces the evaluation of a Nix value.
*
@ -106,10 +129,8 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, V
*
* This function converts these Values into their final type.
*
* @note You don't need this function for basic API usage, since all functions
* that return a value call it for you. The only place you will see a
* NIX_TYPE_THUNK is in the arguments that are passed to a PrimOp function
* you supplied to nix_alloc_primop.
* @note You don't need this function for basic API usage very often, since all functions that return a `Value` call it
* for you. This function is mainly needed before calling @ref getters.
*
* @param[out] context Optional, stores error information
* @param[in] state The state of the evaluation.
@ -140,7 +161,7 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value *
* @brief Create a new Nix language evaluator state.
*
* @param[out] context Optional, stores error information
* @param[in] lookupPath Array of strings corresponding to entries in NIX_PATH.
* @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH.
* @param[in] store The Nix store to use.
* @return A new Nix state or NULL on failure.
*/

6
src/libexpr-c/nix_api_external.h
View file

@ -48,7 +48,7 @@ void nix_set_string_return(nix_string_return * str, const char * c);
* Print to the nix_printer
*
* @param[out] context Optional, stores error information
* @param printer The nix_printer to print to
* @param[out] printer The nix_printer to print to
* @param[in] str The string to print
* @returns NIX_OK if everything worked
*/
@ -136,7 +136,7 @@ typedef struct NixCExternalValueDesc
* or setting it to the empty string, will make the conversion throw an error.
*/
void (*printValueAsJSON)(
void * self, EvalState *, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res);
void * self, EvalState * state, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res);
/**
* @brief Convert the external value to XML
*
@ -155,7 +155,7 @@ typedef struct NixCExternalValueDesc
*/
void (*printValueAsXML)(
void * self,
EvalState *,
EvalState * state,
int strict,
int location,
void * doc,

41
src/libexpr-c/nix_api_value.cc
View file

@ -73,10 +73,43 @@ static void nix_c_primop_wrapper(
PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v)
{
nix_c_context ctx;
f(userdata, &ctx, (EvalState *) &state, (Value **) args, (Value *) &v);
/* TODO: In the future, this should throw different errors depending on the error code */
if (ctx.last_err_code != NIX_OK)
state.error<nix::EvalError>("Error from builtin function: %s", *ctx.last_err).atPos(pos).debugThrow();
// v currently has a thunk, but the C API initializers require an uninitialized value.
//
// We can't destroy the thunk, because that makes it impossible to retry,
// which is needed for tryEval and for evaluation drivers that evaluate more
// than one value (e.g. an attrset with two derivations, both of which
// reference v).
//
// Instead we create a temporary value, and then assign the result to v.
// This does not give the primop definition access to the thunk, but that's
// ok because we don't see a need for this yet (e.g. inspecting thunks,
// or maybe something to make blackholes work better; we don't know).
nix::Value vTmp;
f(userdata, &ctx, (EvalState *) &state, (Value **) args, (Value *) &vTmp);
if (ctx.last_err_code != NIX_OK) {
/* TODO: Throw different errors depending on the error code */
state.error<nix::EvalError>("Error from custom function: %s", *ctx.last_err).atPos(pos).debugThrow();
}
if (!vTmp.isValid()) {
state.error<nix::EvalError>("Implementation error in custom function: return value was not initialized")
.atPos(pos)
.debugThrow();
}
if (vTmp.type() == nix::nThunk) {
// We might allow this in the future if it makes sense for the evaluator
// e.g. implementing tail recursion by returning a thunk to the next
// "iteration". Until then, this is most likely a mistake or misunderstanding.
state.error<nix::EvalError>("Implementation error in custom function: return value must not be a thunk")
.atPos(pos)
.debugThrow();
}
v = vTmp;
}
PrimOp * nix_alloc_primop(

4
src/libexpr-c/nix_api_value.h
View file

@ -79,6 +79,7 @@ typedef struct nix_realised_string nix_realised_string;
* @{
*/
/** @brief Function pointer for primops
*
* When you want to return an error, call nix_set_err_msg(context, NIX_ERR_UNKNOWN, "your error message here").
*
* @param[in] user_data Arbitrary data that was initially supplied to nix_alloc_primop
@ -147,7 +148,8 @@ Value * nix_alloc_value(nix_c_context * context, EvalState * state);
* @brief Functions to inspect and change Nix language values, represented by Value.
* @{
*/
/** @name Getters
/** @anchor getters
* @name Getters
*/
/**@{*/
/** @brief Get value type

39
src/libexpr/eval-cache.cc
View file

@ -7,6 +7,25 @@
namespace nix::eval_cache {
CachedEvalError::CachedEvalError(ref<AttrCursor> cursor, Symbol attr)
: EvalError(cursor->root->state, "cached failure of attribute '%s'", cursor->getAttrPathStr(attr))
, cursor(cursor), attr(attr)
{ }
void CachedEvalError::force()
{
auto & v = cursor->forceValue();
if (v.type() == nAttrs) {
auto a = v.attrs()->get(this->attr);
state.forceValue(*a->value, a->pos);
}
// Shouldn't happen.
throw EvalError(state, "evaluation of cached failed attribute '%s' unexpectedly succeeded", cursor->getAttrPathStr(attr));
}
static const char * schema = R"sql(
create table if not exists Attributes (
parent integer not null,
@ -470,7 +489,7 @@ Suggestions AttrCursor::getSuggestionsForAttr(Symbol name)
return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]);
}
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
{
if (root->db) {
if (!cachedValue)
@ -487,12 +506,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
if (attr) {
if (std::get_if<missing_t>(&attr->second))
return nullptr;
else if (std::get_if<failed_t>(&attr->second)) {
if (forceErrors)
debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name));
else if (std::get_if<failed_t>(&attr->second))
throw CachedEvalError(ref(shared_from_this()), name);
else
throw CachedEvalError(root->state, "cached failure of attribute '%s'", getAttrPathStr(name));
} else
return std::make_shared<AttrCursor>(root,
std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
}
@ -537,9 +553,9 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
return maybeGetAttr(root->state.symbols.create(name));
}
ref<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
ref<AttrCursor> AttrCursor::getAttr(Symbol name)
{
auto p = maybeGetAttr(name, forceErrors);
auto p = maybeGetAttr(name);
if (!p)
throw Error("attribute '%s' does not exist", getAttrPathStr(name));
return ref(p);
@ -550,11 +566,11 @@ ref<AttrCursor> AttrCursor::getAttr(std::string_view name)
return getAttr(root->state.symbols.create(name));
}
OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force)
OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
{
auto res = shared_from_this();
for (auto & attr : attrPath) {
auto child = res->maybeGetAttr(attr, force);
auto child = res->maybeGetAttr(attr);
if (!child) {
auto suggestions = res->getSuggestionsForAttr(attr);
return OrSuggestions<ref<AttrCursor>>::failed(suggestions);
@ -751,8 +767,9 @@ bool AttrCursor::isDerivation()
StorePath AttrCursor::forceDerivation()
{
auto aDrvPath = getAttr(root->state.sDrvPath, true);
auto aDrvPath = getAttr(root->state.sDrvPath);
auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
drvPath.requireDerivation();
if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
/* The eval cache contains 'drvPath', but the actual path has
been garbage-collected. So force it to be regenerated. */

25
src/libexpr/eval-cache.hh
View file

@ -10,14 +10,28 @@
namespace nix::eval_cache {
MakeError(CachedEvalError, EvalError);
struct AttrDb;
class AttrCursor;
struct CachedEvalError : EvalError
{
const ref<AttrCursor> cursor;
const Symbol attr;
CachedEvalError(ref<AttrCursor> cursor, Symbol attr);
/**
* Evaluate this attribute, which should result in a regular
* `EvalError` exception being thrown.
*/
[[noreturn]]
void force();
};
class EvalCache : public std::enable_shared_from_this<EvalCache>
{
friend class AttrCursor;
friend class CachedEvalError;
std::shared_ptr<AttrDb> db;
EvalState & state;
@ -73,6 +87,7 @@ typedef std::variant<
class AttrCursor : public std::enable_shared_from_this<AttrCursor>
{
friend class EvalCache;
friend class CachedEvalError;
ref<EvalCache> root;
typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
@ -102,11 +117,11 @@ public:
Suggestions getSuggestionsForAttr(Symbol name);
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
ref<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
ref<AttrCursor> getAttr(Symbol name);
ref<AttrCursor> getAttr(std::string_view name);
@ -114,7 +129,7 @@ public:
* Get an attribute along a chain of attrsets. Note that this does
* not auto-call functors or functions.
*/
OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath);
std::string getString();

4
src/libexpr/eval-error.cc
View file

@ -27,8 +27,7 @@ EvalErrorBuilder<T> & EvalErrorBuilder<T>::atPos(Value & value, PosIdx fallback)
template<class T>
EvalErrorBuilder<T> & EvalErrorBuilder<T>::withTrace(PosIdx pos, const std::string_view text)
{
error.err.traces.push_front(
Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text))});
error.addTrace(error.state.positions[pos], text);
return *this;
}
@ -99,7 +98,6 @@ template class EvalErrorBuilder<TypeError>;
template class EvalErrorBuilder<UndefinedVarError>;
template class EvalErrorBuilder<MissingArgumentError>;
template class EvalErrorBuilder<InfiniteRecursionError>;
template class EvalErrorBuilder<CachedEvalError>;
template class EvalErrorBuilder<InvalidPathError>;
}

1
src/libexpr/eval-error.hh
View file

@ -43,7 +43,6 @@ MakeError(Abort, EvalError);
MakeError(TypeError, EvalError);
MakeError(UndefinedVarError, EvalError);
MakeError(MissingArgumentError, EvalError);
MakeError(CachedEvalError, EvalError);
MakeError(InfiniteRecursionError, EvalError);
struct InvalidPathError : public EvalError

20
src/libexpr/eval-settings.hh
View file

@ -15,8 +15,24 @@ struct EvalSettings : Config
static std::string resolvePseudoUrl(std::string_view url);
Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
"Whether builtin functions that allow executing native code should be enabled."};
Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", R"(
Enable built-in functions that allow executing native code.
In particular, this adds:
- `builtins.importNative` *path*
Load a dynamic shared object (DSO) at *path* which exposes a function pointer to a procedure that initialises a Nix language value, and return that value.
The procedure must have the following signature:
```cpp
extern "C" typedef void (*ValueInitialiser) (EvalState & state, Value & v);
```
The [Nix C++ API documentation](@docroot@/contributing/documentation.md#api-documentation) has more details on evaluator internals.
- `builtins.exec` *arguments*
Execute a program, where *arguments* are specified as a list of strings, and parse its output as a Nix expression.
)"};
Setting<Strings> nixPath{
this, getDefaultNixPath(), "nix-path",

2
src/libexpr/eval.cc
View file

@ -28,13 +28,13 @@
#include <algorithm>
#include <chrono>
#include <iostream>
#include <sstream>
#include <cstring>
#include <optional>
#include <unistd.h>
#include <sys/time.h>
#include <fstream>
#include <functional>
#include <iostream>
#include <nlohmann/json.hpp>
#include <boost/container/small_vector.hpp>

2
src/libexpr/flake/config.cc
View file

@ -32,7 +32,7 @@ static void writeTrustedList(const TrustedList & trustedList)
void ConfigFile::apply()
{
std::set<std::string> whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lockfile-summary"};
std::set<std::string> whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lock-file-summary", "commit-lockfile-summary"};
for (auto & [name, value] : settings) {

16
src/libexpr/get-drvs.cc
View file

@ -69,13 +69,21 @@ std::string PackageInfo::querySystem() const
std::optional<StorePath> PackageInfo::queryDrvPath() const
{
if (!drvPath && attrs) {
if (auto i = attrs->get(state->sDrvPath)) {
NixStringContext context;
if (auto i = attrs->get(state->sDrvPath))
drvPath = {state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation")};
else
auto found = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation");
try {
found.requireDerivation();
} catch (Error & e) {
e.addTrace(state->positions[i->pos], "while evaluating the 'drvPath' attribute of a derivation");
throw;
}
drvPath = {std::move(found)};
} else
drvPath = {std::nullopt};
}
return drvPath.value_or(std::nullopt);
drvPath.value_or(std::nullopt);
return *drvPath;
}

9
src/libexpr/lexer.l
View file

@ -20,8 +20,6 @@
#pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
#endif
#include <boost/lexical_cast.hpp>
#include "nixexpr.hh"
#include "parser-tab.hh"
@ -129,9 +127,10 @@ or { return OR_KW; }
{ID} { yylval->id = {yytext, (size_t) yyleng}; return ID; }
{INT} { errno = 0;
try {
yylval->n = boost::lexical_cast<int64_t>(yytext);
} catch (const boost::bad_lexical_cast &) {
std::optional<int64_t> numMay = string2Int<int64_t>(yytext);
if (numMay.has_value()) {
yylval->n = *numMay;
} else {
throw ParseError(ErrorInfo{
.msg = HintFmt("invalid integer '%1%'", yytext),
.pos = state->positions[CUR_POS],

1
src/libexpr/nixexpr.cc
View file

@ -6,6 +6,7 @@
#include "print.hh"
#include <cstdlib>
#include <sstream>
namespace nix {

20
src/libexpr/primops.cc
View file

@ -26,6 +26,7 @@
#include <algorithm>
#include <cstring>
#include <sstream>
#include <regex>
#ifndef _WIN32
@ -2261,7 +2262,7 @@ static void addPath(
std::string_view name,
SourcePath path,
Value * filterFun,
FileIngestionMethod method,
ContentAddressMethod method,
const std::optional<Hash> expectedHash,
Value & v,
const NixStringContext & context)
@ -2293,11 +2294,10 @@ static void addPath(
std::optional<StorePath> expectedStorePath;
if (expectedHash)
expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
.method = method,
.hash = *expectedHash,
.references = {},
});
expectedStorePath = state.store->makeFixedOutputPathFromCA(name, ContentAddressWithReferences::fromParts(
method,
*expectedHash,
{}));
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
auto dstPath = fetchToStore(
@ -2393,7 +2393,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
std::optional<SourcePath> path;
std::string name;
Value * filterFun = nullptr;
auto method = FileIngestionMethod::Recursive;
ContentAddressMethod method = FileIngestionMethod::Recursive;
std::optional<Hash> expectedHash;
NixStringContext context;
@ -2408,7 +2408,9 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value
else if (n == "filter")
state.forceFunction(*(filterFun = attr.value), attr.pos, "while evaluating the `filter` parameter passed to builtins.path");
else if (n == "recursive")
method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") };
method = state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path")
? FileIngestionMethod::Recursive
: FileIngestionMethod::Flat;
else if (n == "sha256")
expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256);
else
@ -4515,7 +4517,7 @@ void EvalState::createBaseEnv()
1683705525
```
The [store path](@docroot@/glossary.md#gloss-store-path) of a derivation depending on `currentTime` will differ for each evaluation, unless both evaluate `builtins.currentTime` in the same second.
The [store path](@docroot@/store/store-path.md) of a derivation depending on `currentTime` will differ for each evaluation, unless both evaluate `builtins.currentTime` in the same second.
)",
.impureOnly = true,
});

4
src/libexpr/primops/fetchTree.cc
View file

@ -200,8 +200,8 @@ static RegisterPrimOp primop_fetchTree({
.doc = R"(
Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
- the resulting fixed-output [store path](@docroot@/glossary.md#gloss-store-path)
- the corresponding [NAR](@docroot@/glossary.md#gloss-nar) hash
- the resulting fixed-output [store path](@docroot@/store/store-path.md)
- the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash
- backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
*input* must be an attribute set with the following attributes:

22
src/libexpr/print.cc
View file

@ -1,5 +1,6 @@
#include <limits>
#include <unordered_set>
#include <sstream>
#include "print.hh"
#include "ansicolor.hh"
@ -271,16 +272,27 @@ private:
void printDerivation(Value & v)
{
std::optional<StorePath> storePath;
if (auto i = v.attrs()->get(state.sDrvPath)) {
NixStringContext context;
std::string storePath;
if (auto i = v.attrs()->get(state.sDrvPath))
storePath = state.store->printStorePath(state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation"));
storePath = state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation");
}
/* This unforutately breaks printing nested values because of
how the pretty printer is used (when pretting printing and warning
to same terminal / std stream). */
#if 0
if (storePath && !storePath->isDerivation())
warn(
"drvPath attribute '%s' is not a valid store path to a derivation, this value not work properly",
state.store->printStorePath(*storePath));
#endif
if (options.ansiColors)
output << ANSI_GREEN;
output << "«derivation";
if (!storePath.empty()) {
output << " " << storePath;
if (storePath) {
output << " " << state.store->printStorePath(*storePath);
}
output << "»";
if (options.ansiColors)

2
src/libexpr/value.hh
View file

@ -449,7 +449,7 @@ public:
return std::string_view(payload.string.c_str);
}
const char * const c_str() const
const char * c_str() const
{
assert(internalType == tString);
return payload.string.c_str;

4
src/libfetchers/fetch-settings.hh
View file

@ -87,12 +87,12 @@ struct FetchSettings : public Config
{}, true, Xp::Flakes};
Setting<std::string> commitLockFileSummary{
this, "", "commit-lockfile-summary",
this, "", "commit-lock-file-summary",
R"(
The commit summary to use when committing changed flake lock files. If
empty, the summary is generated based on the action performed.
)",
{}, true, Xp::Flakes};
{"commit-lockfile-summary"}, true, Xp::Flakes};
Setting<bool> trustTarballsFromGitForges{
this, true, "trust-tarballs-from-git-forges",

6
src/libfetchers/filtering-source-accessor.cc
View file

@ -2,6 +2,12 @@
namespace nix {
std::optional<std::filesystem::path> FilteringSourceAccessor::getPhysicalPath(const CanonPath & path)
{
checkAccess(path);
return next->getPhysicalPath(prefix / path);
}
std::string FilteringSourceAccessor::readFile(const CanonPath & path)
{
checkAccess(path);

2
src/libfetchers/filtering-source-accessor.hh
View file

@ -30,6 +30,8 @@ struct FilteringSourceAccessor : SourceAccessor
displayPrefix.clear();
}
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override;
std::string readFile(const CanonPath & path) override;
bool pathExists(const CanonPath & path) override;

25
src/libfetchers/unix/git.cc → src/libfetchers/git.cc
View file

@ -19,7 +19,10 @@
#include <regex>
#include <string.h>
#include <sys/time.h>
#ifndef _WIN32
# include <sys/wait.h>
#endif
using namespace std::string_literals;
@ -40,6 +43,7 @@ bool isCacheFileWithinTtl(time_t now, const struct stat & st)
bool touchCacheFile(const Path & path, time_t touch_time)
{
#ifndef _WIN32 // TODO implement
struct timeval times[2];
times[0].tv_sec = touch_time;
times[0].tv_usec = 0;
@ -47,6 +51,9 @@ bool touchCacheFile(const Path & path, time_t touch_time)
times[1].tv_usec = 0;
return lutimes(path.c_str(), times) == 0;
#else
return false;
#endif
}
Path getCachePath(std::string_view key, bool shallow)
@ -98,7 +105,15 @@ bool storeCachedHead(const std::string & actualUrl, const std::string & headRef)
try {
runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef });
} catch (ExecError &e) {
if (!WIFEXITED(e.status)) throw;
if (
#ifndef WIN32 // TODO abstract over exit status handling on Windows
!WIFEXITED(e.status)
#else
e.status != 0
#endif
)
throw;
return false;
}
/* No need to touch refs/HEAD, because `git symbolic-ref` updates the mtime. */
@ -329,7 +344,13 @@ struct GitInputScheme : InputScheme
.program = "git",
.args = {"-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())},
});
auto exitCode = WEXITSTATUS(result.first);
auto exitCode =
#ifndef WIN32 // TODO abstract over exit status handling on Windows
WEXITSTATUS(result.first)
#else
result.first
#endif
;
if (exitCode != 0) {
// The path is not `.gitignore`d, we can add the file.

6
src/libfetchers/github.cc
View file

@ -433,9 +433,15 @@ struct GitLabInputScheme : GitArchiveInputScheme
store->toRealPath(
downloadFile(store, url, "source", headers).storePath)));
if (json.is_array() && json.size() == 1 && json[0]["id"] != nullptr) {
return RefInfo {
.rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)
};
} if (json.is_array() && json.size() == 0) {
throw Error("No commits returned by GitLab API -- does the git ref really exist?");
} else {
throw Error("Unexpected response received from GitLab: %s", json);
}
}
DownloadUrl getDownloadUrl(const Input & input) const override

6
src/libfetchers/local.mk
View file

@ -5,16 +5,10 @@ libfetchers_NAME = libnixfetchers
libfetchers_DIR := $(d)
libfetchers_SOURCES := $(wildcard $(d)/*.cc)
ifdef HOST_UNIX
libfetchers_SOURCES += $(wildcard $(d)/unix/*.cc)
endif
# Not just for this library itself, but also for downstream libraries using this library
INCLUDE_libfetchers := -I $(d)
ifdef HOST_UNIX
INCLUDE_libfetchers += -I $(d)/unix
endif
libfetchers_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers)

0
src/libfetchers/unix/mercurial.cc → src/libfetchers/mercurial.cc
View file

22
src/libfetchers/tarball.cc
View file

@ -145,9 +145,27 @@ DownloadTarballResult downloadTarball(
// TODO: fall back to cached value if download fails.
AutoDelete cleanupTemp;
/* Note: if the download is cached, `importTarball()` will receive
no data, which causes it to import an empty tarball. */
TarArchive archive { *source };
auto archive =
hasSuffix(toLower(parseURL(url).path), ".zip")
? ({
/* In streaming mode, libarchive doesn't handle
symlinks in zip files correctly (#10649). So write
the entire file to disk so libarchive can access it
in random-access mode. */
auto [fdTemp, path] = createTempFile("nix-zipfile");
cleanupTemp.reset(path);
debug("downloading '%s' into '%s'...", url, path);
{
FdSink sink(fdTemp.get());
source->drainInto(sink);
}
TarArchive{path};
})
: TarArchive{*source};
auto parseSink = getTarballCache()->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink);
@ -184,7 +202,7 @@ struct CurlInputScheme : InputScheme
{
const std::set<std::string> transportUrlSchemes = {"file", "http", "https"};
const bool hasTarballExtension(std::string_view path) const
bool hasTarballExtension(std::string_view path) const
{
return hasSuffix(path, ".zip") || hasSuffix(path, ".tar")
|| hasSuffix(path, ".tgz") || hasSuffix(path, ".tar.gz")

Some files were not shown because too many files have changed in this diff Show more