Merge remote-tracking branch 'upstream/master' into multi-output-hashDerivationModulo

This commit is contained in:
John Ericson 2020-06-17 03:51:02 +00:00
commit 02928f76fd
251 changed files with 10279 additions and 3660 deletions

View file

@ -1,27 +0,0 @@
<!--
# Filing a Nix issue
*WAIT* Are you sure you're filing your issue in the right repository?
We appreciate you taking the time to tell us about issues you encounter, but routing the issue to the right place will get you help sooner and save everyone time.
This is the Nix repository, and issues here should be about Nix the build and package management *_tool_*.
If you have a problem with a specific package on NixOS or when using Nix, you probably want to file an issue with _nixpkgs_, whose issue tracker is over at https://github.com/NixOS/nixpkgs/issues.
Examples of _Nix_ issues:
- Nix segfaults when I run `nix-build -A blahblah`
- The Nix language needs a new builtin: `builtins.foobar`
- Regression in the behavior of `nix-env` in Nix 2.0
Examples of _nixpkgs_ issues:
- glibc is b0rked on aarch64
- chromium in NixOS doesn't support U2F but google-chrome does!
- The OpenJDK package on macOS is missing a key component
Chances are if you're a newcomer to the Nix world, you'll probably want the [nixpkgs tracker](https://github.com/NixOS/nixpkgs/issues). It also gets a lot more eyeball traffic so you'll probably get a response a lot more quickly.
-->

32
.github/ISSUE_TEMPLATE/bug_report.md vendored Normal file
View file

@ -0,0 +1,32 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: bug
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
If you have a problem with a specific package or NixOS,
you probably want to file an issue at https://github.com/NixOS/nixpkgs/issues.
**Steps To Reproduce**
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Expected behavior**
A clear and concise description of what you expected to happen.
**`nix-env --version` output**
**Additional context**
Add any other context about the problem here.

View file

@ -0,0 +1,20 @@
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: improvement
assignees: ''
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.

View file

@ -6,7 +6,7 @@ jobs:
tests: tests:
strategy: strategy:
matrix: matrix:
os: [ubuntu-18.04, macos] os: [ubuntu-latest, macos-latest]
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2

7
.gitignore vendored
View file

@ -47,7 +47,10 @@ perl/Makefile.config
/src/libexpr/nix.tbl /src/libexpr/nix.tbl
# /src/libstore/ # /src/libstore/
/src/libstore/*.gen.hh *.gen.*
# /src/libutil/
/src/libutil/tests/libutil-tests
/src/nix/nix /src/nix/nix
@ -75,6 +78,8 @@ perl/Makefile.config
/src/nix-copy-closure/nix-copy-closure /src/nix-copy-closure/nix-copy-closure
/src/error-demo/error-demo
/src/build-remote/build-remote /src/build-remote/build-remote
# /tests/ # /tests/

View file

@ -1,8 +0,0 @@
matrix:
include:
- language: osx
script: ./tests/install-darwin.sh
- language: nix
script: nix-build release.nix -A build.x86_64-linux
notifications:
email: false

View file

@ -1,9 +1,10 @@
makefiles = \ makefiles = \
mk/precompiled-headers.mk \ mk/precompiled-headers.mk \
local.mk \ local.mk \
nix-rust/local.mk \
src/libutil/local.mk \ src/libutil/local.mk \
src/libutil/tests/local.mk \
src/libstore/local.mk \ src/libstore/local.mk \
src/libfetchers/local.mk \
src/libmain/local.mk \ src/libmain/local.mk \
src/libexpr/local.mk \ src/libexpr/local.mk \
src/nix/local.mk \ src/nix/local.mk \

View file

@ -1,42 +1,44 @@
AR = @AR@ AR = @AR@
BDW_GC_LIBS = @BDW_GC_LIBS@ BDW_GC_LIBS = @BDW_GC_LIBS@
BOOST_LDFLAGS = @BOOST_LDFLAGS@
BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@ BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@
CC = @CC@ CC = @CC@
CFLAGS = @CFLAGS@ CFLAGS = @CFLAGS@
CXX = @CXX@ CXX = @CXX@
CXXFLAGS = @CXXFLAGS@ CXXFLAGS = @CXXFLAGS@
LDFLAGS = @LDFLAGS@ EDITLINE_LIBS = @EDITLINE_LIBS@
ENABLE_S3 = @ENABLE_S3@ ENABLE_S3 = @ENABLE_S3@
HAVE_SODIUM = @HAVE_SODIUM@ GTEST_LIBS = @GTEST_LIBS@
HAVE_SECCOMP = @HAVE_SECCOMP@ HAVE_SECCOMP = @HAVE_SECCOMP@
BOOST_LDFLAGS = @BOOST_LDFLAGS@ HAVE_SODIUM = @HAVE_SODIUM@
LDFLAGS = @LDFLAGS@
LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
LIBCURL_LIBS = @LIBCURL_LIBS@ LIBCURL_LIBS = @LIBCURL_LIBS@
LIBLZMA_LIBS = @LIBLZMA_LIBS@
OPENSSL_LIBS = @OPENSSL_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@
PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_NAME = @PACKAGE_NAME@
PACKAGE_VERSION = @PACKAGE_VERSION@ PACKAGE_VERSION = @PACKAGE_VERSION@
SODIUM_LIBS = @SODIUM_LIBS@ SODIUM_LIBS = @SODIUM_LIBS@
LIBLZMA_LIBS = @LIBLZMA_LIBS@
SQLITE3_LIBS = @SQLITE3_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@
LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
EDITLINE_LIBS = @EDITLINE_LIBS@
bash = @bash@ bash = @bash@
bindir = @bindir@ bindir = @bindir@
lsof = @lsof@
datadir = @datadir@ datadir = @datadir@
datarootdir = @datarootdir@ datarootdir = @datarootdir@
doc_generate = @doc_generate@
docdir = @docdir@ docdir = @docdir@
exec_prefix = @exec_prefix@ exec_prefix = @exec_prefix@
includedir = @includedir@ includedir = @includedir@
libdir = @libdir@ libdir = @libdir@
libexecdir = @libexecdir@ libexecdir = @libexecdir@
localstatedir = @localstatedir@ localstatedir = @localstatedir@
lsof = @lsof@
mandir = @mandir@ mandir = @mandir@
pkglibdir = $(libdir)/$(PACKAGE_NAME) pkglibdir = $(libdir)/$(PACKAGE_NAME)
prefix = @prefix@ prefix = @prefix@
sandbox_shell = @sandbox_shell@ sandbox_shell = @sandbox_shell@
storedir = @storedir@ storedir = @storedir@
sysconfdir = @sysconfdir@ sysconfdir = @sysconfdir@
doc_generate = @doc_generate@ system = @system@
xmllint = @xmllint@ xmllint = @xmllint@
xsltproc = @xsltproc@ xsltproc = @xsltproc@

View file

@ -1,21 +1,54 @@
# Nix
[![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos) [![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos)
[![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions)
Nix, the purely functional package manager Nix is a powerful package manager for Linux and other Unix systems that makes package
------------------------------------------ management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual)
for more details.
Nix is a new take on package management that is fairly unique. Because of its ## Installation
purity aspects, a lot of issues found in traditional package managers don't
appear with Nix.
To find out more about the tool, usage and installation instructions, please On Linux and macOS the easiest way to Install Nix is to run the following shell command
read the manual, which is available on the Nix website at (as a user other than root):
<https://nixos.org/nix/manual>.
## Contributing ```
$ curl -L https://nixos.org/nix/install | sh
```
Take a look at the [Hacking Section](https://nixos.org/nix/manual/#chap-hacking) Information on additional installation methods is available on the [Nix download page](https://nixos.org/download.html).
of the manual. It helps you to get started with building Nix from source.
## Building And Developing
### Building Nix
You can build Nix using one of the targets provided by [release.nix](./release.nix):
```
$ nix-build ./release.nix -A build.aarch64-linux
$ nix-build ./release.nix -A build.x86_64-darwin
$ nix-build ./release.nix -A build.i686-linux
$ nix-build ./release.nix -A build.x86_64-linux
```
### Development Environment
You can use the provided `shell.nix` to get a working development environment:
```
$ nix-shell
$ ./bootstrap.sh
$ ./configure
$ make
```
## Additional Resources
- [Nix manual](https://nixos.org/nix/manual)
- [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix)
- [NixOS Discourse](https://discourse.nixos.org/)
- [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos)
## License ## License
Nix is released under the LGPL v2.1 Nix is released under the [LGPL v2.1](./COPYING).

View file

@ -266,6 +266,10 @@ if test "$gc" = yes; then
fi fi
# Look for gtest.
PKG_CHECK_MODULES([GTEST], [gtest_main])
# documentation generation switch # documentation generation switch
AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen], AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen],
[disable documentation generation]), [disable documentation generation]),

View file

@ -61,7 +61,7 @@ substituters = https://cache.nixos.org/ s3://example-nix-cache
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM=
</programlisting> </programlisting>
<para>we will restart the Nix daemon a later step.</para> <para>We will restart the Nix daemon in a later step.</para>
</section> </section>
<section> <section>
@ -139,7 +139,7 @@ $ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
<para>Now, copy the path back from the cache:</para> <para>Now, copy the path back from the cache:</para>
<screen> <screen>
$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example $ nix-store --realise /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example
copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'... copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'...
warning: you did not specify '--add-root'; the result might be removed by the garbage collector warning: you did not specify '--add-root'; the result might be removed by the garbage collector
/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example /nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example

View file

@ -19,26 +19,30 @@
<refsection><title>Description</title> <refsection><title>Description</title>
<para>Nix reads settings from two configuration files:</para> <para>By default Nix reads settings from the following places:</para>
<itemizedlist> <para>The system-wide configuration file
<filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
(i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or
<filename>$NIX_CONF_DIR/nix.conf</filename> if
<envar>NIX_CONF_DIR</envar> is set. Values loaded in this file are not forwarded to the Nix daemon. The
client assumes that the daemon has already loaded them.
</para>
<listitem> <para>User-specific configuration files:</para>
<para>The system-wide configuration file
<filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename>
(i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or
<filename>$NIX_CONF_DIR/nix.conf</filename> if
<envar>NIX_CONF_DIR</envar> is set.</para>
</listitem>
<listitem> <para>
<para>The user configuration file If <envar>NIX_USER_CONF_FILES</envar> is set, then each path separated by
<filename>$XDG_CONFIG_HOME/nix/nix.conf</filename>, or <literal>:</literal> will be loaded in reverse order.
<filename>~/.config/nix/nix.conf</filename> if </para>
<envar>XDG_CONFIG_HOME</envar> is not set.</para>
</listitem>
</itemizedlist> <para>
Otherwise it will look for <filename>nix/nix.conf</filename> files in
<envar>XDG_CONFIG_DIRS</envar> and <envar>XDG_CONFIG_HOME</envar>.
The default location is <filename>$HOME/.config/nix.conf</filename> if
those environment variables are unset.
</para>
<para>The configuration files consist of <para>The configuration files consist of
<literal><replaceable>name</replaceable> = <literal><replaceable>name</replaceable> =
@ -382,7 +386,7 @@ false</literal>.</para>
<programlisting> <programlisting>
builtins.fetchurl { builtins.fetchurl {
url = https://example.org/foo-1.2.3.tar.xz; url = "https://example.org/foo-1.2.3.tar.xz";
sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
} }
</programlisting> </programlisting>

View file

@ -33,7 +33,7 @@
will cause Nix to look for paths relative to will cause Nix to look for paths relative to
<filename>/home/eelco/Dev</filename> and <filename>/home/eelco/Dev</filename> and
<filename>/etc/nixos</filename>, in that order. It is also <filename>/etc/nixos</filename>, in this order. It is also
possible to match paths against a prefix. For example, the value possible to match paths against a prefix. For example, the value
<screen> <screen>
@ -53,7 +53,7 @@ nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos</screen>
<envar>NIX_PATH</envar> to <envar>NIX_PATH</envar> to
<screen> <screen>
nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz</screen> nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz</screen>
tells Nix to download the latest revision in the Nixpkgs/NixOS tells Nix to download the latest revision in the Nixpkgs/NixOS
15.09 channel.</para> 15.09 channel.</para>
@ -137,12 +137,19 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
<varlistentry><term><envar>NIX_CONF_DIR</envar></term> <varlistentry><term><envar>NIX_CONF_DIR</envar></term>
<listitem><para>Overrides the location of the Nix configuration <listitem><para>Overrides the location of the system Nix configuration
directory (default directory (default
<filename><replaceable>prefix</replaceable>/etc/nix</filename>).</para></listitem> <filename><replaceable>prefix</replaceable>/etc/nix</filename>).</para></listitem>
</varlistentry> </varlistentry>
<varlistentry><term><envar>NIX_USER_CONF_FILES</envar></term>
<listitem><para>Overrides the location of the user Nix configuration files
to load from (defaults to the XDG spec locations). The variable is treated
as a list separated by the <literal>:</literal> token.</para></listitem>
</varlistentry>
<varlistentry><term><envar>TMPDIR</envar></term> <varlistentry><term><envar>TMPDIR</envar></term>

View file

@ -526,13 +526,10 @@ these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked):
14.12 channel: 14.12 channel:
<screen> <screen>
$ nix-env -f https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz -iA firefox $ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox
</screen> </screen>
(The GitHub repository <literal>nixpkgs-channels</literal> is updated </para>
automatically from the main <literal>nixpkgs</literal> repository
after certain tests have succeeded and binaries have been built and
uploaded to the binary cache at <uri>cache.nixos.org</uri>.)</para>
</refsection> </refsection>

View file

@ -258,7 +258,7 @@ path. You can override it by passing <option>-I</option> or setting
containing the Pan package from a specific revision of Nixpkgs: containing the Pan package from a specific revision of Nixpkgs:
<screen> <screen>
$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz $ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz
[nix-shell:~]$ pan --version [nix-shell:~]$ pan --version
Pan 0.139 Pan 0.139
@ -352,7 +352,7 @@ following Haskell script uses a specific branch of Nixpkgs/NixOS (the
<programlisting><![CDATA[ <programlisting><![CDATA[
#! /usr/bin/env nix-shell #! /usr/bin/env nix-shell
#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])" #! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])"
#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-18.03.tar.gz #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz
import Network.HTTP import Network.HTTP
import Text.HTML.TagSoup import Text.HTML.TagSoup
@ -370,7 +370,7 @@ If you want to be even more precise, you can specify a specific
revision of Nixpkgs: revision of Nixpkgs:
<programlisting> <programlisting>
#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz
</programlisting> </programlisting>
</para> </para>

View file

@ -360,7 +360,6 @@ EOF
<arg choice='plain'><option>--print-roots</option></arg> <arg choice='plain'><option>--print-roots</option></arg>
<arg choice='plain'><option>--print-live</option></arg> <arg choice='plain'><option>--print-live</option></arg>
<arg choice='plain'><option>--print-dead</option></arg> <arg choice='plain'><option>--print-dead</option></arg>
<arg choice='plain'><option>--delete</option></arg>
</group> </group>
<arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg> <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
</cmdsynopsis> </cmdsynopsis>
@ -407,14 +406,6 @@ the Nix store not reachable via file system references from a set of
</varlistentry> </varlistentry>
<varlistentry><term><option>--delete</option></term>
<listitem><para>This operation performs an actual garbage
collection. All dead paths are removed from the
store. This is the default.</para></listitem>
</varlistentry>
</variablelist> </variablelist>
<para>By default, all unreachable paths are deleted. The following <para>By default, all unreachable paths are deleted. The following
@ -444,10 +435,10 @@ and <link
linkend="conf-keep-derivations"><literal>keep-derivations</literal></link> linkend="conf-keep-derivations"><literal>keep-derivations</literal></link>
variables in the Nix configuration file.</para> variables in the Nix configuration file.</para>
<para>With <option>--delete</option>, the collector prints the total <para>By default, the collector prints the total number of freed bytes
number of freed bytes when it finishes (or when it is interrupted). when it finishes (or when it is interrupted). With
With <option>--print-dead</option>, it prints the number of bytes that <option>--print-dead</option>, it prints the number of bytes that would
would be freed.</para> be freed.</para>
</refsection> </refsection>
@ -945,7 +936,7 @@ $ nix-store --add ./foo.c
<para>The operation <option>--add-fixed</option> adds the specified paths to <para>The operation <option>--add-fixed</option> adds the specified paths to
the Nix store. Unlike <option>--add</option> paths are registered using the the Nix store. Unlike <option>--add</option> paths are registered using the
specified hashing algorithm, resulting in the same output path as a fixed output specified hashing algorithm, resulting in the same output path as a fixed-output
derivation. This can be used for sources that are not available from a public derivation. This can be used for sources that are not available from a public
url or broke since the download expression was written. url or broke since the download expression was written.
</para> </para>
@ -1148,7 +1139,7 @@ the information that Nix considers important. For instance,
timestamps are elided because all files in the Nix store have their timestamps are elided because all files in the Nix store have their
timestamp set to 0 anyway. Likewise, all permissions are left out timestamp set to 0 anyway. Likewise, all permissions are left out
except for the execute bit, because all files in the Nix store have except for the execute bit, because all files in the Nix store have
644 or 755 permission.</para> 444 or 555 permission.</para>
<para>Also, a NAR archive is <emphasis>canonical</emphasis>, meaning <para>Also, a NAR archive is <emphasis>canonical</emphasis>, meaning
that “equal” paths always produce the same NAR archive. For instance, that “equal” paths always produce the same NAR archive. For instance,

View file

@ -11,6 +11,10 @@
<arg> <arg>
<arg choice='plain'><option>--quiet</option></arg> <arg choice='plain'><option>--quiet</option></arg>
</arg> </arg>
<arg>
<option>--log-format</option>
<replaceable>format</replaceable>
</arg>
<arg> <arg>
<group choice='plain'> <group choice='plain'>
<arg choice='plain'><option>--no-build-output</option></arg> <arg choice='plain'><option>--no-build-output</option></arg>

View file

@ -92,6 +92,37 @@
</varlistentry> </varlistentry>
<varlistentry xml:id="opt-log-format"><term><option>--log-format</option> <replaceable>format</replaceable></term>
<listitem>
<para>This option can be used to change the output of the log format, with
<replaceable>format</replaceable> being one of:</para>
<variablelist>
<varlistentry><term>raw</term>
<listitem><para>This is the raw format, as outputted by nix-build.</para></listitem>
</varlistentry>
<varlistentry><term>internal-json</term>
<listitem><para>Outputs the logs in a structured manner. NOTE: the json schema is not guarantees to be stable between releases.</para></listitem>
</varlistentry>
<varlistentry><term>bar</term>
<listitem><para>Only display a progress bar during the builds.</para></listitem>
</varlistentry>
<varlistentry><term>bar-with-logs</term>
<listitem><para>Display the raw logs, with the progress bar at the bottom.</para></listitem>
</varlistentry>
</variablelist>
</listitem>
</varlistentry>
<varlistentry><term><option>--no-build-output</option> / <option>-Q</option></term> <varlistentry><term><option>--no-build-output</option> / <option>-Q</option></term>
<listitem><para>By default, output written by builders to standard <listitem><para>By default, output written by builders to standard

View file

@ -178,7 +178,7 @@ impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ];
<programlisting> <programlisting>
fetchurl { fetchurl {
url = http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz; url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz";
sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
} }
</programlisting> </programlisting>
@ -189,7 +189,7 @@ fetchurl {
<programlisting> <programlisting>
fetchurl { fetchurl {
url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz; url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
} }
</programlisting> </programlisting>

View file

@ -324,7 +324,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
particular version of Nixpkgs, e.g. particular version of Nixpkgs, e.g.
<programlisting> <programlisting>
with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz) {}; with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {};
stdenv.mkDerivation { … } stdenv.mkDerivation { … }
</programlisting> </programlisting>
@ -349,7 +349,7 @@ stdenv.mkDerivation { … }
<programlisting> <programlisting>
with import (fetchTarball { with import (fetchTarball {
url = https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz; url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz";
sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2"; sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2";
}) {}; }) {};
@ -422,6 +422,16 @@ stdenv.mkDerivation { … }
</para> </para>
</listitem> </listitem>
</varlistentry> </varlistentry>
<varlistentry>
<term>submodules</term>
<listitem>
<para>
A Boolean parameter that specifies whether submodules
should be checked out. Defaults to
<literal>false</literal>.
</para>
</listitem>
</varlistentry>
</variablelist> </variablelist>
<example> <example>
@ -1396,7 +1406,7 @@ stdenv.mkDerivation {
"; ";
src = fetchurl { src = fetchurl {
url = http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz; url = "http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
}; };
inherit perl; inherit perl;

View file

@ -15,7 +15,7 @@ stdenv.mkDerivation { <co xml:id='ex-hello-nix-co-2' />
name = "hello-2.1.1"; <co xml:id='ex-hello-nix-co-3' /> name = "hello-2.1.1"; <co xml:id='ex-hello-nix-co-3' />
builder = ./builder.sh; <co xml:id='ex-hello-nix-co-4' /> builder = ./builder.sh; <co xml:id='ex-hello-nix-co-4' />
src = fetchurl { <co xml:id='ex-hello-nix-co-5' /> src = fetchurl { <co xml:id='ex-hello-nix-co-5' />
url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz; url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz";
sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465";
}; };
inherit perl; <co xml:id='ex-hello-nix-co-6' /> inherit perl; <co xml:id='ex-hello-nix-co-6' />

View file

@ -73,12 +73,4 @@ waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x'</
So it is always safe to run multiple instances of Nix in parallel So it is always safe to run multiple instances of Nix in parallel
(which isnt the case with, say, <command>make</command>).</para> (which isnt the case with, say, <command>make</command>).</para>
<para>If you have a system with multiple CPUs, you may want to have
Nix build different derivations in parallel (insofar as possible).
Just pass the option <link linkend='opt-max-jobs'><option>-j
<replaceable>N</replaceable></option></link>, where
<replaceable>N</replaceable> is the maximum number of jobs to be run
in parallel, or set. Typically this should be the number of
CPUs.</para>
</section> </section>

View file

@ -6,16 +6,30 @@
<title>Installing a Binary Distribution</title> <title>Installing a Binary Distribution</title>
<para>If you are using Linux or macOS, the easiest way to install Nix <para>
is to run the following command: If you are using Linux or macOS versions up to 10.14 (Mojave), the
easiest way to install Nix is to run the following command:
</para>
<screen> <screen>
$ sh &lt;(curl https://nixos.org/nix/install) $ sh &lt;(curl https://nixos.org/nix/install)
</screen> </screen>
As of Nix 2.1.0, the Nix installer will always default to creating a <para>
single-user installation, however opting in to the multi-user If you're using macOS 10.15 (Catalina) or newer, consult
installation is highly recommended. <link linkend="sect-macos-installation">the macOS installation instructions</link>
before installing.
</para>
<para>
As of Nix 2.1.0, the Nix installer will always default to creating a
single-user installation, however opting in to the multi-user
installation is highly recommended.
<!-- TODO: this explains *neither* why the default version is
single-user, nor why we'd recommend multi-user over the default.
True prospective users don't have much basis for evaluating this.
What's it to me? Who should pick which? Why? What if I pick wrong?
-->
</para> </para>
<section xml:id="sect-single-user-installation"> <section xml:id="sect-single-user-installation">
@ -36,7 +50,7 @@ run this under your usual user account, <emphasis>not</emphasis> as
root. The script will invoke <command>sudo</command> to create root. The script will invoke <command>sudo</command> to create
<filename>/nix</filename> if it doesnt already exist. If you dont <filename>/nix</filename> if it doesnt already exist. If you dont
have <command>sudo</command>, you should manually create have <command>sudo</command>, you should manually create
<command>/nix</command> first as root, e.g.: <filename>/nix</filename> first as root, e.g.:
<screen> <screen>
$ mkdir /nix $ mkdir /nix
@ -47,7 +61,7 @@ The install script will modify the first writable file from amongst
<filename>.bash_profile</filename>, <filename>.bash_login</filename> <filename>.bash_profile</filename>, <filename>.bash_login</filename>
and <filename>.profile</filename> to source and <filename>.profile</filename> to source
<filename>~/.nix-profile/etc/profile.d/nix.sh</filename>. You can set <filename>~/.nix-profile/etc/profile.d/nix.sh</filename>. You can set
the <command>NIX_INSTALLER_NO_MODIFY_PROFILE</command> environment the <envar>NIX_INSTALLER_NO_MODIFY_PROFILE</envar> environment
variable before executing the install script to disable this variable before executing the install script to disable this
behaviour. behaviour.
</para> </para>
@ -81,12 +95,10 @@ $ rm -rf /nix
<para> <para>
You can instruct the installer to perform a multi-user You can instruct the installer to perform a multi-user
installation on your system: installation on your system:
<screen>
sh &lt;(curl https://nixos.org/nix/install) --daemon
</screen>
</para> </para>
<screen>sh &lt;(curl https://nixos.org/nix/install) --daemon</screen>
<para> <para>
The multi-user installation of Nix will create build users between The multi-user installation of Nix will create build users between
the user IDs 30001 and 30032, and a group with the group ID 30000. the user IDs 30001 and 30032, and a group with the group ID 30000.
@ -136,6 +148,273 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
</section> </section>
<section xml:id="sect-macos-installation">
<title>macOS Installation</title>
<para>
Starting with macOS 10.15 (Catalina), the root filesystem is read-only.
This means <filename>/nix</filename> can no longer live on your system
volume, and that you'll need a workaround to install Nix.
</para>
<para>
The recommended approach, which creates an unencrypted APFS volume
for your Nix store and a "synthetic" empty directory to mount it
over at <filename>/nix</filename>, is least likely to impair Nix
or your system.
</para>
<note><para>
With all separate-volume approaches, it's possible something on
your system (particularly daemons/services and restored apps) may
need access to your Nix store before the volume is mounted. Adding
additional encryption makes this more likely.
</para></note>
<para>
If you're using a recent Mac with a
<link xlink:href="https://www.apple.com/euro/mac/shared/docs/Apple_T2_Security_Chip_Overview.pdf">T2 chip</link>,
your drive will still be encrypted at rest (in which case "unencrypted"
is a bit of a misnomer). To use this approach, just install Nix with:
</para>
<screen>$ sh &lt;(curl https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume</screen>
<para>
If you don't like the sound of this, you'll want to weigh the
other approaches and tradeoffs detailed in this section.
</para>
<note>
<title>Eventual solutions?</title>
<para>
All of the known workarounds have drawbacks, but we hope
better solutions will be available in the future. Some that
we have our eye on are:
</para>
<orderedlist>
<listitem>
<para>
A true firmlink would enable the Nix store to live on the
primary data volume without the build problems caused by
the symlink approach. End users cannot currently
create true firmlinks.
</para>
</listitem>
<listitem>
<para>
If the Nix store volume shared FileVault encryption
with the primary data volume (probably by using the same
volume group and role), FileVault encryption could be
easily supported by the installer without requiring
manual setup by each user.
</para>
</listitem>
</orderedlist>
</note>
<section xml:id="sect-macos-installation-change-store-prefix">
<title>Change the Nix store path prefix</title>
<para>
Changing the default prefix for the Nix store is a simple
approach which enables you to leave it on your root volume,
where it can take full advantage of FileVault encryption if
enabled. Unfortunately, this approach also opts your device out
of some benefits that are enabled by using the same prefix
across systems:
<itemizedlist>
<listitem>
<para>
Your system won't be able to take advantage of the binary
cache (unless someone is able to stand up and support
duplicate caching infrastructure), which means you'll
spend more time waiting for builds.
</para>
</listitem>
<listitem>
<para>
It's harder to build and deploy packages to Linux systems.
</para>
</listitem>
<!-- TODO: may be more here -->
</itemizedlist>
<!-- TODO: Yes, but how?! -->
It would also possible (and often requested) to just apply this
change ecosystem-wide, but it's an intrusive process that has
side effects we want to avoid for now.
<!-- magnificent hand-wavy gesture -->
</para>
<para>
</para>
</section>
<section xml:id="sect-macos-installation-encrypted-volume">
<title>Use a separate encrypted volume</title>
<para>
If you like, you can also add encryption to the recommended
approach taken by the installer. You can do this by pre-creating
an encrypted volume before you run the installer--or you can
run the installer and encrypt the volume it creates later.
<!-- TODO: see later note about whether this needs both add-encryption and from-scratch directions -->
</para>
<para>
In either case, adding encryption to a second volume isn't quite
as simple as enabling FileVault for your boot volume. Before you
dive in, there are a few things to weigh:
</para>
<orderedlist>
<listitem>
<para>
The additional volume won't be encrypted with your existing
FileVault key, so you'll need another mechanism to decrypt
the volume.
</para>
</listitem>
<listitem>
<para>
You can store the password in Keychain to automatically
decrypt the volume on boot--but it'll have to wait on Keychain
and may not mount before your GUI apps restore. If any of
your launchd agents or apps depend on Nix-installed software
(for example, if you use a Nix-installed login shell), the
restore may fail or break.
</para>
<para>
On a case-by-case basis, you may be able to work around this
problem by using <command>wait4path</command> to block
execution until your executable is available.
</para>
<para>
It's also possible to decrypt and mount the volume earlier
with a login hook--but this mechanism appears to be
deprecated and its future is unclear.
</para>
</listitem>
<listitem>
<para>
You can hard-code the password in the clear, so that your
store volume can be decrypted before Keychain is available.
</para>
</listitem>
</orderedlist>
<para>
If you are comfortable navigating these tradeoffs, you can encrypt the volume with
something along the lines of:
<!-- TODO:
I don't know if this also needs from-scratch instructions?
can we just recommend use-the-installer-and-then-encrypt?
-->
</para>
<!--
TODO: it looks like this option can be encryptVolume|encrypt|enableFileVault
It may be more clear to use encryptVolume, here? FileVault seems
heavily associated with the boot-volume behavior; I worry
a little that it can mislead here, especially as it gets
copied around minus doc context...?
-->
<screen>alice$ diskutil apfs enableFileVault /nix -user disk</screen>
<!-- TODO: and then go into detail on the mount/decrypt approaches? -->
</section>
<section xml:id="sect-macos-installation-symlink">
<!--
Maybe a good razor is: if we'd hate having to support someone who
installed Nix this way, it shouldn't even be detailed?
-->
<title>Symlink the Nix store to a custom location</title>
<para>
Another simple approach is using <filename>/etc/synthetic.conf</filename>
to symlink the Nix store to the data volume. This option also
enables your store to share any configured FileVault encryption.
Unfortunately, builds that resolve the symlink may leak the
canonical path or even fail.
</para>
<para>
Because of these downsides, we can't recommend this approach.
</para>
<!-- Leaving out instructions for this one. -->
</section>
<section xml:id="sect-macos-installation-recommended-notes">
<title>Notes on the recommended approach</title>
<para>
This section goes into a little more detail on the recommended
approach. You don't need to understand it to run the installer,
but it can serve as a helpful reference if you run into trouble.
</para>
<orderedlist>
<listitem>
<para>
In order to compose user-writable locations into the new
read-only system root, Apple introduced a new concept called
<literal>firmlinks</literal>, which it describes as a
"bi-directional wormhole" between two filesystems. You can
see the current firmlinks in <filename>/usr/share/firmlinks</filename>.
Unfortunately, firmlinks aren't (currently?) user-configurable.
</para>
<para>
For special cases like NFS mount points or package manager roots,
<link xlink:href="https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man5/synthetic.conf.5.html">synthetic.conf(5)</link>
supports limited user-controlled file-creation (of symlinks,
and synthetic empty directories) at <filename>/</filename>.
To create a synthetic empty directory for mounting at <filename>/nix</filename>,
add the following line to <filename>/etc/synthetic.conf</filename>
(create it if necessary):
</para>
<screen>nix</screen>
</listitem>
<listitem>
<para>
This configuration is applied at boot time, but you can use
<command>apfs.util</command> to trigger creation (not deletion)
of new entries without a reboot:
</para>
<screen>alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B</screen>
</listitem>
<listitem>
<para>
Create the new APFS volume with diskutil:
</para>
<screen>alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix</screen>
</listitem>
<listitem>
<para>
Using <command>vifs</command>, add the new mount to
<filename>/etc/fstab</filename>. If it doesn't already have
other entries, it should look something like:
</para>
<screen>
#
# Warning - this file should only be modified with vifs(8)
#
# Failure to do so is unsupported and may be destructive.
#
LABEL=Nix\040Store /nix apfs rw,nobrowse
</screen>
<para>
The nobrowse setting will keep Spotlight from indexing this
volume, and keep it from showing up on your desktop.
</para>
</listitem>
</orderedlist>
</section>
</section>
<section xml:id="sect-nix-install-pinned-version-url"> <section xml:id="sect-nix-install-pinned-version-url">
<title>Installing a pinned Nix version from a URL</title> <title>Installing a pinned Nix version from a URL</title>

View file

@ -17,6 +17,11 @@
<para> <para>
Single-user installations of Nix should run this: Single-user installations of Nix should run this:
<command>nix-channel --update; nix-env -iA nixpkgs.nix</command> <command>nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert</command>
</para>
<para>
Multi-user Nix users on Linux should run this with sudo:
<command>nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert; systemctl daemon-reload; systemctl restart nix-daemon</command>
</para> </para>
</chapter> </chapter>

View file

@ -8,7 +8,7 @@
<para>NOTE: the hashing scheme in Nix 0.8 changed (as detailed below). <para>NOTE: the hashing scheme in Nix 0.8 changed (as detailed below).
As a result, <command>nix-pull</command> manifests and channels built As a result, <command>nix-pull</command> manifests and channels built
for Nix 0.7 and below will now work anymore. However, the Nix for Nix 0.7 and below will not work anymore. However, the Nix
expression language has not changed, so you can still build from expression language has not changed, so you can still build from
source. Also, existing user environments continue to work. Nix 0.8 source. Also, existing user environments continue to work. Nix 0.8
will automatically upgrade the database schema of previous will automatically upgrade the database schema of previous

View file

@ -6,9 +6,11 @@ dist-files += configure config.h.in perl/configure
clean-files += Makefile.config clean-files += Makefile.config
GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix -Wno-deprecated-declarations GLOBAL_CXXFLAGS += -Wno-deprecated-declarations
$(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \ $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
$(GCH) $(PCH): src/libutil/util.hh config.h $(GCH) $(PCH): src/libutil/util.hh config.h
GCH_CXXFLAGS = -I src/libutil

View file

@ -1,5 +1,5 @@
#! /usr/bin/env nix-shell #! /usr/bin/env nix-shell
#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp gnupg1 #! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 gnupg1
use strict; use strict;
use Data::Dumper; use Data::Dumper;
@ -9,12 +9,16 @@ use File::Slurp;
use File::Copy; use File::Copy;
use JSON::PP; use JSON::PP;
use LWP::UserAgent; use LWP::UserAgent;
use Net::Amazon::S3;
my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
my $releasesDir = "/home/eelco/mnt/releases"; my $releasesBucketName = "nix-releases";
my $channelsBucketName = "nix-channels";
my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine"; my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine";
my $TMPDIR = $ENV{'TMPDIR'} // "/tmp";
# FIXME: cut&paste from nixos-channel-scripts. # FIXME: cut&paste from nixos-channel-scripts.
sub fetch { sub fetch {
my ($url, $type) = @_; my ($url, $type) = @_;
@ -42,13 +46,31 @@ my $version = $1;
print STDERR "Nix revision is $nixRev, version is $version\n"; print STDERR "Nix revision is $nixRev, version is $version\n";
File::Path::make_path($releasesDir); my $releaseDir = "nix/$releaseName";
if (system("mountpoint -q $releasesDir") != 0) {
system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die;
}
my $releaseDir = "$releasesDir/nix/$releaseName"; my $tmpDir = "$TMPDIR/nix-release/$releaseName";
File::Path::make_path($releaseDir); File::Path::make_path($tmpDir);
# S3 setup.
my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given.";
my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given.";
my $s3 = Net::Amazon::S3->new(
{ aws_access_key_id => $aws_access_key_id,
aws_secret_access_key => $aws_secret_access_key,
retry => 1,
host => "s3-eu-west-1.amazonaws.com",
});
my $releasesBucket = $s3->bucket($releasesBucketName) or die;
my $s3_us = Net::Amazon::S3->new(
{ aws_access_key_id => $aws_access_key_id,
aws_secret_access_key => $aws_secret_access_key,
retry => 1,
});
my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
sub downloadFile { sub downloadFile {
my ($jobName, $productNr, $dstName) = @_; my ($jobName, $productNr, $dstName) = @_;
@ -57,40 +79,49 @@ sub downloadFile {
my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n"; my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n";
$dstName //= basename($srcFile); $dstName //= basename($srcFile);
my $dstFile = "$releaseDir/" . $dstName; my $tmpFile = "$tmpDir/$dstName";
if (! -e $dstFile) { if (!-e $tmpFile) {
print STDERR "downloading $srcFile to $dstFile...\n"; print STDERR "downloading $srcFile to $tmpFile...\n";
system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0 system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$tmpFile'") == 0
or die "unable to fetch $srcFile\n"; or die "unable to fetch $srcFile\n";
rename("$dstFile.tmp", $dstFile) or die;
} }
my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die; my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die;
my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`; my $sha256_actual = `nix hash-file --base16 --type sha256 '$tmpFile'`;
chomp $sha256_actual; chomp $sha256_actual;
if ($sha256_expected ne $sha256_actual) { if ($sha256_expected ne $sha256_actual) {
print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n"; print STDERR "file $tmpFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
exit 1; exit 1;
} }
write_file("$dstFile.sha256", $sha256_expected); write_file("$tmpFile.sha256", $sha256_expected);
if (! -e "$dstFile.asc") { if (! -e "$tmpFile.asc") {
system("gpg2 --detach-sign --armor $dstFile") == 0 or die "unable to sign $dstFile\n"; system("gpg2 --detach-sign --armor $tmpFile") == 0 or die "unable to sign $tmpFile\n";
} }
return ($dstFile, $sha256_expected); return $sha256_expected;
} }
downloadFile("tarball", "2"); # .tar.bz2 downloadFile("tarball", "2"); # .tar.bz2
my ($tarball, $tarballHash) = downloadFile("tarball", "3"); # .tar.xz my $tarballHash = downloadFile("tarball", "3"); # .tar.xz
downloadFile("binaryTarball.i686-linux", "1"); downloadFile("binaryTarball.i686-linux", "1");
downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.x86_64-linux", "1");
downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1");
downloadFile("binaryTarball.x86_64-darwin", "1"); downloadFile("binaryTarball.x86_64-darwin", "1");
downloadFile("installerScript", "1"); downloadFile("installerScript", "1");
for my $fn (glob "$tmpDir/*") {
my $name = basename($fn);
my $dstKey = "$releaseDir/" . $name;
unless (defined $releasesBucket->head_key($dstKey)) {
print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
$releasesBucket->add_key_filename($dstKey, $fn)
or die $releasesBucket->err . ": " . $releasesBucket->errstr;
}
}
exit if $version =~ /pre/; exit if $version =~ /pre/;
# Update Nixpkgs in a very hacky way. # Update Nixpkgs in a very hacky way.
@ -111,8 +142,12 @@ $oldName =~ s/"//g;
sub getStorePath { sub getStorePath {
my ($jobName) = @_; my ($jobName) = @_;
my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
die unless $buildInfo->{buildproducts}->{1}->{type} eq "nix-build"; for my $product (values %{$buildInfo->{buildproducts}}) {
return $buildInfo->{buildproducts}->{1}->{path}; next unless $product->{type} eq "nix-build";
next if $product->{path} =~ /[a-z]+$/;
return $product->{path};
}
die;
} }
write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
@ -125,18 +160,11 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die; system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die;
# Extract the HTML manual.
File::Path::make_path("$releaseDir/manual");
system("tar xvf $tarball --strip-components=3 -C $releaseDir/manual --wildcards '*/doc/manual/*.html' '*/doc/manual/*.css' '*/doc/manual/*.gif' '*/doc/manual/*.png'") == 0 or die;
if (! -e "$releaseDir/manual/index.html") {
symlink("manual.html", "$releaseDir/manual/index.html") or die;
}
# Update the "latest" symlink. # Update the "latest" symlink.
symlink("$releaseName", "$releasesDir/nix/latest-tmp") or die; $channelsBucket->add_key(
rename("$releasesDir/nix/latest-tmp", "$releasesDir/nix/latest") or die; "nix-latest/install", "",
{ "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
or die $channelsBucket->err . ": " . $channelsBucket->errstr;
# Tag the release in Git. # Tag the release in Git.
chdir("/home/eelco/Dev/nix-pristine") or die; chdir("/home/eelco/Dev/nix-pristine") or die;

View file

@ -125,7 +125,8 @@ define build-library
$(1)_PATH := $$(_d)/$$($(1)_NAME).a $(1)_PATH := $$(_d)/$$($(1)_NAME).a
$$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/ $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/
$(trace-ar) $(AR) crs $$@ $$? $(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$?
$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o
$(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS) $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS)

View file

@ -8,14 +8,14 @@ GCH = $(buildprefix)precompiled-headers.h.gch
$(GCH): precompiled-headers.h $(GCH): precompiled-headers.h
@rm -f $@ @rm -f $@
@mkdir -p "$(dir $@)" @mkdir -p "$(dir $@)"
$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
PCH = $(buildprefix)precompiled-headers.h.pch PCH = $(buildprefix)precompiled-headers.h.pch
$(PCH): precompiled-headers.h $(PCH): precompiled-headers.h
@rm -f $@ @rm -f $@
@mkdir -p "$(dir $@)" @mkdir -p "$(dir $@)"
$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
clean-files += $(GCH) $(PCH) clean-files += $(GCH) $(PCH)

View file

@ -35,24 +35,28 @@ define build-program
$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE))
$(1)_INSTALL_DIR ?= $$(bindir) $(1)_INSTALL_DIR ?= $$(bindir)
$(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1)
$$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) ifdef $(1)_INSTALL_DIR
install: $(DESTDIR)$$($(1)_INSTALL_PATH) $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1)
ifeq ($(BUILD_SHARED_LIBS), 1) $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR)))
_libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH)) install: $(DESTDIR)$$($(1)_INSTALL_PATH)
$(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ ifeq ($(BUILD_SHARED_LIBS), 1)
_libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH))
$(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
else else
$(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/
install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$< install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$<
endif
endif endif
# Propagate CFLAGS and CXXFLAGS to the individual object files. # Propagate CFLAGS and CXXFLAGS to the individual object files.
@ -76,4 +80,10 @@ define build-program
programs-list += $$($(1)_PATH) programs-list += $$($(1)_PATH)
clean-files += $$($(1)_PATH) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS) clean-files += $$($(1)_PATH) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS)
dist-files += $$(_srcs) dist-files += $$(_srcs)
# Phony target to run this program (typically as a dependency of 'check').
.PHONY: $(1)_RUN
$(1)_RUN: $$($(1)_PATH)
$(trace-test) $$($(1)_PATH)
endef endef

View file

@ -11,6 +11,7 @@ ifeq ($(V), 0)
trace-javac = @echo " JAVAC " $@; trace-javac = @echo " JAVAC " $@;
trace-jar = @echo " JAR " $@; trace-jar = @echo " JAR " $@;
trace-mkdir = @echo " MKDIR " $@; trace-mkdir = @echo " MKDIR " $@;
trace-test = @echo " TEST " $@;
suppress = @ suppress = @

View file

@ -41,5 +41,5 @@ ifneq ($(OS), Darwin)
check: rust-tests check: rust-tests
rust-tests: rust-tests:
cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi) $(trace-test) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi)
endif endif

View file

@ -80,7 +80,7 @@ SV * queryReferences(char * path)
SV * queryPathHash(char * path) SV * queryPathHash(char * path)
PPCODE: PPCODE:
try { try {
auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(); auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());
@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32)
XPUSHs(&PL_sv_undef); XPUSHs(&PL_sv_undef);
else else
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
auto s = info->narHash.to_string(base32 ? Base32 : Base16); auto s = info->narHash.to_string(base32 ? Base32 : Base16, true);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
mXPUSHi(info->registrationTime); mXPUSHi(info->registrationTime);
mXPUSHi(info->narSize); mXPUSHi(info->narSize);
@ -274,7 +274,8 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg)
SV * addToStore(char * srcPath, int recursive, char * algo) SV * addToStore(char * srcPath, int recursive, char * algo)
PPCODE: PPCODE:
try { try {
auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, recursive, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo));
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());
@ -285,7 +286,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
PPCODE: PPCODE:
try { try {
Hash h(hash, parseHashType(algo)); Hash h(hash, parseHashType(algo));
auto path = store()->makeFixedOutputPath(recursive, h, name); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->makeFixedOutputPath(method, h, name);
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());

View file

@ -56,6 +56,3 @@
#include <sys/wait.h> #include <sys/wait.h>
#include <termios.h> #include <termios.h>
#include <unistd.h> #include <unistd.h>
#include "util.hh"
#include "args.hh"

View file

@ -50,11 +50,11 @@ rec {
libarchive libarchive
boost boost
nlohmann_json nlohmann_json
rustc cargo
# Tests # Tests
git git
mercurial mercurial
gmock
] ]
++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium

View file

@ -14,50 +14,6 @@ let
jobs = rec { jobs = rec {
# Create a "vendor" directory that contains the crates listed in
# Cargo.lock. This allows Nix to be built without network access.
vendoredCrates =
let
lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
files = map (pkg: import <nix/fetchurl.nix> {
url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download";
sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)";
}) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package);
in pkgs.runCommand "cargo-vendor-dir" {}
''
mkdir -p $out/vendor
cat > $out/vendor/config <<EOF
[source.crates-io]
replace-with = "vendored-sources"
[source.vendored-sources]
directory = "vendor"
EOF
${toString (builtins.map (file: ''
mkdir $out/vendor/tmp
tar xvf ${file} -C $out/vendor/tmp
dir=$(echo $out/vendor/tmp/*)
# Add just enough metadata to keep Cargo happy.
printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json"
# Clean up some cruft from the winapi crates. FIXME: find
# a way to remove winapi* from our dependencies.
if [[ $dir =~ /winapi ]]; then
find $dir -name "*.a" -print0 | xargs -0 rm -f --
fi
mv "$dir" $out/vendor/
rm -rf $out/vendor/tmp
'') files)}
'';
build = pkgs.lib.genAttrs systems (system: build = pkgs.lib.genAttrs systems (system:
let pkgs = import nixpkgs { inherit system; }; in let pkgs = import nixpkgs { inherit system; }; in
@ -89,8 +45,6 @@ let
patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
''} ''}
ln -sfn ${vendoredCrates}/vendor/ nix-rust/vendor
(cd perl; autoreconf --install --force --verbose) (cd perl; autoreconf --install --force --verbose)
''; '';
@ -103,17 +57,17 @@ let
installFlags = "sysconfdir=$(out)/etc"; installFlags = "sysconfdir=$(out)/etc";
postInstall = ''
mkdir -p $doc/nix-support
echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
'';
doCheck = true; doCheck = true;
doInstallCheck = true; doInstallCheck = true;
installCheckFlags = "sysconfdir=$(out)/etc"; installCheckFlags = "sysconfdir=$(out)/etc";
separateDebugInfo = true; separateDebugInfo = true;
preDist = ''
mkdir -p $doc/nix-support
echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
'';
}); });
@ -165,10 +119,10 @@ let
} }
'' ''
cp ${installerClosureInfo}/registration $TMPDIR/reginfo cp ${installerClosureInfo}/registration $TMPDIR/reginfo
cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
--subst-var-by nix ${toplevel} \ --subst-var-by nix ${toplevel} \
--subst-var-by cacert ${cacert} --subst-var-by cacert ${cacert}
substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
--subst-var-by nix ${toplevel} \ --subst-var-by nix ${toplevel} \
--subst-var-by cacert ${cacert} --subst-var-by cacert ${cacert}
@ -183,6 +137,7 @@ let
# SC1090: Don't worry about not being able to find # SC1090: Don't worry about not being able to find
# $nix/etc/profile.d/nix.sh # $nix/etc/profile.d/nix.sh
shellcheck --exclude SC1090 $TMPDIR/install shellcheck --exclude SC1090 $TMPDIR/install
shellcheck $TMPDIR/create-darwin-volume.sh
shellcheck $TMPDIR/install-darwin-multi-user.sh shellcheck $TMPDIR/install-darwin-multi-user.sh
shellcheck $TMPDIR/install-systemd-multi-user.sh shellcheck $TMPDIR/install-systemd-multi-user.sh
@ -198,6 +153,7 @@ let
fi fi
chmod +x $TMPDIR/install chmod +x $TMPDIR/install
chmod +x $TMPDIR/create-darwin-volume.sh
chmod +x $TMPDIR/install-darwin-multi-user.sh chmod +x $TMPDIR/install-darwin-multi-user.sh
chmod +x $TMPDIR/install-systemd-multi-user.sh chmod +x $TMPDIR/install-systemd-multi-user.sh
chmod +x $TMPDIR/install-multi-user chmod +x $TMPDIR/install-multi-user
@ -210,11 +166,15 @@ let
--absolute-names \ --absolute-names \
--hard-dereference \ --hard-dereference \
--transform "s,$TMPDIR/install,$dir/install," \ --transform "s,$TMPDIR/install,$dir/install," \
--transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
--transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
--transform "s,$NIX_STORE,$dir/store,S" \ --transform "s,$NIX_STORE,$dir/store,S" \
$TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \ $TMPDIR/install \
$TMPDIR/create-darwin-volume.sh \
$TMPDIR/install-darwin-multi-user.sh \
$TMPDIR/install-systemd-multi-user.sh \ $TMPDIR/install-systemd-multi-user.sh \
$TMPDIR/install-multi-user $TMPDIR/reginfo \ $TMPDIR/install-multi-user \
$TMPDIR/reginfo \
$(cat ${installerClosureInfo}/store-paths) $(cat ${installerClosureInfo}/store-paths)
''); '');

185
scripts/create-darwin-volume.sh Executable file
View file

@ -0,0 +1,185 @@
#!/bin/sh
set -e
root_disk() {
diskutil info -plist /
}
apfs_volumes_for() {
disk=$1
diskutil apfs list -plist "$disk"
}
disk_identifier() {
xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" 2>/dev/null
}
volume_list_true() {
key=$1
xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='$key']/following-sibling::true[1]" 2> /dev/null
}
volume_get_string() {
key=$1 i=$2
xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict[$i]/key[text()='$key']/following-sibling::string[1]/text()" 2> /dev/null
}
find_nix_volume() {
disk=$1
i=1
volumes=$(apfs_volumes_for "$disk")
while true; do
name=$(echo "$volumes" | volume_get_string "Name" "$i")
if [ -z "$name" ]; then
break
fi
case "$name" in
[Nn]ix*)
echo "$name"
break
;;
esac
i=$((i+1))
done
}
test_fstab() {
grep -q "/nix apfs rw" /etc/fstab 2>/dev/null
}
test_nix_symlink() {
[ -L "/nix" ] || grep -q "^nix." /etc/synthetic.conf 2>/dev/null
}
test_synthetic_conf() {
grep -q "^nix$" /etc/synthetic.conf 2>/dev/null
}
test_nix() {
test -d "/nix"
}
test_t2_chip_present(){
# Use xartutil to see if system has a t2 chip.
#
# This isn't well-documented on its own; until it is,
# let's keep track of knowledge/assumptions.
#
# Warnings:
# - Don't search "xart" if porn will cause you trouble :)
# - Other xartutil flags do dangerous things. Don't run them
# naively. If you must, search "xartutil" first.
#
# Assumptions:
# - the "xART session seeds recovery utility"
# appears to interact with xartstorageremoted
# - `sudo xartutil --list` lists xART sessions
# and their seeds and exits 0 if successful. If
# not, it exits 1 and prints an error such as:
# xartutil: ERROR: No supported link to the SEP present
# - xART sessions/seeds are present when a T2 chip is
# (and not, otherwise)
# - the presence of a T2 chip means a newly-created
# volume on the primary drive will be
# encrypted at rest
# - all together: `sudo xartutil --list`
# should exit 0 if a new Nix Store volume will
# be encrypted at rest, and exit 1 if not.
sudo xartutil --list >/dev/null 2>/dev/null
}
test_filevault_in_use() {
disk=$1
# list vols on disk | get value of Filevault key | value is true
apfs_volumes_for "$disk" | volume_list_true FileVault | grep -q true
}
# use after error msg for conditions we don't understand
suggest_report_error(){
# ex "error: something sad happened :(" >&2
echo " please report this @ https://github.com/nixos/nix/issues" >&2
}
main() {
(
echo ""
echo " ------------------------------------------------------------------ "
echo " | This installer will create a volume for the nix store and |"
echo " | configure it to mount at /nix. Follow these steps to uninstall. |"
echo " ------------------------------------------------------------------ "
echo ""
echo " 1. Remove the entry from fstab using 'sudo vifs'"
echo " 2. Destroy the data volume using 'diskutil apfs deleteVolume'"
echo " 3. Remove the 'nix' line from /etc/synthetic.conf or the file"
echo ""
) >&2
if test_nix_symlink; then
echo "error: /nix is a symlink, please remove it and make sure it's not in synthetic.conf (in which case a reboot is required)" >&2
echo " /nix -> $(readlink "/nix")" >&2
exit 2
fi
if ! test_synthetic_conf; then
echo "Configuring /etc/synthetic.conf..." >&2
echo nix | sudo tee -a /etc/synthetic.conf
if ! test_synthetic_conf; then
echo "error: failed to configure synthetic.conf;" >&2
suggest_report_error
exit 1
fi
fi
if ! test_nix; then
echo "Creating mountpoint for /nix..." >&2
/System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B || true
if ! test_nix; then
sudo mkdir -p /nix 2>/dev/null || true
fi
if ! test_nix; then
echo "error: failed to bootstrap /nix; if a reboot doesn't help," >&2
suggest_report_error
exit 1
fi
fi
disk=$(root_disk | disk_identifier)
volume=$(find_nix_volume "$disk")
if [ -z "$volume" ]; then
echo "Creating a Nix Store volume..." >&2
if test_filevault_in_use "$disk"; then
# TODO: Not sure if it's in-scope now, but `diskutil apfs list`
# shows both filevault and encrypted at rest status, and it
# may be the more semantic way to test for this? It'll show
# `FileVault: No (Encrypted at rest)`
# `FileVault: No`
# `FileVault: Yes (Unlocked)`
# and so on.
if test_t2_chip_present; then
echo "warning: boot volume is FileVault-encrypted, but the Nix store volume" >&2
echo " is only encrypted at rest." >&2
echo " See https://nixos.org/nix/manual/#sect-macos-installation" >&2
else
echo "error: refusing to create Nix store volume because the boot volume is" >&2
echo " FileVault encrypted, but encryption-at-rest is not available." >&2
echo " Manually create a volume for the store and re-run this script." >&2
echo " See https://nixos.org/nix/manual/#sect-macos-installation" >&2
exit 1
fi
fi
sudo diskutil apfs addVolume "$disk" APFS 'Nix Store' -mountpoint /nix
volume="Nix Store"
else
echo "Using existing '$volume' volume" >&2
fi
if ! test_fstab; then
echo "Configuring /etc/fstab..." >&2
label=$(echo "$volume" | sed 's/ /\\040/g')
printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs
fi
}
main "$@"

View file

@ -13,22 +13,25 @@ set -o pipefail
# however tracking which bits came from which would be impossible. # however tracking which bits came from which would be impossible.
readonly ESC='\033[0m' readonly ESC='\033[0m'
readonly BOLD='\033[38;1m' readonly BOLD='\033[1m'
readonly BLUE='\033[38;34m' readonly BLUE='\033[34m'
readonly BLUE_UL='\033[38;4;34m' readonly BLUE_UL='\033[4;34m'
readonly GREEN='\033[38;32m' readonly GREEN='\033[32m'
readonly GREEN_UL='\033[38;4;32m' readonly GREEN_UL='\033[4;32m'
readonly RED='\033[38;31m' readonly RED='\033[31m'
readonly NIX_USER_COUNT="32" # installer allows overriding build user count to speed up installation
# as creating each user takes non-trivial amount of time on macos
readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32}
readonly NIX_BUILD_GROUP_ID="30000" readonly NIX_BUILD_GROUP_ID="30000"
readonly NIX_BUILD_GROUP_NAME="nixbld" readonly NIX_BUILD_GROUP_NAME="nixbld"
readonly NIX_FIRST_BUILD_UID="30001" readonly NIX_FIRST_BUILD_UID="30001"
# Please don't change this. We don't support it, because the # Please don't change this. We don't support it, because the
# default shell profile that comes with Nix doesn't support it. # default shell profile that comes with Nix doesn't support it.
readonly NIX_ROOT="/nix" readonly NIX_ROOT="/nix"
readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-}
readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc") readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv")
readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
@ -450,9 +453,11 @@ create_directories() {
} }
place_channel_configuration() { place_channel_configuration() {
echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels" if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then
_sudo "to set up the default system channel (part 1)" \ echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels"
install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" _sudo "to set up the default system channel (part 1)" \
install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels"
fi
} }
welcome_to_nix() { welcome_to_nix() {
@ -567,7 +572,7 @@ install_from_extracted_nix() {
cd "$EXTRACTED_NIX_PATH" cd "$EXTRACTED_NIX_PATH"
_sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \ _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
rsync -rlpt ./store/* "$NIX_ROOT/store/" rsync -rlpt --chmod=-w ./store/* "$NIX_ROOT/store/"
if [ -d "$NIX_INSTALLED_NIX" ]; then if [ -d "$NIX_INSTALLED_NIX" ]; then
echo " Alright! We have our first nix at $NIX_INSTALLED_NIX" echo " Alright! We have our first nix at $NIX_INSTALLED_NIX"
@ -634,18 +639,20 @@ setup_default_profile() {
export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
fi fi
# Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call, if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then
# otherwise it will be lost in environments where sudo doesn't pass # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call,
# all the environment variables by default. # otherwise it will be lost in environments where sudo doesn't pass
_sudo "to update the default channel in the default profile" \ # all the environment variables by default.
HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \ _sudo "to update the default channel in the default profile" \
|| channel_update_failed=1 HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \
|| channel_update_failed=1
fi
} }
place_nix_configuration() { place_nix_configuration() {
cat <<EOF > "$SCRATCH/nix.conf" cat <<EOF > "$SCRATCH/nix.conf"
$NIX_EXTRA_CONF
build-users-group = $NIX_BUILD_GROUP_NAME build-users-group = $NIX_BUILD_GROUP_NAME
EOF EOF
_sudo "to place the default nix daemon configuration (part 2)" \ _sudo "to place the default nix daemon configuration (part 2)" \

View file

@ -40,29 +40,85 @@ elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
fi fi
INSTALL_MODE=no-daemon INSTALL_MODE=no-daemon
# Trivially handle the --daemon / --no-daemon options CREATE_DARWIN_VOLUME=0
if [ "x${1:-}" = "x--no-daemon" ]; then # handle the command line flags
INSTALL_MODE=no-daemon while [ $# -gt 0 ]; do
elif [ "x${1:-}" = "x--daemon" ]; then case $1 in
INSTALL_MODE=daemon --daemon)
elif [ "x${1:-}" != "x" ]; then INSTALL_MODE=daemon;;
( --no-daemon)
echo "Nix Installer [--daemon|--no-daemon]" INSTALL_MODE=no-daemon;;
--no-channel-add)
export NIX_INSTALLER_NO_CHANNEL_ADD=1;;
--daemon-user-count)
export NIX_USER_COUNT=$2
shift;;
--no-modify-profile)
NIX_INSTALLER_NO_MODIFY_PROFILE=1;;
--darwin-use-unencrypted-nix-store-volume)
CREATE_DARWIN_VOLUME=1;;
--nix-extra-conf-file)
export NIX_EXTRA_CONF="$(cat $2)"
shift;;
*)
(
echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--darwin-use-unencrypted-nix-store-volume] [--nix-extra-conf-file FILE]"
echo "Choose installation method." echo "Choose installation method."
echo "" echo ""
echo " --daemon: Installs and configures a background daemon that manages the store," echo " --daemon: Installs and configures a background daemon that manages the store,"
echo " providing multi-user support and better isolation for local builds." echo " providing multi-user support and better isolation for local builds."
echo " Both for security and reproducibility, this method is recommended if" echo " Both for security and reproducibility, this method is recommended if"
echo " supported on your platform." echo " supported on your platform."
echo " See https://nixos.org/nix/manual/#sect-multi-user-installation" echo " See https://nixos.org/nix/manual/#sect-multi-user-installation"
echo "" echo ""
echo " --no-daemon: Simple, single-user installation that does not require root and is" echo " --no-daemon: Simple, single-user installation that does not require root and is"
echo " trivial to uninstall." echo " trivial to uninstall."
echo " (default)" echo " (default)"
echo "" echo ""
) >&2 echo " --no-channel-add: Don't add any channels. nixpkgs-unstable is installed by default."
exit echo ""
echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable"
echo " is installed by default."
echo ""
echo " --daemon-user-count: Number of build users to create. Defaults to 32."
echo ""
echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix.conf"
echo ""
) >&2
# darwin and Catalina+
if [ "$(uname -s)" = "Darwin" ] && [ "$macos_major" -gt 14 ]; then
(
echo " --darwin-use-unencrypted-nix-store-volume: Create an APFS volume for the Nix"
echo " store and mount it at /nix. This is the recommended way to create"
echo " /nix with a read-only / on macOS >=10.15."
echo " See: https://nixos.org/nix/manual/#sect-macos-installation"
echo ""
) >&2
fi
exit;;
esac
shift
done
if [ "$(uname -s)" = "Darwin" ]; then
if [ "$CREATE_DARWIN_VOLUME" = 1 ]; then
printf '\e[1;31mCreating volume and mountpoint /nix.\e[0m\n'
"$self/create-darwin-volume.sh"
fi
info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null)
if ! [ -e $dest ] && [ -n "$info" ] && [ "$macos_major" -gt 14 ]; then
(
echo ""
echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume."
echo "Use sh <(curl https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually."
echo "See https://nixos.org/nix/manual/#sect-macos-installation"
echo ""
) >&2
exit 1
fi
fi fi
if [ "$INSTALL_MODE" = "daemon" ]; then if [ "$INSTALL_MODE" = "daemon" ]; then
@ -130,13 +186,15 @@ if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then
fi fi
# Subscribe the user to the Nixpkgs channel and fetch it. # Subscribe the user to the Nixpkgs channel and fetch it.
if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then
$nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then
fi $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
if [ -z "$_NIX_INSTALLER_TEST" ]; then fi
if ! $nix/bin/nix-channel --update nixpkgs; then if [ -z "$_NIX_INSTALLER_TEST" ]; then
echo "Fetching the nixpkgs channel failed. (Are you offline?)" if ! $nix/bin/nix-channel --update nixpkgs; then
echo "To try again later, run \"nix-channel --update nixpkgs\"." echo "Fetching the nixpkgs channel failed. (Are you offline?)"
echo "To try again later, run \"nix-channel --update nixpkgs\"."
fi
fi fi
fi fi
@ -155,6 +213,17 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
break break
fi fi
done done
for i in .zshenv .zshrc; do
fn="$HOME/$i"
if [ -w "$fn" ]; then
if ! grep -q "$p" "$fn"; then
echo "modifying $fn..." >&2
echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
fi
added=1
break
fi
done
fi fi
if [ -z "$added" ]; then if [ -z "$added" ]; then

View file

@ -36,6 +36,9 @@ tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")"
require_util curl "download the binary tarball" require_util curl "download the binary tarball"
require_util tar "unpack the binary tarball" require_util tar "unpack the binary tarball"
if [ "$(uname -s)" != "Darwin" ]; then
require_util xz "unpack the binary tarball"
fi
echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
curl -L "$url" -o "$tarball" || oops "failed to download '$url'" curl -L "$url" -o "$tarball" || oops "failed to download '$url'"

View file

@ -17,7 +17,7 @@
#include "store-api.hh" #include "store-api.hh"
#include "derivations.hh" #include "derivations.hh"
#include "local-store.hh" #include "local-store.hh"
#include "legacy.hh" #include "../nix/legacy.hh"
using namespace nix; using namespace nix;
using std::cin; using std::cin;
@ -200,9 +200,12 @@ static int _main(int argc, char * * argv)
} catch (std::exception & e) { } catch (std::exception & e) {
auto msg = chomp(drainFD(5, false)); auto msg = chomp(drainFD(5, false));
printError("cannot build on '%s': %s%s", logError({
bestMachine->storeUri, e.what(), .name = "Remote build",
(msg.empty() ? "" : ": " + msg)); .hint = hintfmt("cannot build on '%s': %s%s",
bestMachine->storeUri, e.what(),
(msg.empty() ? "" : ": " + msg))
});
bestMachine->enabled = false; bestMachine->enabled = false;
continue; continue;
} }
@ -241,7 +244,7 @@ connected:
uploadLock = -1; uploadLock = -1;
BasicDerivation drv(readDerivation(*store, store->realStoreDir + "/" + std::string(drvPath->to_string()))); auto drv = store->readDerivation(*drvPath);
drv.inputSrcs = store->parseStorePathSet(inputs); drv.inputSrcs = store->parseStorePathSet(inputs);
auto result = sshStore->buildDerivation(*drvPath, drv); auto result = sshStore->buildDerivation(*drvPath, drv);

View file

@ -19,7 +19,7 @@ static Strings parseAttrPath(const string & s)
++i; ++i;
while (1) { while (1) {
if (i == s.end()) if (i == s.end())
throw Error(format("missing closing quote in selection path '%1%'") % s); throw Error("missing closing quote in selection path '%1%'", s);
if (*i == '"') break; if (*i == '"') break;
cur.push_back(*i++); cur.push_back(*i++);
} }
@ -32,15 +32,13 @@ static Strings parseAttrPath(const string & s)
} }
Value * findAlongAttrPath(EvalState & state, const string & attrPath, std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
Bindings & autoArgs, Value & vIn) Bindings & autoArgs, Value & vIn)
{ {
Strings tokens = parseAttrPath(attrPath); Strings tokens = parseAttrPath(attrPath);
Error attrError =
Error(format("attribute selection path '%1%' does not match expression") % attrPath);
Value * v = &vIn; Value * v = &vIn;
Pos pos = noPos;
for (auto & attr : tokens) { for (auto & attr : tokens) {
@ -62,34 +60,36 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
if (v->type != tAttrs) if (v->type != tAttrs)
throw TypeError( throw TypeError(
format("the expression selected by the selection path '%1%' should be a set but is %2%") "the expression selected by the selection path '%1%' should be a set but is %2%",
% attrPath % showType(*v)); attrPath,
showType(*v));
if (attr.empty()) if (attr.empty())
throw Error(format("empty attribute name in selection path '%1%'") % attrPath); throw Error("empty attribute name in selection path '%1%'", attrPath);
Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
if (a == v->attrs->end()) if (a == v->attrs->end())
throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath); throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
v = &*a->value; v = &*a->value;
pos = *a->pos;
} }
else if (apType == apIndex) { else if (apType == apIndex) {
if (!v->isList()) if (!v->isList())
throw TypeError( throw TypeError(
format("the expression selected by the selection path '%1%' should be a list but is %2%") "the expression selected by the selection path '%1%' should be a list but is %2%",
% attrPath % showType(*v)); attrPath,
showType(*v));
if (attrIndex >= v->listSize()) if (attrIndex >= v->listSize())
throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath); throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
v = v->listElems()[attrIndex]; v = v->listElems()[attrIndex];
pos = noPos;
} }
} }
return v; return {v, pos};
} }
@ -98,9 +98,9 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
Value * v2; Value * v2;
try { try {
auto dummyArgs = state.allocBindings(0); auto dummyArgs = state.allocBindings(0);
v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v); v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first;
} catch (Error &) { } catch (Error &) {
throw Error("package '%s' has no source location information", what); throw NoPositionInfo("package '%s' has no source location information", what);
} }
// FIXME: is it possible to extract the Pos object instead of doing this // FIXME: is it possible to extract the Pos object instead of doing this

View file

@ -7,7 +7,10 @@
namespace nix { namespace nix {
Value * findAlongAttrPath(EvalState & state, const string & attrPath, MakeError(AttrPathNotFound, Error);
MakeError(NoPositionInfo, Error);
std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
Bindings & autoArgs, Value & vIn); Bindings & autoArgs, Value & vIn);
/* Heuristic to find the filename and lineno or a nix value. */ /* Heuristic to find the filename and lineno or a nix value. */

View file

@ -43,6 +43,12 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
} }
Value * EvalState::allocAttr(Value & vAttrs, const std::string & name)
{
return allocAttr(vAttrs, symbols.create(name));
}
void Bindings::sort() void Bindings::sort()
{ {
std::sort(begin(), end()); std::sort(begin(), end());

View file

@ -76,7 +76,11 @@ public:
{ {
auto a = get(name); auto a = get(name);
if (!a) if (!a)
throw Error("attribute '%s' missing, at %s", name, pos); throw Error({
.hint = hintfmt("attribute '%s' missing", name),
.nixCode = NixCode { .errPos = pos }
});
return *a; return *a;
} }

View file

@ -1,31 +1,36 @@
#include "common-eval-args.hh" #include "common-eval-args.hh"
#include "shared.hh" #include "shared.hh"
#include "download.hh" #include "filetransfer.hh"
#include "util.hh" #include "util.hh"
#include "eval.hh" #include "eval.hh"
#include "fetchers.hh"
#include "store-api.hh"
namespace nix { namespace nix {
MixEvalArgs::MixEvalArgs() MixEvalArgs::MixEvalArgs()
{ {
mkFlag() addFlag({
.longName("arg") .longName = "arg",
.description("argument to be passed to Nix functions") .description = "argument to be passed to Nix functions",
.labels({"name", "expr"}) .labels = {"name", "expr"},
.handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; }); .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }}
});
mkFlag() addFlag({
.longName("argstr") .longName = "argstr",
.description("string-valued argument to be passed to Nix functions") .description = "string-valued argument to be passed to Nix functions",
.labels({"name", "string"}) .labels = {"name", "string"},
.handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; }); .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }},
});
mkFlag() addFlag({
.shortName('I') .longName = "include",
.longName("include") .shortName = 'I',
.description("add a path to the list of locations used to look up <...> file names") .description = "add a path to the list of locations used to look up <...> file names",
.label("path") .labels = {"path"},
.handler([&](std::string s) { searchPath.push_back(s); }); .handler = {[&](std::string s) { searchPath.push_back(s); }}
});
} }
Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
@ -46,9 +51,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
Path lookupFileArg(EvalState & state, string s) Path lookupFileArg(EvalState & state, string s)
{ {
if (isUri(s)) { if (isUri(s)) {
CachedDownloadRequest request(s); return state.store->toRealPath(
request.unpack = true; fetchers::downloadTarball(
return getDownloader()->downloadCached(state.store, request).path; state.store, resolveUri(s), "source", false).storePath);
} else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
Path p = s.substr(1, s.size() - 2); Path p = s.substr(1, s.size() - 2);
return state.findFile(p); return state.findFile(p);

View file

@ -7,20 +7,26 @@
namespace nix { namespace nix {
LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos)) LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
{ {
throw EvalError(format(s) % pos); throw EvalError({
.hint = hintfmt(s),
.nixCode = NixCode { .errPos = pos }
});
} }
LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
{ {
throw TypeError(format(s) % showType(v)); throw TypeError(s, showType(v));
} }
LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos)) LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v))
{ {
throw TypeError(format(s) % showType(v) % pos); throw TypeError({
.hint = hintfmt(s, showType(v)),
.nixCode = NixCode { .errPos = pos }
});
} }
@ -43,7 +49,7 @@ void EvalState::forceValue(Value & v, const Pos & pos)
else if (v.type == tApp) else if (v.type == tApp)
callFunction(*v.app.left, *v.app.right, v, noPos); callFunction(*v.app.left, *v.app.right, v, noPos);
else if (v.type == tBlackhole) else if (v.type == tBlackhole)
throwEvalError("infinite recursion encountered, at %1%", pos); throwEvalError(pos, "infinite recursion encountered");
} }
@ -57,9 +63,9 @@ inline void EvalState::forceAttrs(Value & v)
inline void EvalState::forceAttrs(Value & v, const Pos & pos) inline void EvalState::forceAttrs(Value & v, const Pos & pos)
{ {
forceValue(v); forceValue(v, pos);
if (v.type != tAttrs) if (v.type != tAttrs)
throwTypeError("value is %1% while a set was expected, at %2%", v, pos); throwTypeError(pos, "value is %1% while a set was expected", v);
} }
@ -73,9 +79,9 @@ inline void EvalState::forceList(Value & v)
inline void EvalState::forceList(Value & v, const Pos & pos) inline void EvalState::forceList(Value & v, const Pos & pos)
{ {
forceValue(v); forceValue(v, pos);
if (!v.isList()) if (!v.isList())
throwTypeError("value is %1% while a list was expected, at %2%", v, pos); throwTypeError(pos, "value is %1% while a list was expected", v);
} }
/* Note: Various places expect the allocated memory to be zeroed. */ /* Note: Various places expect the allocated memory to be zeroed. */

View file

@ -5,7 +5,7 @@
#include "derivations.hh" #include "derivations.hh"
#include "globals.hh" #include "globals.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "download.hh" #include "filetransfer.hh"
#include "json.hh" #include "json.hh"
#include "function-trace.hh" #include "function-trace.hh"
@ -22,6 +22,8 @@
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
#define GC_INCLUDE_NEW
#include <gc/gc.h> #include <gc/gc.h>
#include <gc/gc_cpp.h> #include <gc/gc_cpp.h>
@ -56,6 +58,12 @@ static char * dupStringWithLen(const char * s, size_t size)
} }
RootValue allocRootValue(Value * v)
{
return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v);
}
static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v) static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
{ {
checkInterrupt(); checkInterrupt();
@ -493,52 +501,74 @@ Value & EvalState::getBuiltin(const string & name)
LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2)) LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2))
{ {
throw EvalError(format(s) % s2); throw EvalError(s, s2);
} }
LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const Pos & pos)) LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2))
{ {
throw EvalError(format(s) % s2 % pos); throw EvalError({
.hint = hintfmt(s, s2),
.nixCode = NixCode { .errPos = pos }
});
} }
LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3)) LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3))
{ {
throw EvalError(format(s) % s2 % s3); throw EvalError(s, s2, s3);
} }
LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3, const Pos & pos)) LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3))
{ {
throw EvalError(format(s) % s2 % s3 % pos); throw EvalError({
.hint = hintfmt(s, s2, s3),
.nixCode = NixCode { .errPos = pos }
});
} }
LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2)) LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const Symbol & sym, const Pos & p2))
{ {
throw EvalError(format(s) % sym % p1 % p2); // p1 is where the error occurred; p2 is a position mentioned in the message.
throw EvalError({
.hint = hintfmt(s, sym, p2),
.nixCode = NixCode { .errPos = p1 }
});
} }
LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos)) LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
{ {
throw TypeError(format(s) % pos); throw TypeError({
.hint = hintfmt(s),
.nixCode = NixCode { .errPos = pos }
});
} }
LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1))
{ {
throw TypeError(format(s) % s1); throw TypeError(s, s1);
} }
LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2, const Pos & pos)) LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2))
{ {
throw TypeError(format(s) % fun.showNamePos() % s2 % pos); throw TypeError({
.hint = hintfmt(s, fun.showNamePos(), s2),
.nixCode = NixCode { .errPos = pos }
});
} }
LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos)) LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1))
{ {
throw AssertionError(format(s) % s1 % pos); throw AssertionError({
.hint = hintfmt(s, s1),
.nixCode = NixCode { .errPos = pos }
});
} }
LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos)) LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1))
{ {
throw UndefinedVarError(format(s) % s1 % pos); throw UndefinedVarError({
.hint = hintfmt(s, s1),
.nixCode = NixCode { .errPos = pos }
});
} }
LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2)) LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2))
@ -606,7 +636,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
return j->value; return j->value;
} }
if (!env->prevWith) if (!env->prevWith)
throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, var.pos); throwUndefinedVarError(var.pos, "undefined variable '%1%'", var.name);
for (size_t l = env->prevWith; l; --l, env = env->up) ; for (size_t l = env->prevWith; l; --l, env = env->up) ;
} }
} }
@ -804,7 +834,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
Value v; Value v;
e->eval(*this, env, v); e->eval(*this, env, v);
if (v.type != tBool) if (v.type != tBool)
throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); throwTypeError(pos, "value is %1% while a Boolean was expected", v);
return v.boolean; return v.boolean;
} }
@ -918,7 +948,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
Symbol nameSym = state.symbols.create(nameVal.string.s); Symbol nameSym = state.symbols.create(nameVal.string.s);
Bindings::iterator j = v.attrs->find(nameSym); Bindings::iterator j = v.attrs->find(nameSym);
if (j != v.attrs->end()) if (j != v.attrs->end())
throwEvalError("dynamic attribute '%1%' at %2% already defined at %3%", nameSym, i.pos, *j->pos); throwEvalError(i.pos, "dynamic attribute '%1%' already defined at %2%", nameSym, *j->pos);
i.valueExpr->setName(nameSym); i.valueExpr->setName(nameSym);
/* Keep sorted order so find can catch duplicates */ /* Keep sorted order so find can catch duplicates */
@ -1006,7 +1036,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
} else { } else {
state.forceAttrs(*vAttrs, pos); state.forceAttrs(*vAttrs, pos);
if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
throwEvalError("attribute '%1%' missing, at %2%", name, pos); throwEvalError(pos, "attribute '%1%' missing", name);
} }
vAttrs = j->value; vAttrs = j->value;
pos2 = j->pos; pos2 = j->pos;
@ -1132,7 +1162,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
} }
if (fun.type != tLambda) if (fun.type != tLambda)
throwTypeError("attempt to call something which is not a function but %1%, at %2%", fun, pos); throwTypeError(pos, "attempt to call something which is not a function but %1%", fun);
ExprLambda & lambda(*fun.lambda.fun); ExprLambda & lambda(*fun.lambda.fun);
@ -1160,8 +1190,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
for (auto & i : lambda.formals->formals) { for (auto & i : lambda.formals->formals) {
Bindings::iterator j = arg.attrs->find(i.name); Bindings::iterator j = arg.attrs->find(i.name);
if (j == arg.attrs->end()) { if (j == arg.attrs->end()) {
if (!i.def) throwTypeError("%1% called without required argument '%2%', at %3%", if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'",
lambda, i.name, pos); lambda, i.name);
env2.values[displ++] = i.def->maybeThunk(*this, env2); env2.values[displ++] = i.def->maybeThunk(*this, env2);
} else { } else {
attrsUsed++; attrsUsed++;
@ -1176,7 +1206,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
user. */ user. */
for (auto & i : *arg.attrs) for (auto & i : *arg.attrs)
if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end())
throwTypeError("%1% called with unexpected argument '%2%', at %3%", lambda, i.name, pos); throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name);
abort(); // can't happen abort(); // can't happen
} }
} }
@ -1256,7 +1286,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
void ExprIf::eval(EvalState & state, Env & env, Value & v) void ExprIf::eval(EvalState & state, Env & env, Value & v)
{ {
(state.evalBool(env, cond) ? then : else_)->eval(state, env, v); (state.evalBool(env, cond, pos) ? then : else_)->eval(state, env, v);
} }
@ -1265,7 +1295,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
if (!state.evalBool(env, cond, pos)) { if (!state.evalBool(env, cond, pos)) {
std::ostringstream out; std::ostringstream out;
cond->show(out); cond->show(out);
throwAssertionError("assertion '%1%' failed at %2%", out.str(), pos); throwAssertionError(pos, "assertion '%1%' failed at %2%", out.str());
} }
body->eval(state, env, v); body->eval(state, env, v);
} }
@ -1417,14 +1447,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
nf = n; nf = n;
nf += vTmp.fpoint; nf += vTmp.fpoint;
} else } else
throwEvalError("cannot add %1% to an integer, at %2%", showType(vTmp), pos); throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp));
} else if (firstType == tFloat) { } else if (firstType == tFloat) {
if (vTmp.type == tInt) { if (vTmp.type == tInt) {
nf += vTmp.integer; nf += vTmp.integer;
} else if (vTmp.type == tFloat) { } else if (vTmp.type == tFloat) {
nf += vTmp.fpoint; nf += vTmp.fpoint;
} else } else
throwEvalError("cannot add %1% to a float, at %2%", showType(vTmp), pos); throwEvalError(pos, "cannot add %1% to a float", showType(vTmp));
} else } else
s << state.coerceToString(pos, vTmp, context, false, firstType == tString); s << state.coerceToString(pos, vTmp, context, false, firstType == tString);
} }
@ -1435,7 +1465,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
mkFloat(v, nf); mkFloat(v, nf);
else if (firstType == tPath) { else if (firstType == tPath) {
if (!context.empty()) if (!context.empty())
throwEvalError("a string that refers to a store path cannot be appended to a path, at %1%", pos); throwEvalError(pos, "a string that refers to a store path cannot be appended to a path");
auto path = canonPath(s.str()); auto path = canonPath(s.str());
mkPath(v, path.c_str()); mkPath(v, path.c_str());
} else } else
@ -1484,7 +1514,7 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos)
{ {
forceValue(v, pos); forceValue(v, pos);
if (v.type != tInt) if (v.type != tInt)
throwTypeError("value is %1% while an integer was expected, at %2%", v, pos); throwTypeError(pos, "value is %1% while an integer was expected", v);
return v.integer; return v.integer;
} }
@ -1495,16 +1525,16 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
if (v.type == tInt) if (v.type == tInt)
return v.integer; return v.integer;
else if (v.type != tFloat) else if (v.type != tFloat)
throwTypeError("value is %1% while a float was expected, at %2%", v, pos); throwTypeError(pos, "value is %1% while a float was expected", v);
return v.fpoint; return v.fpoint;
} }
bool EvalState::forceBool(Value & v, const Pos & pos) bool EvalState::forceBool(Value & v, const Pos & pos)
{ {
forceValue(v); forceValue(v, pos);
if (v.type != tBool) if (v.type != tBool)
throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); throwTypeError(pos, "value is %1% while a Boolean was expected", v);
return v.boolean; return v.boolean;
} }
@ -1517,9 +1547,9 @@ bool EvalState::isFunctor(Value & fun)
void EvalState::forceFunction(Value & v, const Pos & pos) void EvalState::forceFunction(Value & v, const Pos & pos)
{ {
forceValue(v); forceValue(v, pos);
if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v)) if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v))
throwTypeError("value is %1% while a function was expected, at %2%", v, pos); throwTypeError(pos, "value is %1% while a function was expected", v);
} }
@ -1528,7 +1558,7 @@ string EvalState::forceString(Value & v, const Pos & pos)
forceValue(v, pos); forceValue(v, pos);
if (v.type != tString) { if (v.type != tString) {
if (pos) if (pos)
throwTypeError("value is %1% while a string was expected, at %2%", v, pos); throwTypeError(pos, "value is %1% while a string was expected", v);
else else
throwTypeError("value is %1% while a string was expected", v); throwTypeError("value is %1% while a string was expected", v);
} }
@ -1557,8 +1587,8 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
string s = forceString(v, pos); string s = forceString(v, pos);
if (v.string.context) { if (v.string.context) {
if (pos) if (pos)
throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%'), at %3%", throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')",
v.string.s, v.string.context[0], pos); v.string.s, v.string.context[0]);
else else
throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')", throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')",
v.string.s, v.string.context[0]); v.string.s, v.string.context[0]);
@ -1594,7 +1624,7 @@ std::optional<string> EvalState::tryAttrsToString(const Pos & pos, Value & v,
string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
bool coerceMore, bool copyToStore) bool coerceMore, bool copyToStore)
{ {
forceValue(v); forceValue(v, pos);
string s; string s;
@ -1614,7 +1644,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
return *maybeString; return *maybeString;
} }
auto i = v.attrs->find(sOutPath); auto i = v.attrs->find(sOutPath);
if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string, at %1%", pos); if (i == v.attrs->end()) throwTypeError(pos, "cannot coerce a set to a string");
return coerceToString(pos, *i->value, context, coerceMore, copyToStore); return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
} }
@ -1645,7 +1675,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
} }
} }
throwTypeError("cannot coerce %1% to a string, at %2%", v, pos); throwTypeError(pos, "cannot coerce %1% to a string", v);
} }
@ -1661,7 +1691,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
else { else {
auto p = settings.readOnlyMode auto p = settings.readOnlyMode
? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
: store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
dstPath = store->printStorePath(p); dstPath = store->printStorePath(p);
srcToStore.insert_or_assign(path, std::move(p)); srcToStore.insert_or_assign(path, std::move(p));
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);
@ -1676,7 +1706,7 @@ Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context)
{ {
string path = coerceToString(pos, v, context, false, false); string path = coerceToString(pos, v, context, false, false);
if (path == "" || path[0] != '/') if (path == "" || path[0] != '/')
throwEvalError("string '%1%' doesn't represent an absolute path, at %2%", path, pos); throwEvalError(pos, "string '%1%' doesn't represent an absolute path", path);
return path; return path;
} }
@ -1883,8 +1913,10 @@ void EvalState::printStats()
string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const
{ {
throw TypeError(format("cannot coerce %1% to a string, at %2%") % throw TypeError({
showType() % pos); .hint = hintfmt("cannot coerce %1% to a string", showType()),
.nixCode = NixCode { .errPos = pos }
});
} }

View file

@ -272,6 +272,7 @@ public:
Env & allocEnv(size_t size); Env & allocEnv(size_t size);
Value * allocAttr(Value & vAttrs, const Symbol & name); Value * allocAttr(Value & vAttrs, const Symbol & name);
Value * allocAttr(Value & vAttrs, const std::string & name);
Bindings * allocBindings(size_t capacity); Bindings * allocBindings(size_t capacity);
@ -367,7 +368,7 @@ struct EvalSettings : Config
"Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
Setting<bool> traceFunctionCalls{this, false, "trace-function-calls", Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
"Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"}; "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
}; };
extern EvalSettings evalSettings; extern EvalSettings evalSettings;

View file

@ -1,4 +1,5 @@
#include "function-trace.hh" #include "function-trace.hh"
#include "logging.hh"
namespace nix { namespace nix {

View file

@ -4,7 +4,6 @@
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
using json = nlohmann::json; using json = nlohmann::json;
using std::unique_ptr;
namespace nix { namespace nix {
@ -13,69 +12,69 @@ namespace nix {
class JSONSax : nlohmann::json_sax<json> { class JSONSax : nlohmann::json_sax<json> {
class JSONState { class JSONState {
protected: protected:
unique_ptr<JSONState> parent; std::unique_ptr<JSONState> parent;
Value * v; RootValue v;
public: public:
virtual unique_ptr<JSONState> resolve(EvalState &) virtual std::unique_ptr<JSONState> resolve(EvalState &)
{ {
throw std::logic_error("tried to close toplevel json parser state"); throw std::logic_error("tried to close toplevel json parser state");
}; }
explicit JSONState(unique_ptr<JSONState>&& p) : parent(std::move(p)), v(nullptr) {}; explicit JSONState(std::unique_ptr<JSONState> && p) : parent(std::move(p)) {}
explicit JSONState(Value* v) : v(v) {}; explicit JSONState(Value * v) : v(allocRootValue(v)) {}
JSONState(JSONState& p) = delete; JSONState(JSONState & p) = delete;
Value& value(EvalState & state) Value & value(EvalState & state)
{ {
if (v == nullptr) if (!v)
v = state.allocValue(); v = allocRootValue(state.allocValue());
return *v; return **v;
}; }
virtual ~JSONState() {}; virtual ~JSONState() {}
virtual void add() {}; virtual void add() {}
}; };
class JSONObjectState : public JSONState { class JSONObjectState : public JSONState {
using JSONState::JSONState; using JSONState::JSONState;
ValueMap attrs = ValueMap(); ValueMap attrs;
virtual unique_ptr<JSONState> resolve(EvalState & state) override std::unique_ptr<JSONState> resolve(EvalState & state) override
{ {
Value& v = parent->value(state); Value & v = parent->value(state);
state.mkAttrs(v, attrs.size()); state.mkAttrs(v, attrs.size());
for (auto & i : attrs) for (auto & i : attrs)
v.attrs->push_back(Attr(i.first, i.second)); v.attrs->push_back(Attr(i.first, i.second));
return std::move(parent); return std::move(parent);
} }
virtual void add() override { v = nullptr; }; void add() override { v = nullptr; }
public: public:
void key(string_t& name, EvalState & state) void key(string_t & name, EvalState & state)
{ {
attrs[state.symbols.create(name)] = &value(state); attrs.insert_or_assign(state.symbols.create(name), &value(state));
} }
}; };
class JSONListState : public JSONState { class JSONListState : public JSONState {
ValueVector values = ValueVector(); ValueVector values;
virtual unique_ptr<JSONState> resolve(EvalState & state) override std::unique_ptr<JSONState> resolve(EvalState & state) override
{ {
Value& v = parent->value(state); Value & v = parent->value(state);
state.mkList(v, values.size()); state.mkList(v, values.size());
for (size_t n = 0; n < values.size(); ++n) { for (size_t n = 0; n < values.size(); ++n) {
v.listElems()[n] = values[n]; v.listElems()[n] = values[n];
} }
return std::move(parent); return std::move(parent);
} }
virtual void add() override { void add() override {
values.push_back(v); values.push_back(*v);
v = nullptr; v = nullptr;
}; }
public: public:
JSONListState(unique_ptr<JSONState>&& p, std::size_t reserve) : JSONState(std::move(p)) JSONListState(std::unique_ptr<JSONState> && p, std::size_t reserve) : JSONState(std::move(p))
{ {
values.reserve(reserve); values.reserve(reserve);
} }
}; };
EvalState & state; EvalState & state;
unique_ptr<JSONState> rs; std::unique_ptr<JSONState> rs;
template<typename T, typename... Args> inline bool handle_value(T f, Args... args) template<typename T, typename... Args> inline bool handle_value(T f, Args... args)
{ {
@ -107,12 +106,12 @@ public:
return handle_value(mkInt, val); return handle_value(mkInt, val);
} }
bool number_float(number_float_t val, const string_t& s) bool number_float(number_float_t val, const string_t & s)
{ {
return handle_value(mkFloat, val); return handle_value(mkFloat, val);
} }
bool string(string_t& val) bool string(string_t & val)
{ {
return handle_value<void(Value&, const char*)>(mkString, val.c_str()); return handle_value<void(Value&, const char*)>(mkString, val.c_str());
} }
@ -123,7 +122,7 @@ public:
return true; return true;
} }
bool key(string_t& name) bool key(string_t & name)
{ {
dynamic_cast<JSONObjectState*>(rs.get())->key(name, state); dynamic_cast<JSONObjectState*>(rs.get())->key(name, state);
return true; return true;

View file

@ -127,14 +127,14 @@ or { return OR_KW; }
try { try {
yylval->n = boost::lexical_cast<int64_t>(yytext); yylval->n = boost::lexical_cast<int64_t>(yytext);
} catch (const boost::bad_lexical_cast &) { } catch (const boost::bad_lexical_cast &) {
throw ParseError(format("invalid integer '%1%'") % yytext); throw ParseError("invalid integer '%1%'", yytext);
} }
return INT; return INT;
} }
{FLOAT} { errno = 0; {FLOAT} { errno = 0;
yylval->nf = strtod(yytext, 0); yylval->nf = strtod(yytext, 0);
if (errno != 0) if (errno != 0)
throw ParseError(format("invalid float '%1%'") % yytext); throw ParseError("invalid float '%1%'", yytext);
return FLOAT; return FLOAT;
} }
@ -219,4 +219,3 @@ or { return OR_KW; }
} }
%% %%

View file

@ -6,7 +6,9 @@ libexpr_DIR := $(d)
libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
libexpr_LIBS = libutil libstore libnixrust libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
libexpr_LIBS = libutil libstore libfetchers
libexpr_LDFLAGS = libexpr_LDFLAGS =
ifneq ($(OS), FreeBSD) ifneq ($(OS), FreeBSD)

View file

@ -267,8 +267,11 @@ void ExprVar::bindVars(const StaticEnv & env)
/* Otherwise, the variable must be obtained from the nearest /* Otherwise, the variable must be obtained from the nearest
enclosing `with'. If there is no `with', then we can issue an enclosing `with'. If there is no `with', then we can issue an
"undefined variable" error now. */ "undefined variable" error now. */
if (withLevel == -1) throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % pos); if (withLevel == -1)
throw UndefinedVarError({
.hint = hintfmt("undefined variable '%1%'", name),
.nixCode = NixCode { .errPos = pos }
});
fromWith = true; fromWith = true;
this->level = withLevel; this->level = withLevel;
} }

View file

@ -2,6 +2,7 @@
#include "value.hh" #include "value.hh"
#include "symbol-table.hh" #include "symbol-table.hh"
#include "error.hh"
#include <map> #include <map>
@ -209,9 +210,10 @@ struct ExprList : Expr
struct Formal struct Formal
{ {
Pos pos;
Symbol name; Symbol name;
Expr * def; Expr * def;
Formal(const Symbol & name, Expr * def) : name(name), def(def) { }; Formal(const Pos & pos, const Symbol & name, Expr * def) : pos(pos), name(name), def(def) { };
}; };
struct Formals struct Formals
@ -234,8 +236,10 @@ struct ExprLambda : Expr
: pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body) : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
{ {
if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
throw ParseError(format("duplicate formal function argument '%1%' at %2%") throw ParseError({
% arg % pos); .hint = hintfmt("duplicate formal function argument '%1%'", arg),
.nixCode = NixCode { .errPos = pos }
});
}; };
void setName(Symbol & name); void setName(Symbol & name);
string showNamePos() const; string showNamePos() const;
@ -261,8 +265,9 @@ struct ExprWith : Expr
struct ExprIf : Expr struct ExprIf : Expr
{ {
Pos pos;
Expr * cond, * then, * else_; Expr * cond, * then, * else_;
ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { }; ExprIf(const Pos & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { };
COMMON_METHODS COMMON_METHODS
}; };

View file

@ -31,7 +31,7 @@ namespace nix {
Expr * result; Expr * result;
Path basePath; Path basePath;
Symbol path; Symbol path;
string error; ErrorInfo error;
Symbol sLetBody; Symbol sLetBody;
ParseData(EvalState & state) ParseData(EvalState & state)
: state(state) : state(state)
@ -64,15 +64,20 @@ namespace nix {
static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos) static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
{ {
throw ParseError(format("attribute '%1%' at %2% already defined at %3%") throw ParseError({
% showAttrPath(attrPath) % pos % prevPos); .hint = hintfmt("attribute '%1%' already defined at %2%",
showAttrPath(attrPath), prevPos),
.nixCode = NixCode { .errPos = pos },
});
} }
static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos) static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
{ {
throw ParseError(format("attribute '%1%' at %2% already defined at %3%") throw ParseError({
% attr % pos % prevPos); .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
.nixCode = NixCode { .errPos = pos },
});
} }
@ -140,8 +145,11 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
static void addFormal(const Pos & pos, Formals * formals, const Formal & formal) static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
{ {
if (!formals->argNames.insert(formal.name).second) if (!formals->argNames.insert(formal.name).second)
throw ParseError(format("duplicate formal function argument '%1%' at %2%") throw ParseError({
% formal.name % pos); .hint = hintfmt("duplicate formal function argument '%1%'",
formal.name),
.nixCode = NixCode { .errPos = pos },
});
formals->formals.push_front(formal); formals->formals.push_front(formal);
} }
@ -249,8 +257,10 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error) void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error)
{ {
data->error = (format("%1%, at %2%") data->error = {
% error % makeCurPos(*loc, data)).str(); .hint = hintfmt(error),
.nixCode = NixCode { .errPos = makeCurPos(*loc, data) }
};
} }
@ -327,15 +337,17 @@ expr_function
{ $$ = new ExprWith(CUR_POS, $2, $4); } { $$ = new ExprWith(CUR_POS, $2, $4); }
| LET binds IN expr_function | LET binds IN expr_function
{ if (!$2->dynamicAttrs.empty()) { if (!$2->dynamicAttrs.empty())
throw ParseError(format("dynamic attributes not allowed in let at %1%") throw ParseError({
% CUR_POS); .hint = hintfmt("dynamic attributes not allowed in let"),
.nixCode = NixCode { .errPos = CUR_POS },
});
$$ = new ExprLet($2, $4); $$ = new ExprLet($2, $4);
} }
| expr_if | expr_if
; ;
expr_if expr_if
: IF expr THEN expr ELSE expr { $$ = new ExprIf($2, $4, $6); } : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); }
| expr_op | expr_op
; ;
@ -405,7 +417,10 @@ expr_simple
| URI { | URI {
static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals"); static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
if (noURLLiterals) if (noURLLiterals)
throw ParseError("URL literals are disabled, at %s", CUR_POS); throw ParseError({
.hint = hintfmt("URL literals are disabled"),
.nixCode = NixCode { .errPos = CUR_POS }
});
$$ = new ExprString(data->symbols.create($1)); $$ = new ExprString(data->symbols.create($1));
} }
| '(' expr ')' { $$ = $2; } | '(' expr ')' { $$ = $2; }
@ -475,8 +490,10 @@ attrs
$$->push_back(AttrName(str->s)); $$->push_back(AttrName(str->s));
delete str; delete str;
} else } else
throw ParseError(format("dynamic attributes not allowed in inherit at %1%") throw ParseError({
% makeCurPos(@2, data)); .hint = hintfmt("dynamic attributes not allowed in inherit"),
.nixCode = NixCode { .errPos = makeCurPos(@2, data) },
});
} }
| { $$ = new AttrPath; } | { $$ = new AttrPath; }
; ;
@ -531,8 +548,8 @@ formals
; ;
formal formal
: ID { $$ = new Formal(data->symbols.create($1), 0); } : ID { $$ = new Formal(CUR_POS, data->symbols.create($1), 0); }
| ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); } | ID '?' expr { $$ = new Formal(CUR_POS, data->symbols.create($1), $3); }
; ;
%% %%
@ -544,7 +561,8 @@ formal
#include <unistd.h> #include <unistd.h>
#include "eval.hh" #include "eval.hh"
#include "download.hh" #include "filetransfer.hh"
#include "fetchers.hh"
#include "store-api.hh" #include "store-api.hh"
@ -670,11 +688,13 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
Path res = r.second + suffix; Path res = r.second + suffix;
if (pathExists(res)) return canonPath(res); if (pathExists(res)) return canonPath(res);
} }
format f = format( throw ThrownError({
"file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)" .hint = hintfmt(evalSettings.pureEval
+ string(pos ? ", at %2%" : "")); ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
throw ThrownError(f % path % pos); path),
.nixCode = NixCode { .errPos = pos }
});
} }
@ -687,11 +707,13 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
if (isUri(elem.second)) { if (isUri(elem.second)) {
try { try {
CachedDownloadRequest request(elem.second); res = { true, store->toRealPath(fetchers::downloadTarball(
request.unpack = true; store, resolveUri(elem.second), "source", false).storePath) };
res = { true, getDownloader()->downloadCached(store, request).path }; } catch (FileTransferError & e) {
} catch (DownloadError & e) { logWarning({
printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); .name = "Entry download",
.hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
});
res = { false, "" }; res = { false, "" };
} }
} else { } else {
@ -699,7 +721,10 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
if (pathExists(path)) if (pathExists(path))
res = { true, path }; res = { true, path };
else { else {
printError(format("warning: Nix search path entry '%1%' does not exist, ignoring") % elem.second); logWarning({
.name = "Entry not found",
.hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
});
res = { false, "" }; res = { false, "" };
} }
} }

File diff suppressed because it is too large Load diff

View file

@ -20,6 +20,7 @@ struct RegisterPrimOp
them. */ them. */
/* Load a ValueInitializer from a DSO and return whatever it initializes */ /* Load a ValueInitializer from a DSO and return whatever it initializes */
void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v); void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v);
/* Execute a program and parse its output */ /* Execute a program and parse its output */
void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v); void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v);

View file

@ -146,7 +146,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
auto sAllOutputs = state.symbols.create("allOutputs"); auto sAllOutputs = state.symbols.create("allOutputs");
for (auto & i : *args[1]->attrs) { for (auto & i : *args[1]->attrs) {
if (!state.store->isStorePath(i.name)) if (!state.store->isStorePath(i.name))
throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos); throw EvalError({
.hint = hintfmt("Context key '%s' is not a store path", i.name),
.nixCode = NixCode { .errPos = *i.pos }
});
if (!settings.readOnlyMode) if (!settings.readOnlyMode)
state.store->ensurePath(state.store->parseStorePath(i.name)); state.store->ensurePath(state.store->parseStorePath(i.name));
state.forceAttrs(*i.value, *i.pos); state.forceAttrs(*i.value, *i.pos);
@ -160,7 +163,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
if (iter != i.value->attrs->end()) { if (iter != i.value->attrs->end()) {
if (state.forceBool(*iter->value, *iter->pos)) { if (state.forceBool(*iter->value, *iter->pos)) {
if (!isDerivation(i.name)) { if (!isDerivation(i.name)) {
throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); throw EvalError({
.hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
.nixCode = NixCode { .errPos = *i.pos }
});
} }
context.insert("=" + string(i.name)); context.insert("=" + string(i.name));
} }
@ -170,7 +176,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
if (iter != i.value->attrs->end()) { if (iter != i.value->attrs->end()) {
state.forceList(*iter->value, *iter->pos); state.forceList(*iter->value, *iter->pos);
if (iter->value->listSize() && !isDerivation(i.name)) { if (iter->value->listSize() && !isDerivation(i.name)) {
throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); throw EvalError({
.hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
.nixCode = NixCode { .errPos = *i.pos }
});
} }
for (unsigned int n = 0; n < iter->value->listSize(); ++n) { for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos); auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);

View file

@ -1,203 +1,19 @@
#include "primops.hh" #include "primops.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "download.hh"
#include "store-api.hh" #include "store-api.hh"
#include "pathlocks.hh"
#include "hash.hh" #include "hash.hh"
#include "tarfile.hh" #include "fetchers.hh"
#include "url.hh"
#include <sys/time.h>
#include <regex>
#include <nlohmann/json.hpp>
using namespace std::string_literals;
namespace nix { namespace nix {
struct GitInfo
{
Path storePath;
std::string rev;
std::string shortRev;
uint64_t revCount = 0;
};
std::regex revRegex("^[0-9a-fA-F]{40}$");
GitInfo exportGit(ref<Store> store, const std::string & uri,
std::optional<std::string> ref, std::string rev,
const std::string & name)
{
if (evalSettings.pureEval && rev == "")
throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) {
bool clean = true;
try {
runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" });
} catch (ExecError & e) {
if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
clean = false;
}
if (!clean) {
/* This is an unclean working tree. So copy all tracked files. */
GitInfo gitInfo;
gitInfo.rev = "0000000000000000000000000000000000000000";
gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
auto files = tokenizeString<std::set<std::string>>(
runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s);
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, uri));
std::string file(p, uri.size() + 1);
auto st = lstat(p);
if (S_ISDIR(st.st_mode)) {
auto prefix = file + "/";
auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
}
return files.count(file);
};
gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
return gitInfo;
}
// clean working tree, but no ref or rev specified. Use 'HEAD'.
rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" }));
ref = "HEAD"s;
}
if (!ref) ref = "HEAD"s;
if (rev != "" && !std::regex_match(rev, revRegex))
throw Error("invalid Git revision '%s'", rev);
deletePath(getCacheDir() + "/nix/git");
Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
if (!pathExists(cacheDir)) {
createDirs(dirOf(cacheDir));
runProgram("git", true, { "init", "--bare", cacheDir });
}
Path localRefFile;
if (ref->compare(0, 5, "refs/") == 0)
localRefFile = cacheDir + "/" + *ref;
else
localRefFile = cacheDir + "/refs/heads/" + *ref;
bool doFetch;
time_t now = time(0);
/* If a rev was specified, we need to fetch if it's not in the
repo. */
if (rev != "") {
try {
runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev });
doFetch = false;
} catch (ExecError & e) {
if (WIFEXITED(e.status)) {
doFetch = true;
} else {
throw;
}
}
} else {
/* If the local ref is older than tarball-ttl seconds, do a
git fetch to update the local ref to the remote ref. */
struct stat st;
doFetch = stat(localRefFile.c_str(), &st) != 0 ||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
}
if (doFetch)
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));
// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
struct timeval times[2];
times[0].tv_sec = now;
times[0].tv_usec = 0;
times[1].tv_sec = now;
times[1].tv_usec = 0;
utimes(localRefFile.c_str(), times);
}
// FIXME: check whether rev is an ancestor of ref.
GitInfo gitInfo;
gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile));
gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri);
std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false);
Path storeLink = cacheDir + "/" + storeLinkName + ".link";
PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken
try {
auto json = nlohmann::json::parse(readFile(storeLink));
assert(json["name"] == name && json["rev"] == gitInfo.rev);
gitInfo.storePath = json["storePath"];
if (store->isValidPath(store->parseStorePath(gitInfo.storePath))) {
gitInfo.revCount = json["revCount"];
return gitInfo;
}
} catch (SysError & e) {
if (e.errNo != ENOENT) throw;
}
auto source = sinkToSource([&](Sink & sink) {
RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
gitOptions.standardOut = &sink;
runProgram2(gitOptions);
});
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
unpackTarfile(*source, tmpDir);
gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));
nlohmann::json json;
json["storePath"] = gitInfo.storePath;
json["uri"] = uri;
json["name"] = name;
json["rev"] = gitInfo.rev;
json["revCount"] = gitInfo.revCount;
writeFile(storeLink, json.dump());
return gitInfo;
}
static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
{ {
std::string url; std::string url;
std::optional<std::string> ref; std::optional<std::string> ref;
std::string rev; std::optional<Hash> rev;
std::string name = "source"; std::string name = "source";
bool fetchSubmodules = false;
PathSet context; PathSet context;
state.forceValue(*args[0]); state.forceValue(*args[0]);
@ -213,15 +29,23 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
else if (n == "ref") else if (n == "ref")
ref = state.forceStringNoCtx(*attr.value, *attr.pos); ref = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "rev") else if (n == "rev")
rev = state.forceStringNoCtx(*attr.value, *attr.pos); rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1);
else if (n == "name") else if (n == "name")
name = state.forceStringNoCtx(*attr.value, *attr.pos); name = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "submodules")
fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
else else
throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos); throw EvalError({
.hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
.nixCode = NixCode { .errPos = *attr.pos }
});
} }
if (url.empty()) if (url.empty())
throw EvalError(format("'url' argument required, at %1%") % pos); throw EvalError({
.hint = hintfmt("'url' argument required"),
.nixCode = NixCode { .errPos = pos }
});
} else } else
url = state.coerceToString(pos, *args[0], context, false, false); url = state.coerceToString(pos, *args[0], context, false, false);
@ -230,17 +54,36 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
// whitelist. Ah well. // whitelist. Ah well.
state.checkURI(url); state.checkURI(url);
auto gitInfo = exportGit(state.store, url, ref, rev, name); if (evalSettings.pureEval && !rev)
throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "git");
attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
if (ref) attrs.insert_or_assign("ref", *ref);
if (rev) attrs.insert_or_assign("rev", rev->gitRev());
if (fetchSubmodules) attrs.insert_or_assign("submodules", true);
auto input = fetchers::inputFromAttrs(attrs);
// FIXME: use name?
auto [tree, input2] = input->fetchTree(state.store);
state.mkAttrs(v, 8); state.mkAttrs(v, 8);
mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); auto storePath = state.store->printStorePath(tree.storePath);
mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); // Backward compatibility: set 'rev' to
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); // 0000000000000000000000000000000000000000 for a dirty tree.
auto rev2 = input2->getRev().value_or(Hash(htSHA1));
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev());
// Backward compatibility: set 'revCount' to 0 for a dirty tree.
mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
tree.info.revCount.value_or(0));
mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules);
v.attrs->sort(); v.attrs->sort();
if (state.allowedPaths) if (state.allowedPaths)
state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); state.allowedPaths->insert(tree.actualPath);
} }
static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);

View file

@ -1,174 +1,18 @@
#include "primops.hh" #include "primops.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "download.hh"
#include "store-api.hh" #include "store-api.hh"
#include "pathlocks.hh" #include "fetchers.hh"
#include "url.hh"
#include <sys/time.h>
#include <regex> #include <regex>
#include <nlohmann/json.hpp>
using namespace std::string_literals;
namespace nix { namespace nix {
struct HgInfo
{
Path storePath;
std::string branch;
std::string rev;
uint64_t revCount = 0;
};
std::regex commitHashRegex("^[0-9a-fA-F]{40}$");
HgInfo exportMercurial(ref<Store> store, const std::string & uri,
std::string rev, const std::string & name)
{
if (evalSettings.pureEval && rev == "")
throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) {
bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == "";
if (!clean) {
/* This is an unclean working tree. So copy all tracked
files. */
printTalkative("copying unclean Mercurial working tree '%s'", uri);
HgInfo hgInfo;
hgInfo.rev = "0000000000000000000000000000000000000000";
hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri }));
auto files = tokenizeString<std::set<std::string>>(
runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, uri));
std::string file(p, uri.size() + 1);
auto st = lstat(p);
if (S_ISDIR(st.st_mode)) {
auto prefix = file + "/";
auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
}
return files.count(file);
};
hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
return hgInfo;
}
}
if (rev == "") rev = "default";
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false));
Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false));
/* If we haven't pulled this repo less than tarball-ttl seconds,
do so now. */
time_t now = time(0);
struct stat st;
if (stat(stampFile.c_str(), &st) != 0 ||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now)
{
/* Except that if this is a commit hash that we already have,
we don't have to pull again. */
if (!(std::regex_match(rev, commitHashRegex)
&& pathExists(cacheDir)
&& runProgram(
RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" })
.killStderr(true)).second == "1"))
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri));
if (pathExists(cacheDir)) {
try {
runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
}
catch (ExecError & e) {
string transJournal = cacheDir + "/.hg/store/journal";
/* hg throws "abandoned transaction" error only if this file exists */
if (pathExists(transJournal)) {
runProgram("hg", true, { "recover", "-R", cacheDir });
runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri });
} else {
throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
}
}
} else {
createDirs(dirOf(cacheDir));
runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir });
}
}
writeFile(stampFile, "");
}
auto tokens = tokenizeString<std::vector<std::string>>(
runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" }));
assert(tokens.size() == 3);
HgInfo hgInfo;
hgInfo.rev = tokens[0];
hgInfo.revCount = std::stoull(tokens[1]);
hgInfo.branch = tokens[2];
std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false);
Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName);
try {
auto json = nlohmann::json::parse(readFile(storeLink));
assert(json["name"] == name && json["rev"] == hgInfo.rev);
hgInfo.storePath = json["storePath"];
if (store->isValidPath(store->parseStorePath(hgInfo.storePath))) {
printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath);
return hgInfo;
}
} catch (SysError & e) {
if (e.errNo != ENOENT) throw;
}
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir });
deletePath(tmpDir + "/.hg_archival.txt");
hgInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
nlohmann::json json;
json["storePath"] = hgInfo.storePath;
json["uri"] = uri;
json["name"] = name;
json["branch"] = hgInfo.branch;
json["rev"] = hgInfo.rev;
json["revCount"] = hgInfo.revCount;
writeFile(storeLink, json.dump());
return hgInfo;
}
static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v)
{ {
std::string url; std::string url;
std::string rev; std::optional<Hash> rev;
std::optional<std::string> ref;
std::string name = "source"; std::string name = "source";
PathSet context; PathSet context;
@ -182,16 +26,29 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
string n(attr.name); string n(attr.name);
if (n == "url") if (n == "url")
url = state.coerceToString(*attr.pos, *attr.value, context, false, false); url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
else if (n == "rev") else if (n == "rev") {
rev = state.forceStringNoCtx(*attr.value, *attr.pos); // Ugly: unlike fetchGit, here the "rev" attribute can
// be both a revision or a branch/tag name.
auto value = state.forceStringNoCtx(*attr.value, *attr.pos);
if (std::regex_match(value, revRegex))
rev = Hash(value, htSHA1);
else
ref = value;
}
else if (n == "name") else if (n == "name")
name = state.forceStringNoCtx(*attr.value, *attr.pos); name = state.forceStringNoCtx(*attr.value, *attr.pos);
else else
throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos); throw EvalError({
.hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
.nixCode = NixCode { .errPos = *attr.pos }
});
} }
if (url.empty()) if (url.empty())
throw EvalError(format("'url' argument required, at %1%") % pos); throw EvalError({
.hint = hintfmt("'url' argument required"),
.nixCode = NixCode { .errPos = pos }
});
} else } else
url = state.coerceToString(pos, *args[0], context, false, false); url = state.coerceToString(pos, *args[0], context, false, false);
@ -200,18 +57,35 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
// whitelist. Ah well. // whitelist. Ah well.
state.checkURI(url); state.checkURI(url);
auto hgInfo = exportMercurial(state.store, url, rev, name); if (evalSettings.pureEval && !rev)
throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision");
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "hg");
attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
if (ref) attrs.insert_or_assign("ref", *ref);
if (rev) attrs.insert_or_assign("rev", rev->gitRev());
auto input = fetchers::inputFromAttrs(attrs);
// FIXME: use name
auto [tree, input2] = input->fetchTree(state.store);
state.mkAttrs(v, 8); state.mkAttrs(v, 8);
mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath})); auto storePath = state.store->printStorePath(tree.storePath);
mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev); if (input2->getRef())
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12)); mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef());
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount); // Backward compatibility: set 'rev' to
// 0000000000000000000000000000000000000000 for a dirty tree.
auto rev2 = input2->getRev().value_or(Hash(htSHA1));
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12));
if (tree.info.revCount)
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
v.attrs->sort(); v.attrs->sort();
if (state.allowedPaths) if (state.allowedPaths)
state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath)); state.allowedPaths->insert(tree.actualPath);
} }
static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial);

View file

@ -0,0 +1,172 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "store-api.hh"
#include "fetchers.hh"
#include "filetransfer.hh"
#include <ctime>
#include <iomanip>
namespace nix {
void emitTreeAttrs(
EvalState & state,
const fetchers::Tree & tree,
std::shared_ptr<const fetchers::Input> input,
Value & v)
{
state.mkAttrs(v, 8);
auto storePath = state.store->printStorePath(tree.storePath);
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
assert(tree.info.narHash);
mkString(*state.allocAttr(v, state.symbols.create("narHash")),
tree.info.narHash.to_string(SRI, true));
if (input->getRev()) {
mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev());
}
if (tree.info.revCount)
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
if (tree.info.lastModified)
mkString(*state.allocAttr(v, state.symbols.create("lastModified")),
fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S")));
v.attrs->sort();
}
static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
settings.requireExperimentalFeature("flakes");
std::shared_ptr<const fetchers::Input> input;
PathSet context;
state.forceValue(*args[0]);
if (args[0]->type == tAttrs) {
state.forceAttrs(*args[0], pos);
fetchers::Attrs attrs;
for (auto & attr : *args[0]->attrs) {
state.forceValue(*attr.value);
if (attr.value->type == tString)
attrs.emplace(attr.name, attr.value->string.s);
else if (attr.value->type == tBool)
attrs.emplace(attr.name, attr.value->boolean);
else
throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected",
attr.name, showType(*attr.value));
}
if (!attrs.count("type"))
throw Error({
.hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
.nixCode = NixCode { .errPos = pos }
});
input = fetchers::inputFromAttrs(attrs);
} else
input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false));
if (evalSettings.pureEval && !input->isImmutable())
throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input");
// FIXME: use fetchOrSubstituteTree
auto [tree, input2] = input->fetchTree(state.store);
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
emitTreeAttrs(state, tree, input2, v);
}
static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
const string & who, bool unpack, std::string name)
{
std::optional<std::string> url;
std::optional<Hash> expectedHash;
state.forceValue(*args[0]);
if (args[0]->type == tAttrs) {
state.forceAttrs(*args[0], pos);
for (auto & attr : *args[0]->attrs) {
string n(attr.name);
if (n == "url")
url = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "sha256")
expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
else if (n == "name")
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else
throw EvalError({
.hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
.nixCode = NixCode { .errPos = *attr.pos }
});
}
if (!url)
throw EvalError({
.hint = hintfmt("'url' argument required"),
.nixCode = NixCode { .errPos = pos }
});
} else
url = state.forceStringNoCtx(*args[0], pos);
url = resolveUri(*url);
state.checkURI(*url);
if (name == "")
name = baseNameOf(*url);
if (evalSettings.pureEval && !expectedHash)
throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
auto storePath =
unpack
? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
: fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
auto path = state.store->toRealPath(storePath);
if (expectedHash) {
auto hash = unpack
? state.store->queryPathInfo(storePath)->narHash
: hashFile(htSHA256, path);
if (hash != *expectedHash)
throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
*url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
}
if (state.allowedPaths)
state.allowedPaths->insert(path);
mkString(v, path, PathSet({path}));
}
static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
fetch(state, pos, args, v, "fetchurl", false, "");
}
static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
fetch(state, pos, args, v, "fetchTarball", true, "source");
}
static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
}

View file

@ -1,7 +1,7 @@
#include "primops.hh" #include "primops.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "cpptoml/cpptoml.h" #include "../../cpptoml/cpptoml.h"
namespace nix { namespace nix {
@ -81,7 +81,10 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
try { try {
visit(v, parser(tomlStream).parse()); visit(v, parser(tomlStream).parse());
} catch (std::runtime_error & e) { } catch (std::runtime_error & e) {
throw EvalError("while parsing a TOML string at %s: %s", pos, e.what()); throw EvalError({
.hint = hintfmt("while parsing a TOML string: %s", e.what()),
.nixCode = NixCode { .errPos = pos }
});
} }
} }

View file

@ -79,7 +79,7 @@ void printValueAsJSON(EvalState & state, bool strict,
break; break;
default: default:
throw TypeError(format("cannot convert %1% to JSON") % showType(v)); throw TypeError("cannot convert %1% to JSON", showType(v));
} }
} }
@ -93,7 +93,7 @@ void printValueAsJSON(EvalState & state, bool strict,
void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,
JSONPlaceholder & out, PathSet & context) const JSONPlaceholder & out, PathSet & context) const
{ {
throw TypeError(format("cannot convert %1% to JSON") % showType()); throw TypeError("cannot convert %1% to JSON", showType());
} }

View file

@ -253,12 +253,17 @@ void mkPath(Value & v, const char * s);
#if HAVE_BOEHMGC #if HAVE_BOEHMGC
typedef std::vector<Value *, gc_allocator<Value *> > ValueVector; typedef std::vector<Value *, traceable_allocator<Value *> > ValueVector;
typedef std::map<Symbol, Value *, std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> > > ValueMap; typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *> > > ValueMap;
#else #else
typedef std::vector<Value *> ValueVector; typedef std::vector<Value *> ValueVector;
typedef std::map<Symbol, Value *> ValueMap; typedef std::map<Symbol, Value *> ValueMap;
#endif #endif
/* A value allocated in traceable memory. */
typedef std::shared_ptr<Value *> RootValue;
RootValue allocRootValue(Value * v);
} }

107
src/libfetchers/attrs.cc Normal file
View file

@ -0,0 +1,107 @@
#include "attrs.hh"
#include "fetchers.hh"
#include <nlohmann/json.hpp>
namespace nix::fetchers {
Attrs jsonToAttrs(const nlohmann::json & json)
{
Attrs attrs;
for (auto & i : json.items()) {
if (i.value().is_number())
attrs.emplace(i.key(), i.value().get<int64_t>());
else if (i.value().is_string())
attrs.emplace(i.key(), i.value().get<std::string>());
else if (i.value().is_boolean())
attrs.emplace(i.key(), i.value().get<bool>());
else
throw Error("unsupported input attribute type in lock file");
}
return attrs;
}
nlohmann::json attrsToJson(const Attrs & attrs)
{
nlohmann::json json;
for (auto & attr : attrs) {
if (auto v = std::get_if<int64_t>(&attr.second)) {
json[attr.first] = *v;
} else if (auto v = std::get_if<std::string>(&attr.second)) {
json[attr.first] = *v;
} else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) {
json[attr.first] = v->t;
} else abort();
}
return json;
}
std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name)
{
auto i = attrs.find(name);
if (i == attrs.end()) return {};
if (auto v = std::get_if<std::string>(&i->second))
return *v;
throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump());
}
std::string getStrAttr(const Attrs & attrs, const std::string & name)
{
auto s = maybeGetStrAttr(attrs, name);
if (!s)
throw Error("input attribute '%s' is missing", name);
return *s;
}
std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
{
auto i = attrs.find(name);
if (i == attrs.end()) return {};
if (auto v = std::get_if<int64_t>(&i->second))
return *v;
throw Error("input attribute '%s' is not an integer", name);
}
int64_t getIntAttr(const Attrs & attrs, const std::string & name)
{
auto s = maybeGetIntAttr(attrs, name);
if (!s)
throw Error("input attribute '%s' is missing", name);
return *s;
}
std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name)
{
auto i = attrs.find(name);
if (i == attrs.end()) return {};
if (auto v = std::get_if<int64_t>(&i->second))
return *v;
throw Error("input attribute '%s' is not a Boolean", name);
}
bool getBoolAttr(const Attrs & attrs, const std::string & name)
{
auto s = maybeGetBoolAttr(attrs, name);
if (!s)
throw Error("input attribute '%s' is missing", name);
return *s;
}
std::map<std::string, std::string> attrsToQuery(const Attrs & attrs)
{
std::map<std::string, std::string> query;
for (auto & attr : attrs) {
if (auto v = std::get_if<int64_t>(&attr.second)) {
query.insert_or_assign(attr.first, fmt("%d", *v));
} else if (auto v = std::get_if<std::string>(&attr.second)) {
query.insert_or_assign(attr.first, *v);
} else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) {
query.insert_or_assign(attr.first, v->t ? "1" : "0");
} else abort();
}
return query;
}
}

39
src/libfetchers/attrs.hh Normal file
View file

@ -0,0 +1,39 @@
#pragma once
#include "types.hh"
#include <variant>
#include <nlohmann/json_fwd.hpp>
namespace nix::fetchers {
/* Wrap bools to prevent string literals (i.e. 'char *') from being
cast to a bool in Attr. */
template<typename T>
struct Explicit {
T t;
};
typedef std::variant<std::string, int64_t, Explicit<bool>> Attr;
typedef std::map<std::string, Attr> Attrs;
Attrs jsonToAttrs(const nlohmann::json & json);
nlohmann::json attrsToJson(const Attrs & attrs);
std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name);
std::string getStrAttr(const Attrs & attrs, const std::string & name);
std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
int64_t getIntAttr(const Attrs & attrs, const std::string & name);
std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name);
bool getBoolAttr(const Attrs & attrs, const std::string & name);
std::map<std::string, std::string> attrsToQuery(const Attrs & attrs);
}

121
src/libfetchers/cache.cc Normal file
View file

@ -0,0 +1,121 @@
#include "cache.hh"
#include "sqlite.hh"
#include "sync.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix::fetchers {
static const char * schema = R"sql(
create table if not exists Cache (
input text not null,
info text not null,
path text not null,
immutable integer not null,
timestamp integer not null,
primary key (input)
);
)sql";
struct CacheImpl : Cache
{
struct State
{
SQLite db;
SQLiteStmt add, lookup;
};
Sync<State> _state;
CacheImpl()
{
auto state(_state.lock());
auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite";
createDirs(dirOf(dbPath));
state->db = SQLite(dbPath);
state->db.isCache();
state->db.exec(schema);
state->add.create(state->db,
"insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)");
state->lookup.create(state->db,
"select info, path, immutable, timestamp from Cache where input = ?");
}
void add(
ref<Store> store,
const Attrs & inAttrs,
const Attrs & infoAttrs,
const StorePath & storePath,
bool immutable) override
{
_state.lock()->add.use()
(attrsToJson(inAttrs).dump())
(attrsToJson(infoAttrs).dump())
(store->printStorePath(storePath))
(immutable)
(time(0)).exec();
}
std::optional<std::pair<Attrs, StorePath>> lookup(
ref<Store> store,
const Attrs & inAttrs) override
{
if (auto res = lookupExpired(store, inAttrs)) {
if (!res->expired)
return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath));
debug("ignoring expired cache entry '%s'",
attrsToJson(inAttrs).dump());
}
return {};
}
std::optional<Result> lookupExpired(
ref<Store> store,
const Attrs & inAttrs) override
{
auto state(_state.lock());
auto inAttrsJson = attrsToJson(inAttrs).dump();
auto stmt(state->lookup.use()(inAttrsJson));
if (!stmt.next()) {
debug("did not find cache entry for '%s'", inAttrsJson);
return {};
}
auto infoJson = stmt.getStr(0);
auto storePath = store->parseStorePath(stmt.getStr(1));
auto immutable = stmt.getInt(2) != 0;
auto timestamp = stmt.getInt(3);
store->addTempRoot(storePath);
if (!store->isValidPath(storePath)) {
// FIXME: we could try to substitute 'storePath'.
debug("ignoring disappeared cache entry '%s'", inAttrsJson);
return {};
}
debug("using cache entry '%s' -> '%s', '%s'",
inAttrsJson, infoJson, store->printStorePath(storePath));
return Result {
.expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
.infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)),
.storePath = std::move(storePath)
};
}
};
ref<Cache> getCache()
{
static auto cache = std::make_shared<CacheImpl>();
return ref<Cache>(cache);
}
}

34
src/libfetchers/cache.hh Normal file
View file

@ -0,0 +1,34 @@
#pragma once
#include "fetchers.hh"
namespace nix::fetchers {
struct Cache
{
virtual void add(
ref<Store> store,
const Attrs & inAttrs,
const Attrs & infoAttrs,
const StorePath & storePath,
bool immutable) = 0;
virtual std::optional<std::pair<Attrs, StorePath>> lookup(
ref<Store> store,
const Attrs & inAttrs) = 0;
struct Result
{
bool expired = false;
Attrs infoAttrs;
StorePath storePath;
};
virtual std::optional<Result> lookupExpired(
ref<Store> store,
const Attrs & inAttrs) = 0;
};
ref<Cache> getCache();
}

View file

@ -0,0 +1,75 @@
#include "fetchers.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix::fetchers {
std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr;
void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme)
{
if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>();
inputSchemes->push_back(std::move(inputScheme));
}
std::unique_ptr<Input> inputFromURL(const ParsedURL & url)
{
for (auto & inputScheme : *inputSchemes) {
auto res = inputScheme->inputFromURL(url);
if (res) return res;
}
throw Error("input '%s' is unsupported", url.url);
}
std::unique_ptr<Input> inputFromURL(const std::string & url)
{
return inputFromURL(parseURL(url));
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
{
auto attrs2(attrs);
attrs2.erase("narHash");
for (auto & inputScheme : *inputSchemes) {
auto res = inputScheme->inputFromAttrs(attrs2);
if (res) {
if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
// FIXME: require SRI hash.
res->narHash = newHashAllowEmpty(*narHash, htUnknown);
return res;
}
}
throw Error("input '%s' is unsupported", attrsToJson(attrs));
}
Attrs Input::toAttrs() const
{
auto attrs = toAttrsInternal();
if (narHash)
attrs.emplace("narHash", narHash->to_string(SRI, true));
attrs.emplace("type", type());
return attrs;
}
std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const
{
auto [tree, input] = fetchTreeInternal(store);
if (tree.actualPath == "")
tree.actualPath = store->toRealPath(tree.storePath);
if (!tree.info.narHash)
tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash;
if (input->narHash)
assert(input->narHash == tree.info.narHash);
if (narHash && narHash != input->narHash)
throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true));
return {std::move(tree), input};
}
}

103
src/libfetchers/fetchers.hh Normal file
View file

@ -0,0 +1,103 @@
#pragma once
#include "types.hh"
#include "hash.hh"
#include "path.hh"
#include "tree-info.hh"
#include "attrs.hh"
#include "url.hh"
#include <memory>
namespace nix { class Store; }
namespace nix::fetchers {
struct Input;
struct Tree
{
Path actualPath;
StorePath storePath;
TreeInfo info;
};
struct Input : std::enable_shared_from_this<Input>
{
std::optional<Hash> narHash; // FIXME: implement
virtual std::string type() const = 0;
virtual ~Input() { }
virtual bool operator ==(const Input & other) const { return false; }
/* Check whether this is a "direct" input, that is, not
one that goes through a registry. */
virtual bool isDirect() const { return true; }
/* Check whether this is an "immutable" input, that is,
one that contains a commit hash or content hash. */
virtual bool isImmutable() const { return (bool) narHash; }
virtual bool contains(const Input & other) const { return false; }
virtual std::optional<std::string> getRef() const { return {}; }
virtual std::optional<Hash> getRev() const { return {}; }
virtual ParsedURL toURL() const = 0;
std::string to_string() const
{
return toURL().to_string();
}
Attrs toAttrs() const;
std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const;
private:
virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0;
virtual Attrs toAttrsInternal() const = 0;
};
struct InputScheme
{
virtual ~InputScheme() { }
virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0;
virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0;
};
std::unique_ptr<Input> inputFromURL(const ParsedURL & url);
std::unique_ptr<Input> inputFromURL(const std::string & url);
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs);
void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
struct DownloadFileResult
{
StorePath storePath;
std::string etag;
std::string effectiveUrl;
};
DownloadFileResult downloadFile(
ref<Store> store,
const std::string & url,
const std::string & name,
bool immutable);
Tree downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
bool immutable);
}

441
src/libfetchers/git.cc Normal file
View file

@ -0,0 +1,441 @@
#include "fetchers.hh"
#include "cache.hh"
#include "globals.hh"
#include "tarfile.hh"
#include "store-api.hh"
#include <sys/time.h>
using namespace std::string_literals;
namespace nix::fetchers {
static std::string readHead(const Path & path)
{
return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" }));
}
static bool isNotDotGitDirectory(const Path & path)
{
static const std::regex gitDirRegex("^(?:.*/)?\\.git$");
return not std::regex_match(path, gitDirRegex);
}
struct GitInput : Input
{
ParsedURL url;
std::optional<std::string> ref;
std::optional<Hash> rev;
bool shallow = false;
bool submodules = false;
GitInput(const ParsedURL & url) : url(url)
{ }
std::string type() const override { return "git"; }
bool operator ==(const Input & other) const override
{
auto other2 = dynamic_cast<const GitInput *>(&other);
return
other2
&& url == other2->url
&& rev == other2->rev
&& ref == other2->ref;
}
bool isImmutable() const override
{
return (bool) rev || narHash;
}
std::optional<std::string> getRef() const override { return ref; }
std::optional<Hash> getRev() const override { return rev; }
ParsedURL toURL() const override
{
ParsedURL url2(url);
if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme;
if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
if (ref) url2.query.insert_or_assign("ref", *ref);
if (shallow) url2.query.insert_or_assign("shallow", "1");
return url2;
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("url", url.to_string());
if (ref)
attrs.emplace("ref", *ref);
if (rev)
attrs.emplace("rev", rev->gitRev());
if (shallow)
attrs.emplace("shallow", true);
if (submodules)
attrs.emplace("submodules", true);
return attrs;
}
std::pair<bool, std::string> getActualUrl() const
{
// Don't clone file:// URIs (but otherwise treat them the
// same as remote URIs, i.e. don't use the working tree or
// HEAD).
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
bool isLocal = url.scheme == "file" && !forceHttp;
return {isLocal, isLocal ? url.path : url.base};
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
{
auto name = "source";
auto input = std::make_shared<GitInput>(*this);
assert(!rev || rev->type == htSHA1);
std::string cacheType = "git";
if (shallow) cacheType += "-shallow";
if (submodules) cacheType += "-submodules";
auto getImmutableAttrs = [&]()
{
return Attrs({
{"type", cacheType},
{"name", name},
{"rev", input->rev->gitRev()},
});
};
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-> std::pair<Tree, std::shared_ptr<const Input>>
{
assert(input->rev);
assert(!rev || rev == input->rev);
return {
Tree {
.actualPath = store->toRealPath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
.revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")),
.lastModified = getIntAttr(infoAttrs, "lastModified"),
},
},
input
};
};
if (rev) {
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
}
auto [isLocal, actualUrl_] = getActualUrl();
auto actualUrl = actualUrl_; // work around clang bug
// If this is a local directory and no ref or revision is
// given, then allow the use of an unclean working tree.
if (!input->ref && !input->rev && isLocal) {
bool clean = false;
/* Check whether this repo has any commits. There are
probably better ways to do this. */
auto gitDir = actualUrl + "/.git";
auto commonGitDir = chomp(runProgram(
"git",
true,
{ "-C", actualUrl, "rev-parse", "--git-common-dir" }
));
if (commonGitDir != ".git")
gitDir = commonGitDir;
bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty();
try {
if (haveCommits) {
runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" });
clean = true;
}
} catch (ExecError & e) {
if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
}
if (!clean) {
/* This is an unclean working tree. So copy all tracked files. */
if (!settings.allowDirty)
throw Error("Git tree '%s' is dirty", actualUrl);
if (settings.warnDirty)
warn("Git tree '%s' is dirty", actualUrl);
auto gitOpts = Strings({ "-C", actualUrl, "ls-files", "-z" });
if (submodules)
gitOpts.emplace_back("--recurse-submodules");
auto files = tokenizeString<std::set<std::string>>(
runProgram("git", true, gitOpts), "\0"s);
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, actualUrl));
std::string file(p, actualUrl.size() + 1);
auto st = lstat(p);
if (S_ISDIR(st.st_mode)) {
auto prefix = file + "/";
auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
}
return files.count(file);
};
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
auto tree = Tree {
.actualPath = store->printStorePath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
// FIXME: maybe we should use the timestamp of the last
// modified dirty file?
.lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0,
}
};
return {std::move(tree), input};
}
}
if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master";
Attrs mutableAttrs({
{"type", cacheType},
{"name", name},
{"url", actualUrl},
{"ref", *input->ref},
});
Path repoDir;
if (isLocal) {
if (!input->rev)
input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1);
repoDir = actualUrl;
} else {
if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
if (!rev || rev == rev2) {
input->rev = rev2;
return makeResult(res->first, std::move(res->second));
}
}
Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
repoDir = cacheDir;
if (!pathExists(cacheDir)) {
createDirs(dirOf(cacheDir));
runProgram("git", true, { "init", "--bare", repoDir });
}
Path localRefFile =
input->ref->compare(0, 5, "refs/") == 0
? cacheDir + "/" + *input->ref
: cacheDir + "/refs/heads/" + *input->ref;
bool doFetch;
time_t now = time(0);
/* If a rev was specified, we need to fetch if it's not in the
repo. */
if (input->rev) {
try {
runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() });
doFetch = false;
} catch (ExecError & e) {
if (WIFEXITED(e.status)) {
doFetch = true;
} else {
throw;
}
}
} else {
/* If the local ref is older than tarball-ttl seconds, do a
git fetch to update the local ref to the remote ref. */
struct stat st;
doFetch = stat(localRefFile.c_str(), &st) != 0 ||
(uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
}
if (doFetch) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl));
// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
try {
auto fetchRef = input->ref->compare(0, 5, "refs/") == 0
? *input->ref
: "refs/heads/" + *input->ref;
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
} catch (Error & e) {
if (!pathExists(localRefFile)) throw;
warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
}
struct timeval times[2];
times[0].tv_sec = now;
times[0].tv_usec = 0;
times[1].tv_sec = now;
times[1].tv_usec = 0;
utimes(localRefFile.c_str(), times);
}
if (!input->rev)
input->rev = Hash(chomp(readFile(localRefFile)), htSHA1);
}
bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
if (isShallow && !shallow)
throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl);
// FIXME: check whether rev is an ancestor of ref.
printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl);
/* Now that we know the ref, check again whether we have it in
the store. */
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
PathFilter filter = defaultPathFilter;
if (submodules) {
Path tmpGitDir = createTempDir();
AutoDelete delTmpGitDir(tmpGitDir, true);
runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
// TODO: repoDir might lack the ref (it only checks if rev
// exists, see FIXME above) so use a big hammer and fetch
// everything to ensure we get the rev.
runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
"--update-head-ok", "--", repoDir, "refs/*:refs/*" });
runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() });
runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl });
runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
filter = isNotDotGitDirectory;
} else {
// FIXME: should pipe this, or find some better way to extract a
// revision.
auto source = sinkToSource([&](Sink & sink) {
RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() });
gitOptions.standardOut = &sink;
runProgram2(gitOptions);
});
unpackTarfile(*source, tmpDir);
}
auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
Attrs infoAttrs({
{"rev", input->rev->gitRev()},
{"lastModified", lastModified},
});
if (!shallow)
infoAttrs.insert_or_assign("revCount",
std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })));
if (!this->rev)
getCache()->add(
store,
mutableAttrs,
infoAttrs,
storePath,
false);
getCache()->add(
store,
getImmutableAttrs(),
infoAttrs,
storePath,
true);
return makeResult(infoAttrs, std::move(storePath));
}
};
struct GitInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "git" &&
url.scheme != "git+http" &&
url.scheme != "git+https" &&
url.scheme != "git+ssh" &&
url.scheme != "git+file") return nullptr;
auto url2(url);
if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
url2.query.clear();
Attrs attrs;
attrs.emplace("type", "git");
for (auto &[name, value] : url.query) {
if (name == "rev" || name == "ref")
attrs.emplace(name, value);
else
url2.query.emplace(name, value);
}
attrs.emplace("url", url2.to_string());
return inputFromAttrs(attrs);
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "git") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules")
throw Error("unsupported Git input attribute '%s'", name);
auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (std::regex_search(*ref, badGitRefRegex))
throw BadURL("invalid Git branch/tag name '%s'", *ref);
input->ref = *ref;
}
if (auto rev = maybeGetStrAttr(attrs, "rev"))
input->rev = Hash(*rev, htSHA1);
input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false);
input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
}

195
src/libfetchers/github.cc Normal file
View file

@ -0,0 +1,195 @@
#include "filetransfer.hh"
#include "cache.hh"
#include "fetchers.hh"
#include "globals.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix::fetchers {
std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
struct GitHubInput : Input
{
std::string owner;
std::string repo;
std::optional<std::string> ref;
std::optional<Hash> rev;
std::string type() const override { return "github"; }
bool operator ==(const Input & other) const override
{
auto other2 = dynamic_cast<const GitHubInput *>(&other);
return
other2
&& owner == other2->owner
&& repo == other2->repo
&& rev == other2->rev
&& ref == other2->ref;
}
bool isImmutable() const override
{
return (bool) rev || narHash;
}
std::optional<std::string> getRef() const override { return ref; }
std::optional<Hash> getRev() const override { return rev; }
ParsedURL toURL() const override
{
auto path = owner + "/" + repo;
assert(!(ref && rev));
if (ref) path += "/" + *ref;
if (rev) path += "/" + rev->to_string(Base16, false);
return ParsedURL {
.scheme = "github",
.path = path,
};
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("owner", owner);
attrs.emplace("repo", repo);
if (ref)
attrs.emplace("ref", *ref);
if (rev)
attrs.emplace("rev", rev->gitRev());
return attrs;
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
{
auto rev = this->rev;
auto ref = this->ref.value_or("master");
if (!rev) {
auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
owner, repo, ref);
auto json = nlohmann::json::parse(
readFile(
store->toRealPath(
downloadFile(store, url, "source", false).storePath)));
rev = Hash(std::string { json["sha"] }, htSHA1);
debug("HEAD revision for '%s' is %s", url, rev->gitRev());
}
auto input = std::make_shared<GitHubInput>(*this);
input->ref = {};
input->rev = *rev;
Attrs immutableAttrs({
{"type", "git-tarball"},
{"rev", rev->gitRev()},
});
if (auto res = getCache()->lookup(store, immutableAttrs)) {
return {
Tree{
.actualPath = store->toRealPath(res->second),
.storePath = std::move(res->second),
.info = TreeInfo {
.lastModified = getIntAttr(res->first, "lastModified"),
},
},
input
};
}
// FIXME: use regular /archive URLs instead? api.github.com
// might have stricter rate limits.
auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
owner, repo, rev->to_string(Base16, false));
std::string accessToken = settings.githubAccessToken.get();
if (accessToken != "")
url += "?access_token=" + accessToken;
auto tree = downloadTarball(store, url, "source", true);
getCache()->add(
store,
immutableAttrs,
{
{"rev", rev->gitRev()},
{"lastModified", *tree.info.lastModified}
},
tree.storePath,
true);
return {std::move(tree), input};
}
};
struct GitHubInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "github") return nullptr;
auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
auto input = std::make_unique<GitHubInput>();
if (path.size() == 2) {
} else if (path.size() == 3) {
if (std::regex_match(path[2], revRegex))
input->rev = Hash(path[2], htSHA1);
else if (std::regex_match(path[2], refRegex))
input->ref = path[2];
else
throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
} else
throw BadURL("GitHub URL '%s' is invalid", url.url);
for (auto &[name, value] : url.query) {
if (name == "rev") {
if (input->rev)
throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url);
input->rev = Hash(value, htSHA1);
}
else if (name == "ref") {
if (!std::regex_match(value, refRegex))
throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url);
if (input->ref)
throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url);
input->ref = value;
}
}
if (input->ref && input->rev)
throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url);
input->owner = path[0];
input->repo = path[1];
return input;
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "github") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev")
throw Error("unsupported GitHub input attribute '%s'", name);
auto input = std::make_unique<GitHubInput>();
input->owner = getStrAttr(attrs, "owner");
input->repo = getStrAttr(attrs, "repo");
input->ref = maybeGetStrAttr(attrs, "ref");
if (auto rev = maybeGetStrAttr(attrs, "rev"))
input->rev = Hash(*rev, htSHA1);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
}

11
src/libfetchers/local.mk Normal file
View file

@ -0,0 +1,11 @@
libraries += libfetchers
libfetchers_NAME = libnixfetchers
libfetchers_DIR := $(d)
libfetchers_SOURCES := $(wildcard $(d)/*.cc)
libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
libfetchers_LIBS = libutil libstore

View file

@ -0,0 +1,303 @@
#include "fetchers.hh"
#include "cache.hh"
#include "globals.hh"
#include "tarfile.hh"
#include "store-api.hh"
#include <sys/time.h>
using namespace std::string_literals;
namespace nix::fetchers {
struct MercurialInput : Input
{
ParsedURL url;
std::optional<std::string> ref;
std::optional<Hash> rev;
MercurialInput(const ParsedURL & url) : url(url)
{ }
std::string type() const override { return "hg"; }
bool operator ==(const Input & other) const override
{
auto other2 = dynamic_cast<const MercurialInput *>(&other);
return
other2
&& url == other2->url
&& rev == other2->rev
&& ref == other2->ref;
}
bool isImmutable() const override
{
return (bool) rev || narHash;
}
std::optional<std::string> getRef() const override { return ref; }
std::optional<Hash> getRev() const override { return rev; }
ParsedURL toURL() const override
{
ParsedURL url2(url);
url2.scheme = "hg+" + url2.scheme;
if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
if (ref) url2.query.insert_or_assign("ref", *ref);
return url;
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("url", url.to_string());
if (ref)
attrs.emplace("ref", *ref);
if (rev)
attrs.emplace("rev", rev->gitRev());
return attrs;
}
std::pair<bool, std::string> getActualUrl() const
{
bool isLocal = url.scheme == "file";
return {isLocal, isLocal ? url.path : url.base};
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
{
auto name = "source";
auto input = std::make_shared<MercurialInput>(*this);
auto [isLocal, actualUrl_] = getActualUrl();
auto actualUrl = actualUrl_; // work around clang bug
// FIXME: return lastModified.
// FIXME: don't clone local repositories.
if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) {
bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
if (!clean) {
/* This is an unclean working tree. So copy all tracked
files. */
if (!settings.allowDirty)
throw Error("Mercurial tree '%s' is unclean", actualUrl);
if (settings.warnDirty)
warn("Mercurial tree '%s' is unclean", actualUrl);
input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl }));
auto files = tokenizeString<std::set<std::string>>(
runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, actualUrl));
std::string file(p, actualUrl.size() + 1);
auto st = lstat(p);
if (S_ISDIR(st.st_mode)) {
auto prefix = file + "/";
auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
}
return files.count(file);
};
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
return {Tree {
.actualPath = store->printStorePath(storePath),
.storePath = std::move(storePath),
}, input};
}
}
if (!input->ref) input->ref = "default";
auto getImmutableAttrs = [&]()
{
return Attrs({
{"type", "hg"},
{"name", name},
{"rev", input->rev->gitRev()},
});
};
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-> std::pair<Tree, std::shared_ptr<const Input>>
{
assert(input->rev);
assert(!rev || rev == input->rev);
return {
Tree{
.actualPath = store->toRealPath(storePath),
.storePath = std::move(storePath),
.info = TreeInfo {
.revCount = getIntAttr(infoAttrs, "revCount"),
},
},
input
};
};
if (input->rev) {
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
}
assert(input->rev || input->ref);
auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref;
Attrs mutableAttrs({
{"type", "hg"},
{"name", name},
{"url", actualUrl},
{"ref", *input->ref},
});
if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
if (!rev || rev == rev2) {
input->rev = rev2;
return makeResult(res->first, std::move(res->second));
}
}
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false));
/* If this is a commit hash that we already have, we don't
have to pull again. */
if (!(input->rev
&& pathExists(cacheDir)
&& runProgram(
RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" })
.killStderr(true)).second == "1"))
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
if (pathExists(cacheDir)) {
try {
runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
}
catch (ExecError & e) {
string transJournal = cacheDir + "/.hg/store/journal";
/* hg throws "abandoned transaction" error only if this file exists */
if (pathExists(transJournal)) {
runProgram("hg", true, { "recover", "-R", cacheDir });
runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
} else {
throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
}
}
} else {
createDirs(dirOf(cacheDir));
runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir });
}
}
auto tokens = tokenizeString<std::vector<std::string>>(
runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
assert(tokens.size() == 3);
input->rev = Hash(tokens[0], htSHA1);
auto revCount = std::stoull(tokens[1]);
input->ref = tokens[2];
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir });
deletePath(tmpDir + "/.hg_archival.txt");
auto storePath = store->addToStore(name, tmpDir);
Attrs infoAttrs({
{"rev", input->rev->gitRev()},
{"revCount", (int64_t) revCount},
});
if (!this->rev)
getCache()->add(
store,
mutableAttrs,
infoAttrs,
storePath,
false);
getCache()->add(
store,
getImmutableAttrs(),
infoAttrs,
storePath,
true);
return makeResult(infoAttrs, std::move(storePath));
}
};
struct MercurialInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "hg+http" &&
url.scheme != "hg+https" &&
url.scheme != "hg+ssh" &&
url.scheme != "hg+file") return nullptr;
auto url2(url);
url2.scheme = std::string(url2.scheme, 3);
url2.query.clear();
Attrs attrs;
attrs.emplace("type", "hg");
for (auto &[name, value] : url.query) {
if (name == "rev" || name == "ref")
attrs.emplace(name, value);
else
url2.query.emplace(name, value);
}
attrs.emplace("url", url2.to_string());
return inputFromAttrs(attrs);
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "hg") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "ref" && name != "rev")
throw Error("unsupported Mercurial input attribute '%s'", name);
auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url")));
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (!std::regex_match(*ref, refRegex))
throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
input->ref = *ref;
}
if (auto rev = maybeGetStrAttr(attrs, "rev"))
input->rev = Hash(*rev, htSHA1);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
}

148
src/libfetchers/path.cc Normal file
View file

@ -0,0 +1,148 @@
#include "fetchers.hh"
#include "store-api.hh"
namespace nix::fetchers {
struct PathInput : Input
{
Path path;
/* Allow the user to pass in "fake" tree info attributes. This is
useful for making a pinned tree work the same as the repository
from which is exported
(e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
std::optional<Hash> rev;
std::optional<uint64_t> revCount;
std::optional<time_t> lastModified;
std::string type() const override { return "path"; }
std::optional<Hash> getRev() const override { return rev; }
bool operator ==(const Input & other) const override
{
auto other2 = dynamic_cast<const PathInput *>(&other);
return
other2
&& path == other2->path
&& rev == other2->rev
&& revCount == other2->revCount
&& lastModified == other2->lastModified;
}
bool isImmutable() const override
{
return (bool) narHash;
}
ParsedURL toURL() const override
{
auto query = attrsToQuery(toAttrsInternal());
query.erase("path");
return ParsedURL {
.scheme = "path",
.path = path,
.query = query,
};
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("path", path);
if (rev)
attrs.emplace("rev", rev->gitRev());
if (revCount)
attrs.emplace("revCount", *revCount);
if (lastModified)
attrs.emplace("lastModified", *lastModified);
return attrs;
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
{
auto input = std::make_shared<PathInput>(*this);
// FIXME: check whether access to 'path' is allowed.
auto storePath = store->maybeParseStorePath(path);
if (storePath)
store->addTempRoot(*storePath);
if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath))
// FIXME: try to substitute storePath.
storePath = store->addToStore("source", path);
return
{
Tree {
.actualPath = store->toRealPath(*storePath),
.storePath = std::move(*storePath),
.info = TreeInfo {
.revCount = revCount,
.lastModified = lastModified
}
},
input
};
}
};
struct PathInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "path") return nullptr;
auto input = std::make_unique<PathInput>();
input->path = url.path;
for (auto & [name, value] : url.query)
if (name == "rev")
input->rev = Hash(value, htSHA1);
else if (name == "revCount") {
uint64_t revCount;
if (!string2Int(value, revCount))
throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
input->revCount = revCount;
}
else if (name == "lastModified") {
time_t lastModified;
if (!string2Int(value, lastModified))
throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
input->lastModified = lastModified;
}
else
throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
return input;
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "path") return {};
auto input = std::make_unique<PathInput>();
input->path = getStrAttr(attrs, "path");
for (auto & [name, value] : attrs)
if (name == "rev")
input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1);
else if (name == "revCount")
input->revCount = getIntAttr(attrs, "revCount");
else if (name == "lastModified")
input->lastModified = getIntAttr(attrs, "lastModified");
else if (name == "type" || name == "path")
;
else
throw Error("unsupported path input attribute '%s'", name);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); });
}

275
src/libfetchers/tarball.cc Normal file
View file

@ -0,0 +1,275 @@
#include "fetchers.hh"
#include "cache.hh"
#include "filetransfer.hh"
#include "globals.hh"
#include "store-api.hh"
#include "archive.hh"
#include "tarfile.hh"
namespace nix::fetchers {
DownloadFileResult downloadFile(
ref<Store> store,
const std::string & url,
const std::string & name,
bool immutable)
{
// FIXME: check store
Attrs inAttrs({
{"type", "file"},
{"url", url},
{"name", name},
});
auto cached = getCache()->lookupExpired(store, inAttrs);
auto useCached = [&]() -> DownloadFileResult
{
return {
.storePath = std::move(cached->storePath),
.etag = getStrAttr(cached->infoAttrs, "etag"),
.effectiveUrl = getStrAttr(cached->infoAttrs, "url")
};
};
if (cached && !cached->expired)
return useCached();
FileTransferRequest request(url);
if (cached)
request.expectedETag = getStrAttr(cached->infoAttrs, "etag");
FileTransferResult res;
try {
res = getFileTransfer()->download(request);
} catch (FileTransferError & e) {
if (cached) {
warn("%s; using cached version", e.msg());
return useCached();
} else
throw;
}
// FIXME: write to temporary file.
Attrs infoAttrs({
{"etag", res.etag},
{"url", res.effectiveUri},
});
std::optional<StorePath> storePath;
if (res.cached) {
assert(cached);
assert(request.expectedETag == res.etag);
storePath = std::move(cached->storePath);
} else {
StringSink sink;
dumpString(*res.data, sink);
auto hash = hashString(htSHA256, *res.data);
ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
info.narHash = hashString(htSHA256, *sink.s);
info.narSize = sink.s->size();
info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash);
auto source = StringSource { *sink.s };
store->addToStore(info, source, NoRepair, NoCheckSigs);
storePath = std::move(info.path);
}
getCache()->add(
store,
inAttrs,
infoAttrs,
*storePath,
immutable);
if (url != res.effectiveUri)
getCache()->add(
store,
{
{"type", "file"},
{"url", res.effectiveUri},
{"name", name},
},
infoAttrs,
*storePath,
immutable);
return {
.storePath = std::move(*storePath),
.etag = res.etag,
.effectiveUrl = res.effectiveUri,
};
}
Tree downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
bool immutable)
{
Attrs inAttrs({
{"type", "tarball"},
{"url", url},
{"name", name},
});
auto cached = getCache()->lookupExpired(store, inAttrs);
if (cached && !cached->expired)
return Tree {
.actualPath = store->toRealPath(cached->storePath),
.storePath = std::move(cached->storePath),
.info = TreeInfo {
.lastModified = getIntAttr(cached->infoAttrs, "lastModified"),
},
};
auto res = downloadFile(store, url, name, immutable);
std::optional<StorePath> unpackedStorePath;
time_t lastModified;
if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) {
unpackedStorePath = std::move(cached->storePath);
lastModified = getIntAttr(cached->infoAttrs, "lastModified");
} else {
Path tmpDir = createTempDir();
AutoDelete autoDelete(tmpDir, true);
unpackTarfile(store->toRealPath(res.storePath), tmpDir);
auto members = readDirectory(tmpDir);
if (members.size() != 1)
throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
auto topDir = tmpDir + "/" + members.begin()->name;
lastModified = lstat(topDir).st_mtime;
unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair);
}
Attrs infoAttrs({
{"lastModified", lastModified},
{"etag", res.etag},
});
getCache()->add(
store,
inAttrs,
infoAttrs,
*unpackedStorePath,
immutable);
return Tree {
.actualPath = store->toRealPath(*unpackedStorePath),
.storePath = std::move(*unpackedStorePath),
.info = TreeInfo {
.lastModified = lastModified,
},
};
}
struct TarballInput : Input
{
ParsedURL url;
std::optional<Hash> hash;
TarballInput(const ParsedURL & url) : url(url)
{ }
std::string type() const override { return "tarball"; }
bool operator ==(const Input & other) const override
{
auto other2 = dynamic_cast<const TarballInput *>(&other);
return
other2
&& to_string() == other2->to_string()
&& hash == other2->hash;
}
bool isImmutable() const override
{
return hash || narHash;
}
ParsedURL toURL() const override
{
auto url2(url);
// NAR hashes are preferred over file hashes since tar/zip files
// don't have a canonical representation.
if (narHash)
url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
else if (hash)
url2.query.insert_or_assign("hash", hash->to_string(SRI, true));
return url2;
}
Attrs toAttrsInternal() const override
{
Attrs attrs;
attrs.emplace("url", url.to_string());
if (hash)
attrs.emplace("hash", hash->to_string(SRI, true));
return attrs;
}
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
{
auto tree = downloadTarball(store, url.to_string(), "source", false);
auto input = std::make_shared<TarballInput>(*this);
input->narHash = store->queryPathInfo(tree.storePath)->narHash;
return {std::move(tree), input};
}
};
struct TarballInputScheme : InputScheme
{
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr;
if (!hasSuffix(url.path, ".zip")
&& !hasSuffix(url.path, ".tar")
&& !hasSuffix(url.path, ".tar.gz")
&& !hasSuffix(url.path, ".tar.xz")
&& !hasSuffix(url.path, ".tar.bz2"))
return nullptr;
auto input = std::make_unique<TarballInput>(url);
auto hash = input->url.query.find("hash");
if (hash != input->url.query.end()) {
// FIXME: require SRI hash.
input->hash = Hash(hash->second);
input->url.query.erase(hash);
}
auto narHash = input->url.query.find("narHash");
if (narHash != input->url.query.end()) {
// FIXME: require SRI hash.
input->narHash = Hash(narHash->second);
input->url.query.erase(narHash);
}
return input;
}
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
for (auto & [name, value] : attrs)
if (name != "type" && name != "url" && name != "hash")
throw Error("unsupported tarball input attribute '%s'", name);
auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
if (auto hash = maybeGetStrAttr(attrs, "hash"))
input->hash = newHashAllowEmpty(*hash, htUnknown);
return input;
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
}

View file

@ -0,0 +1,14 @@
#include "tree-info.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
namespace nix::fetchers {
StorePath TreeInfo::computeStorePath(Store & store) const
{
assert(narHash);
return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source");
}
}

View file

@ -0,0 +1,29 @@
#pragma once
#include "path.hh"
#include "hash.hh"
#include <nlohmann/json_fwd.hpp>
namespace nix { class Store; }
namespace nix::fetchers {
struct TreeInfo
{
Hash narHash;
std::optional<uint64_t> revCount;
std::optional<time_t> lastModified;
bool operator ==(const TreeInfo & other) const
{
return
narHash == other.narHash
&& revCount == other.revCount
&& lastModified == other.lastModified;
}
StorePath computeStorePath(Store & store) const;
};
}

View file

@ -1,48 +1,61 @@
#include "common-args.hh" #include "common-args.hh"
#include "globals.hh" #include "globals.hh"
#include "loggers.hh"
namespace nix { namespace nix {
MixCommonArgs::MixCommonArgs(const string & programName) MixCommonArgs::MixCommonArgs(const string & programName)
: programName(programName) : programName(programName)
{ {
mkFlag() addFlag({
.longName("verbose") .longName = "verbose",
.shortName('v') .shortName = 'v',
.description("increase verbosity level") .description = "increase verbosity level",
.handler([]() { verbosity = (Verbosity) (verbosity + 1); }); .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }},
});
mkFlag() addFlag({
.longName("quiet") .longName = "quiet",
.description("decrease verbosity level") .description = "decrease verbosity level",
.handler([]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }); .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }},
});
mkFlag() addFlag({
.longName("debug") .longName = "debug",
.description("enable debug output") .description = "enable debug output",
.handler([]() { verbosity = lvlDebug; }); .handler = {[]() { verbosity = lvlDebug; }},
});
mkFlag() addFlag({
.longName("option") .longName = "option",
.labels({"name", "value"}) .description = "set a Nix configuration option (overriding nix.conf)",
.description("set a Nix configuration option (overriding nix.conf)") .labels = {"name", "value"},
.arity(2) .handler = {[](std::string name, std::string value) {
.handler([](std::vector<std::string> ss) {
try { try {
globalConfig.set(ss[0], ss[1]); globalConfig.set(name, value);
} catch (UsageError & e) { } catch (UsageError & e) {
warn(e.what()); warn(e.what());
} }
}); }},
});
mkFlag() addFlag({
.longName("max-jobs") .longName = "log-format",
.shortName('j') .description = "format of log output; \"raw\", \"internal-json\", \"bar\" "
.label("jobs") "or \"bar-with-logs\"",
.description("maximum number of parallel builds") .labels = {"format"},
.handler([=](std::string s) { .handler = {[](std::string format) { setLogFormat(format); }},
});
addFlag({
.longName = "max-jobs",
.shortName = 'j',
.description = "maximum number of parallel builds",
.labels = Strings{"jobs"},
.handler = {[=](std::string s) {
settings.set("max-jobs", s); settings.set("max-jobs", s);
}); }}
});
std::string cat = "config"; std::string cat = "config";
globalConfig.convertToArgs(*this, cat); globalConfig.convertToArgs(*this, cat);

View file

@ -6,6 +6,8 @@ libmain_DIR := $(d)
libmain_SOURCES := $(wildcard $(d)/*.cc) libmain_SOURCES := $(wildcard $(d)/*.cc)
libmain_CXXFLAGS += -I src/libutil -I src/libstore
libmain_LDFLAGS = $(OPENSSL_LIBS) libmain_LDFLAGS = $(OPENSSL_LIBS)
libmain_LIBS = libstore libutil libmain_LIBS = libstore libutil

52
src/libmain/loggers.cc Normal file
View file

@ -0,0 +1,52 @@
#include "loggers.hh"
#include "progress-bar.hh"
namespace nix {
LogFormat defaultLogFormat = LogFormat::raw;
LogFormat parseLogFormat(const std::string & logFormatStr) {
if (logFormatStr == "raw")
return LogFormat::raw;
else if (logFormatStr == "raw-with-logs")
return LogFormat::rawWithLogs;
else if (logFormatStr == "internal-json")
return LogFormat::internalJson;
else if (logFormatStr == "bar")
return LogFormat::bar;
else if (logFormatStr == "bar-with-logs")
return LogFormat::barWithLogs;
throw Error("option 'log-format' has an invalid value '%s'", logFormatStr);
}
Logger * makeDefaultLogger() {
switch (defaultLogFormat) {
case LogFormat::raw:
return makeSimpleLogger(false);
case LogFormat::rawWithLogs:
return makeSimpleLogger(true);
case LogFormat::internalJson:
return makeJSONLogger(*makeSimpleLogger());
case LogFormat::bar:
return makeProgressBar();
case LogFormat::barWithLogs:
return makeProgressBar(true);
default:
abort();
}
}
void setLogFormat(const std::string & logFormatStr) {
setLogFormat(parseLogFormat(logFormatStr));
}
void setLogFormat(const LogFormat & logFormat) {
defaultLogFormat = logFormat;
createDefaultLogger();
}
void createDefaultLogger() {
logger = makeDefaultLogger();
}
}

20
src/libmain/loggers.hh Normal file
View file

@ -0,0 +1,20 @@
#pragma once
#include "types.hh"
namespace nix {
enum class LogFormat {
raw,
rawWithLogs,
internalJson,
bar,
barWithLogs,
};
void setLogFormat(const std::string & logFormatStr);
void setLogFormat(const LogFormat & logFormat);
void createDefaultLogger();
}

View file

@ -7,6 +7,7 @@
#include <atomic> #include <atomic>
#include <map> #include <map>
#include <thread> #include <thread>
#include <iostream>
namespace nix { namespace nix {
@ -105,25 +106,36 @@ public:
updateThread.join(); updateThread.join();
} }
void stop() void stop() override
{ {
auto state(state_.lock()); auto state(state_.lock());
if (!state->active) return; if (!state->active) return;
state->active = false; state->active = false;
std::string status = getStatus(*state);
writeToStderr("\r\e[K"); writeToStderr("\r\e[K");
if (status != "")
writeToStderr("[" + status + "]\n");
updateCV.notify_one(); updateCV.notify_one();
quitCV.notify_one(); quitCV.notify_one();
} }
bool isVerbose() override {
return printBuildLogs;
}
void log(Verbosity lvl, const FormatOrString & fs) override void log(Verbosity lvl, const FormatOrString & fs) override
{ {
auto state(state_.lock()); auto state(state_.lock());
log(*state, lvl, fs.s); log(*state, lvl, fs.s);
} }
void logEI(const ErrorInfo &ei) override
{
auto state(state_.lock());
std::stringstream oss;
oss << ei;
log(*state, ei.level, oss.str());
}
void log(State & state, Verbosity lvl, const std::string & s) void log(State & state, Verbosity lvl, const std::string & s)
{ {
if (state.active) { if (state.active) {
@ -141,7 +153,7 @@ public:
{ {
auto state(state_.lock()); auto state(state_.lock());
if (lvl <= verbosity && !s.empty()) if (lvl <= verbosity && !s.empty() && type != actBuildWaiting)
log(*state, lvl, s + "..."); log(*state, lvl, s + "...");
state->activities.emplace_back(ActInfo()); state->activities.emplace_back(ActInfo());
@ -153,7 +165,7 @@ public:
state->activitiesByType[type].its.emplace(act, i); state->activitiesByType[type].its.emplace(act, i);
if (type == actBuild) { if (type == actBuild) {
auto name = storePathToName(getS(fields, 0)); std::string name(storePathToName(getS(fields, 0)));
if (hasSuffix(name, ".drv")) if (hasSuffix(name, ".drv"))
name = name.substr(0, name.size() - 4); name = name.substr(0, name.size() - 4);
i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name); i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name);
@ -190,8 +202,8 @@ public:
i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1)); i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1));
} }
if ((type == actDownload && hasAncestor(*state, actCopyPath, parent)) if ((type == actFileTransfer && hasAncestor(*state, actCopyPath, parent))
|| (type == actDownload && hasAncestor(*state, actQueryPathInfo, parent)) || (type == actFileTransfer && hasAncestor(*state, actQueryPathInfo, parent))
|| (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent)))
i->visible = false; i->visible = false;
@ -416,7 +428,7 @@ public:
if (!s2.empty()) { res += " ("; res += s2; res += ')'; } if (!s2.empty()) { res += " ("; res += s2; res += ')'; }
} }
showActivity(actDownload, "%s MiB DL", "%.1f", MiB); showActivity(actFileTransfer, "%s MiB DL", "%.1f", MiB);
{ {
auto s = renderActivity(actOptimiseStore, "%s paths optimised"); auto s = renderActivity(actOptimiseStore, "%s paths optimised");
@ -442,13 +454,31 @@ public:
return res; return res;
} }
void writeToStdout(std::string_view s) override
{
auto state(state_.lock());
if (state->active) {
std::cerr << "\r\e[K";
Logger::writeToStdout(s);
draw(*state);
} else {
Logger::writeToStdout(s);
}
}
}; };
Logger * makeProgressBar(bool printBuildLogs)
{
return new ProgressBar(
printBuildLogs,
isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb"
);
}
void startProgressBar(bool printBuildLogs) void startProgressBar(bool printBuildLogs)
{ {
logger = new ProgressBar( logger = makeProgressBar(printBuildLogs);
printBuildLogs,
isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb");
} }
void stopProgressBar() void stopProgressBar()

View file

@ -4,6 +4,8 @@
namespace nix { namespace nix {
Logger * makeProgressBar(bool printBuildLogs = false);
void startProgressBar(bool printBuildLogs = false); void startProgressBar(bool printBuildLogs = false);
void stopProgressBar(); void stopProgressBar();

View file

@ -2,6 +2,7 @@
#include "shared.hh" #include "shared.hh"
#include "store-api.hh" #include "store-api.hh"
#include "util.hh" #include "util.hh"
#include "loggers.hh"
#include <algorithm> #include <algorithm>
#include <cctype> #include <cctype>
@ -75,7 +76,7 @@ string getArg(const string & opt,
Strings::iterator & i, const Strings::iterator & end) Strings::iterator & i, const Strings::iterator & end)
{ {
++i; ++i;
if (i == end) throw UsageError(format("'%1%' requires an argument") % opt); if (i == end) throw UsageError("'%1%' requires an argument", opt);
return *i; return *i;
} }
@ -155,7 +156,7 @@ void initNix()
sshd). This breaks build users because they don't have access sshd). This breaks build users because they don't have access
to the TMPDIR, in particular in nix-store --serve. */ to the TMPDIR, in particular in nix-store --serve. */
#if __APPLE__ #if __APPLE__
if (getuid() == 0 && hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
unsetenv("TMPDIR"); unsetenv("TMPDIR");
#endif #endif
} }
@ -165,28 +166,32 @@ LegacyArgs::LegacyArgs(const std::string & programName,
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
: MixCommonArgs(programName), parseArg(parseArg) : MixCommonArgs(programName), parseArg(parseArg)
{ {
mkFlag() addFlag({
.longName("no-build-output") .longName = "no-build-output",
.shortName('Q') .shortName = 'Q',
.description("do not show build output") .description = "do not show build output",
.set(&settings.verboseBuild, false); .handler = {[&]() {setLogFormat(LogFormat::raw); }},
});
mkFlag() addFlag({
.longName("keep-failed") .longName = "keep-failed",
.shortName('K') .shortName ='K',
.description("keep temporary directories of failed builds") .description = "keep temporary directories of failed builds",
.set(&(bool&) settings.keepFailed, true); .handler = {&(bool&) settings.keepFailed, true},
});
mkFlag() addFlag({
.longName("keep-going") .longName = "keep-going",
.shortName('k') .shortName ='k',
.description("keep going after a build fails") .description = "keep going after a build fails",
.set(&(bool&) settings.keepGoing, true); .handler = {&(bool&) settings.keepGoing, true},
});
mkFlag() addFlag({
.longName("fallback") .longName = "fallback",
.description("build from source if substitution fails") .description = "build from source if substitution fails",
.set(&(bool&) settings.tryFallback, true); .handler = {&(bool&) settings.tryFallback, true},
});
auto intSettingAlias = [&](char shortName, const std::string & longName, auto intSettingAlias = [&](char shortName, const std::string & longName,
const std::string & description, const std::string & dest) { const std::string & description, const std::string & dest) {
@ -205,11 +210,12 @@ LegacyArgs::LegacyArgs(const std::string & programName,
mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'",
&gcWarning, false); &gcWarning, false);
mkFlag() addFlag({
.longName("store") .longName = "store",
.label("store-uri") .description = "URI of the Nix store to use",
.description("URI of the Nix store to use") .labels = {"store-uri"},
.dest(&(std::string&) settings.storeUri); .handler = {&(std::string&) settings.storeUri},
});
} }
@ -229,7 +235,7 @@ bool LegacyArgs::processArgs(const Strings & args, bool finish)
Strings ss(args); Strings ss(args);
auto pos = ss.begin(); auto pos = ss.begin();
if (!parseArg(pos, ss.end())) if (!parseArg(pos, ss.end()))
throw UsageError(format("unexpected argument '%1%'") % args.front()); throw UsageError("unexpected argument '%1%'", args.front());
return true; return true;
} }
@ -260,7 +266,10 @@ void printVersion(const string & programName)
cfg.push_back("signed-caches"); cfg.push_back("signed-caches");
#endif #endif
std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" << "\n"; std::cout << "System configuration file: " << settings.nixConfDir + "/nix.conf" << "\n";
std::cout << "User configuration files: " <<
concatStringsSep(":", settings.nixUserConfFiles)
<< "\n";
std::cout << "Store directory: " << settings.nixStore << "\n"; std::cout << "Store directory: " << settings.nixStore << "\n";
std::cout << "State directory: " << settings.nixStateDir << "\n"; std::cout << "State directory: " << settings.nixStateDir << "\n";
} }
@ -273,7 +282,7 @@ void showManPage(const string & name)
restoreSignals(); restoreSignals();
setenv("MANPATH", settings.nixManDir.c_str(), 1); setenv("MANPATH", settings.nixManDir.c_str(), 1);
execlp("man", "man", name.c_str(), nullptr); execlp("man", "man", name.c_str(), nullptr);
throw SysError(format("command 'man %1%' failed") % name.c_str()); throw SysError("command 'man %1%' failed", name.c_str());
} }
@ -281,6 +290,8 @@ int handleExceptions(const string & programName, std::function<void()> fun)
{ {
ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this
ErrorInfo::programName = baseNameOf(programName);
string error = ANSI_RED "error:" ANSI_NORMAL " "; string error = ANSI_RED "error:" ANSI_NORMAL " ";
try { try {
try { try {
@ -296,12 +307,13 @@ int handleExceptions(const string & programName, std::function<void()> fun)
} catch (Exit & e) { } catch (Exit & e) {
return e.status; return e.status;
} catch (UsageError & e) { } catch (UsageError & e) {
printError( logError(e.info());
format(error + "%1%\nTry '%2% --help' for more information.") printError("Try '%1% --help' for more information.", programName);
% e.what() % programName);
return 1; return 1;
} catch (BaseError & e) { } catch (BaseError & e) {
printError(format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); if (settings.showTrace && e.prefix() != "")
printError(e.prefix());
logError(e.info());
if (e.prefix() != "" && !settings.showTrace) if (e.prefix() != "" && !settings.showTrace)
printError("(use '--show-trace' to show detailed location information)"); printError("(use '--show-trace' to show detailed location information)");
return e.status; return e.status;
@ -338,7 +350,7 @@ RunPager::RunPager()
execlp("pager", "pager", nullptr); execlp("pager", "pager", nullptr);
execlp("less", "less", nullptr); execlp("less", "less", nullptr);
execlp("more", "more", nullptr); execlp("more", "more", nullptr);
throw SysError(format("executing '%1%'") % pager); throw SysError("executing '%1%'", pager);
}); });
pid.setKillSignal(SIGINT); pid.setKillSignal(SIGINT);

View file

@ -56,7 +56,7 @@ template<class N> N getIntArg(const string & opt,
Strings::iterator & i, const Strings::iterator & end, bool allowUnit) Strings::iterator & i, const Strings::iterator & end, bool allowUnit)
{ {
++i; ++i;
if (i == end) throw UsageError(format("'%1%' requires an argument") % opt); if (i == end) throw UsageError("'%1%' requires an argument", opt);
string s = *i; string s = *i;
N multiplier = 1; N multiplier = 1;
if (allowUnit && !s.empty()) { if (allowUnit && !s.empty()) {
@ -66,13 +66,13 @@ template<class N> N getIntArg(const string & opt,
else if (u == 'M') multiplier = 1ULL << 20; else if (u == 'M') multiplier = 1ULL << 20;
else if (u == 'G') multiplier = 1ULL << 30; else if (u == 'G') multiplier = 1ULL << 30;
else if (u == 'T') multiplier = 1ULL << 40; else if (u == 'T') multiplier = 1ULL << 40;
else throw UsageError(format("invalid unit specifier '%1%'") % u); else throw UsageError("invalid unit specifier '%1%'", u);
s.resize(s.size() - 1); s.resize(s.size() - 1);
} }
} }
N n; N n;
if (!string2Int(s, n)) if (!string2Int(s, n))
throw UsageError(format("'%1%' requires an integer argument") % opt); throw UsageError("'%1%' requires an integer argument", opt);
return n * multiplier; return n * multiplier;
} }

View file

@ -1,4 +1,4 @@
#include "types.hh" #include "error.hh"
#include <cstring> #include <cstring>
#include <cstddef> #include <cstddef>

View file

@ -40,14 +40,14 @@ void BinaryCacheStore::init()
upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info");
} else { } else {
for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) { for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) {
size_t colon = line.find(':'); size_t colon= line.find(':');
if (colon == std::string::npos) continue; if (colon ==std::string::npos) continue;
auto name = line.substr(0, colon); auto name = line.substr(0, colon);
auto value = trim(line.substr(colon + 1, std::string::npos)); auto value = trim(line.substr(colon + 1, std::string::npos));
if (name == "StoreDir") { if (name == "StoreDir") {
if (value != storeDir) if (value != storeDir)
throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'") throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'",
% getUri() % value % storeDir); getUri(), value, storeDir);
} else if (name == "WantMassQuery") { } else if (name == "WantMassQuery") {
wantMassQuery.setDefault(value == "1" ? "true" : "false"); wantMassQuery.setDefault(value == "1" ? "true" : "false");
} else if (name == "Priority") { } else if (name == "Priority") {
@ -93,7 +93,7 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath) std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
{ {
return storePathToHash(printStorePath(storePath)) + ".narinfo"; return std::string(storePath.hashPart()) + ".narinfo";
} }
void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
@ -102,7 +102,7 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
auto hashPart = storePathToHash(printStorePath(narInfo->path)); std::string hashPart(narInfo->path.hashPart());
{ {
auto state_(state.lock()); auto state_(state.lock());
@ -113,9 +113,12 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
} }
void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
{ {
// FIXME: See if we can use the original source to reduce memory usage.
auto nar = make_ref<std::string>(narSource.drain());
if (!repair && isValidPath(info.path)) return; if (!repair && isValidPath(info.path)) return;
/* Verify that all references are valid. This may do some .narinfo /* Verify that all references are valid. This may do some .narinfo
@ -161,7 +164,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
} }
} }
upsertFile(storePathToHash(printStorePath(info.path)) + ".ls", jsonOut.str(), "application/json"); upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
} }
/* Compress the NAR. */ /* Compress the NAR. */
@ -284,7 +287,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
try { try {
getFile(info->url, *decompressor); getFile(info->url, *decompressor);
} catch (NoSuchBinaryCacheFile & e) { } catch (NoSuchBinaryCacheFile & e) {
throw SubstituteGone(e.what()); throw SubstituteGone(e.info());
} }
decompressor->finish(); decompressor->finish();
@ -327,7 +330,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
} }
StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
{ {
// FIXME: some cut&paste from LocalStore::addToStore(). // FIXME: some cut&paste from LocalStore::addToStore().
@ -336,7 +339,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
small files. */ small files. */
StringSink sink; StringSink sink;
Hash h; Hash h;
if (recursive) { if (method == FileIngestionMethod::Recursive) {
dumpPath(srcPath, sink, filter); dumpPath(srcPath, sink, filter);
h = hashString(hashAlgo, *sink.s); h = hashString(hashAlgo, *sink.s);
} else { } else {
@ -345,9 +348,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
h = hashString(hashAlgo, s); h = hashString(hashAlgo, s);
} }
ValidPathInfo info(makeFixedOutputPath(recursive, h, name)); ValidPathInfo info(makeFixedOutputPath(method, h, name));
addToStore(info, sink.s, repair, CheckSigs, nullptr); auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs, nullptr);
return std::move(info.path); return std::move(info.path);
} }
@ -356,12 +360,13 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
const StorePathSet & references, RepairFlag repair) const StorePathSet & references, RepairFlag repair)
{ {
ValidPathInfo info(computeStorePathForText(name, s, references)); ValidPathInfo info(computeStorePathForText(name, s, references));
info.references = cloneStorePathSet(references); info.references = references;
if (repair || !isValidPath(info.path)) { if (repair || !isValidPath(info.path)) {
StringSink sink; StringSink sink;
dumpString(s, sink); dumpString(s, sink);
addToStore(info, sink.s, repair, CheckSigs, nullptr); auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs, nullptr);
} }
return std::move(info.path); return std::move(info.path);
@ -390,14 +395,14 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path) std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
{ {
auto drvPath = path.clone(); auto drvPath = path;
if (!path.isDerivation()) { if (!path.isDerivation()) {
try { try {
auto info = queryPathInfo(path); auto info = queryPathInfo(path);
// FIXME: add a "Log" field to .narinfo // FIXME: add a "Log" field to .narinfo
if (!info->deriver) return nullptr; if (!info->deriver) return nullptr;
drvPath = info->deriver->clone(); drvPath = *info->deriver;
} catch (InvalidPath &) { } catch (InvalidPath &) {
return nullptr; return nullptr;
} }

View file

@ -74,12 +74,12 @@ public:
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ unsupported("queryPathFromHashPart"); } { unsupported("queryPathFromHashPart"); }
void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs, RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override; std::shared_ptr<FSAccessor> accessor) override;
StorePath addToStore(const string & name, const Path & srcPath, StorePath addToStore(const string & name, const Path & srcPath,
bool recursive, HashType hashAlgo, FileIngestionMethod method, HashType hashAlgo,
PathFilter & filter, RepairFlag repair) override; PathFilter & filter, RepairFlag repair) override;
StorePath addTextToStore(const string & name, const string & s, StorePath addTextToStore(const string & name, const string & s,

File diff suppressed because it is too large Load diff

View file

@ -6,7 +6,6 @@ namespace nix {
// TODO: make pluggable. // TODO: make pluggable.
void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData);
void builtinBuildenv(const BasicDerivation & drv);
void builtinUnpackChannel(const BasicDerivation & drv); void builtinUnpackChannel(const BasicDerivation & drv);
} }

View file

@ -1,4 +1,4 @@
#include "builtins.hh" #include "buildenv.hh"
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/types.h> #include <sys/types.h>
@ -7,16 +7,14 @@
namespace nix { namespace nix {
typedef std::map<Path,int> Priorities; struct State
{
// FIXME: change into local variables. std::map<Path, int> priorities;
unsigned long symlinks = 0;
static Priorities priorities; };
static unsigned long symlinks;
/* For each activated package, create symlinks */ /* For each activated package, create symlinks */
static void createLinks(const Path & srcDir, const Path & dstDir, int priority) static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority)
{ {
DirEntries srcFiles; DirEntries srcFiles;
@ -24,7 +22,10 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
srcFiles = readDirectory(srcDir); srcFiles = readDirectory(srcDir);
} catch (SysError & e) { } catch (SysError & e) {
if (e.errNo == ENOTDIR) { if (e.errNo == ENOTDIR) {
printError("warning: not including '%s' in the user environment because it's not a directory", srcDir); logWarning({
.name = "Create links - directory",
.hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir)
});
return; return;
} }
throw; throw;
@ -43,7 +44,10 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
throw SysError("getting status of '%1%'", srcFile); throw SysError("getting status of '%1%'", srcFile);
} catch (SysError & e) { } catch (SysError & e) {
if (e.errNo == ENOENT || e.errNo == ENOTDIR) { if (e.errNo == ENOENT || e.errNo == ENOTDIR) {
printError("warning: skipping dangling symlink '%s'", dstFile); logWarning({
.name = "Create links - skipping symlink",
.hint = hintfmt("skipping dangling symlink '%s'", dstFile)
});
continue; continue;
} }
throw; throw;
@ -67,22 +71,22 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
auto res = lstat(dstFile.c_str(), &dstSt); auto res = lstat(dstFile.c_str(), &dstSt);
if (res == 0) { if (res == 0) {
if (S_ISDIR(dstSt.st_mode)) { if (S_ISDIR(dstSt.st_mode)) {
createLinks(srcFile, dstFile, priority); createLinks(state, srcFile, dstFile, priority);
continue; continue;
} else if (S_ISLNK(dstSt.st_mode)) { } else if (S_ISLNK(dstSt.st_mode)) {
auto target = canonPath(dstFile, true); auto target = canonPath(dstFile, true);
if (!S_ISDIR(lstat(target).st_mode)) if (!S_ISDIR(lstat(target).st_mode))
throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target); throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target);
if (unlink(dstFile.c_str()) == -1) if (unlink(dstFile.c_str()) == -1)
throw SysError(format("unlinking '%1%'") % dstFile); throw SysError("unlinking '%1%'", dstFile);
if (mkdir(dstFile.c_str(), 0755) == -1) if (mkdir(dstFile.c_str(), 0755) == -1)
throw SysError(format("creating directory '%1%'")); throw SysError("creating directory '%1%'", dstFile);
createLinks(target, dstFile, priorities[dstFile]); createLinks(state, target, dstFile, state.priorities[dstFile]);
createLinks(srcFile, dstFile, priority); createLinks(state, srcFile, dstFile, priority);
continue; continue;
} }
} else if (errno != ENOENT) } else if (errno != ENOENT)
throw SysError(format("getting status of '%1%'") % dstFile); throw SysError("getting status of '%1%'", dstFile);
} }
else { else {
@ -90,7 +94,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
auto res = lstat(dstFile.c_str(), &dstSt); auto res = lstat(dstFile.c_str(), &dstSt);
if (res == 0) { if (res == 0) {
if (S_ISLNK(dstSt.st_mode)) { if (S_ISLNK(dstSt.st_mode)) {
auto prevPriority = priorities[dstFile]; auto prevPriority = state.priorities[dstFile];
if (prevPriority == priority) if (prevPriority == priority)
throw Error( throw Error(
"packages '%1%' and '%2%' have the same priority %3%; " "packages '%1%' and '%2%' have the same priority %3%; "
@ -101,75 +105,38 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
if (prevPriority < priority) if (prevPriority < priority)
continue; continue;
if (unlink(dstFile.c_str()) == -1) if (unlink(dstFile.c_str()) == -1)
throw SysError(format("unlinking '%1%'") % dstFile); throw SysError("unlinking '%1%'", dstFile);
} else if (S_ISDIR(dstSt.st_mode)) } else if (S_ISDIR(dstSt.st_mode))
throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile); throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile);
} else if (errno != ENOENT) } else if (errno != ENOENT)
throw SysError(format("getting status of '%1%'") % dstFile); throw SysError("getting status of '%1%'", dstFile);
} }
createSymlink(srcFile, dstFile); createSymlink(srcFile, dstFile);
priorities[dstFile] = priority; state.priorities[dstFile] = priority;
symlinks++; state.symlinks++;
} }
} }
typedef std::set<Path> FileProp; void buildProfile(const Path & out, Packages && pkgs)
static FileProp done;
static FileProp postponed = FileProp{};
static Path out;
static void addPkg(const Path & pkgDir, int priority)
{ {
if (!done.insert(pkgDir).second) return; State state;
createLinks(pkgDir, out, priority);
try { std::set<Path> done, postponed;
for (const auto & p : tokenizeString<std::vector<string>>(
readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
if (!done.count(p))
postponed.insert(p);
} catch (SysError & e) {
if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
}
}
struct Package { auto addPkg = [&](const Path & pkgDir, int priority) {
Path path; if (!done.insert(pkgDir).second) return;
bool active; createLinks(state, pkgDir, out, priority);
int priority;
Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
};
typedef std::vector<Package> Packages; try {
for (const auto & p : tokenizeString<std::vector<string>>(
void builtinBuildenv(const BasicDerivation & drv) readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
{ if (!done.count(p))
auto getAttr = [&](const string & name) { postponed.insert(p);
auto i = drv.env.find(name); } catch (SysError & e) {
if (i == drv.env.end()) throw Error("attribute '%s' missing", name); if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
return i->second;
};
out = getAttr("out");
createDirs(out);
/* Convert the stuff we get from the environment back into a
* coherent data type. */
Packages pkgs;
auto derivations = tokenizeString<Strings>(getAttr("derivations"));
while (!derivations.empty()) {
/* !!! We're trusting the caller to structure derivations env var correctly */
auto active = derivations.front(); derivations.pop_front();
auto priority = stoi(derivations.front()); derivations.pop_front();
auto outputs = stoi(derivations.front()); derivations.pop_front();
for (auto n = 0; n < outputs; n++) {
auto path = derivations.front(); derivations.pop_front();
pkgs.emplace_back(path, active != "false", priority);
} }
} };
/* Symlink to the packages that have been installed explicitly by the /* Symlink to the packages that have been installed explicitly by the
* user. Process in priority order to reduce unnecessary * user. Process in priority order to reduce unnecessary
@ -189,13 +156,42 @@ void builtinBuildenv(const BasicDerivation & drv)
*/ */
auto priorityCounter = 1000; auto priorityCounter = 1000;
while (!postponed.empty()) { while (!postponed.empty()) {
auto pkgDirs = postponed; std::set<Path> pkgDirs;
postponed = FileProp{}; postponed.swap(pkgDirs);
for (const auto & pkgDir : pkgDirs) for (const auto & pkgDir : pkgDirs)
addPkg(pkgDir, priorityCounter++); addPkg(pkgDir, priorityCounter++);
} }
printError("created %d symlinks in user environment", symlinks); debug("created %d symlinks in user environment", state.symlinks);
}
void builtinBuildenv(const BasicDerivation & drv)
{
auto getAttr = [&](const string & name) {
auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
return i->second;
};
Path out = getAttr("out");
createDirs(out);
/* Convert the stuff we get from the environment back into a
* coherent data type. */
Packages pkgs;
auto derivations = tokenizeString<Strings>(getAttr("derivations"));
while (!derivations.empty()) {
/* !!! We're trusting the caller to structure derivations env var correctly */
auto active = derivations.front(); derivations.pop_front();
auto priority = stoi(derivations.front()); derivations.pop_front();
auto outputs = stoi(derivations.front()); derivations.pop_front();
for (auto n = 0; n < outputs; n++) {
auto path = derivations.front(); derivations.pop_front();
pkgs.emplace_back(path, active != "false", priority);
}
}
buildProfile(out, std::move(pkgs));
createSymlink(getAttr("manifest"), out + "/manifest.nix"); createSymlink(getAttr("manifest"), out + "/manifest.nix");
} }

View file

@ -0,0 +1,21 @@
#pragma once
#include "derivations.hh"
#include "store-api.hh"
namespace nix {
struct Package {
Path path;
bool active;
int priority;
Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
};
typedef std::vector<Package> Packages;
void buildProfile(const Path & out, Packages && pkgs);
void builtinBuildenv(const BasicDerivation & drv);
}

View file

@ -1,5 +1,5 @@
#include "builtins.hh" #include "builtins.hh"
#include "download.hh" #include "filetransfer.hh"
#include "store-api.hh" #include "store-api.hh"
#include "archive.hh" #include "archive.hh"
#include "compression.hh" #include "compression.hh"
@ -18,7 +18,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
auto getAttr = [&](const string & name) { auto getAttr = [&](const string & name) {
auto i = drv.env.find(name); auto i = drv.env.find(name);
if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % name); if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
return i->second; return i->second;
}; };
@ -26,9 +26,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
auto mainUrl = getAttr("url"); auto mainUrl = getAttr("url");
bool unpack = get(drv.env, "unpack").value_or("") == "1"; bool unpack = get(drv.env, "unpack").value_or("") == "1";
/* Note: have to use a fresh downloader here because we're in /* Note: have to use a fresh fileTransfer here because we're in
a forked process. */ a forked process. */
auto downloader = makeDownloader(); auto fileTransfer = makeFileTransfer();
auto fetch = [&](const std::string & url) { auto fetch = [&](const std::string & url) {
@ -36,13 +36,13 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
/* No need to do TLS verification, because we check the hash of /* No need to do TLS verification, because we check the hash of
the result anyway. */ the result anyway. */
DownloadRequest request(url); FileTransferRequest request(url);
request.verifyTLS = false; request.verifyTLS = false;
request.decompress = false; request.decompress = false;
auto decompressor = makeDecompressionSink( auto decompressor = makeDecompressionSink(
unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
downloader->download(std::move(request), *decompressor); fileTransfer->download(std::move(request), *decompressor);
decompressor->finish(); decompressor->finish();
}); });
@ -54,7 +54,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
auto executable = drv.env.find("executable"); auto executable = drv.env.find("executable");
if (executable != drv.env.end() && executable->second == "1") { if (executable != drv.env.end() && executable->second == "1") {
if (chmod(storePath.c_str(), 0755) == -1) if (chmod(storePath.c_str(), 0755) == -1)
throw SysError(format("making '%1%' executable") % storePath); throw SysError("making '%1%' executable", storePath);
} }
}; };

View file

@ -73,6 +73,18 @@ struct TunnelLogger : public Logger
enqueueMsg(*buf.s); enqueueMsg(*buf.s);
} }
void logEI(const ErrorInfo & ei) override
{
if (ei.level > verbosity) return;
std::stringstream oss;
oss << ei;
StringSink buf;
buf << STDERR_NEXT << oss.str() << "\n"; // (fs.s + "\n");
enqueueMsg(*buf.s);
}
/* startWork() means that we're starting an operation for which we /* startWork() means that we're starting an operation for which we
want to send out stderr to the client. */ want to send out stderr to the client. */
void startWork() void startWork()
@ -281,7 +293,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto path = store->parseStorePath(readString(from)); auto path = store->parseStorePath(readString(from));
logger->startWork(); logger->startWork();
StorePathSet paths; // FIXME StorePathSet paths; // FIXME
paths.insert(path.clone()); paths.insert(path);
auto res = store->querySubstitutablePaths(paths); auto res = store->querySubstitutablePaths(paths);
logger->stopWork(); logger->stopWork();
to << (res.count(path) != 0); to << (res.count(path) != 0);
@ -315,7 +327,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
StorePathSet paths; StorePathSet paths;
if (op == wopQueryReferences) if (op == wopQueryReferences)
for (auto & i : store->queryPathInfo(path)->references) for (auto & i : store->queryPathInfo(path)->references)
paths.insert(i.clone()); paths.insert(i);
else if (op == wopQueryReferrers) else if (op == wopQueryReferrers)
store->queryReferrers(path, paths); store->queryReferrers(path, paths);
else if (op == wopQueryValidDerivers) else if (op == wopQueryValidDerivers)
@ -329,8 +341,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopQueryDerivationOutputNames: { case wopQueryDerivationOutputNames: {
auto path = store->parseStorePath(readString(from)); auto path = store->parseStorePath(readString(from));
logger->startWork(); logger->startWork();
StringSet names; auto names = store->readDerivation(path).outputNames();
names = store->queryDerivationOutputNames(path);
logger->stopWork(); logger->stopWork();
to << names; to << names;
break; break;
@ -355,20 +366,26 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
} }
case wopAddToStore: { case wopAddToStore: {
bool fixed, recursive;
std::string s, baseName; std::string s, baseName;
from >> baseName >> fixed /* obsolete */ >> recursive >> s; FileIngestionMethod method;
/* Compatibility hack. */ {
if (!fixed) { bool fixed; uint8_t recursive;
s = "sha256"; from >> baseName >> fixed /* obsolete */ >> recursive >> s;
recursive = true; if (recursive > (uint8_t) FileIngestionMethod::Recursive)
throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
method = FileIngestionMethod { recursive };
/* Compatibility hack. */
if (!fixed) {
s = "sha256";
method = FileIngestionMethod::Recursive;
}
} }
HashType hashAlgo = parseHashType(s); HashType hashAlgo = parseHashType(s);
TeeSource savedNAR(from); TeeSource savedNAR(from);
RetrieveRegularNARSink savedRegular; RetrieveRegularNARSink savedRegular;
if (recursive) { if (method == FileIngestionMethod::Recursive) {
/* Get the entire NAR dump from the client and save it to /* Get the entire NAR dump from the client and save it to
a string so that we can pass it to a string so that we can pass it to
addToStoreFromDump(). */ addToStoreFromDump(). */
@ -380,7 +397,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork(); logger->startWork();
if (!savedRegular.regular) throw Error("regular file expected"); if (!savedRegular.regular) throw Error("regular file expected");
auto path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); auto path = store->addToStoreFromDump(
method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s,
baseName,
method,
hashAlgo);
logger->stopWork(); logger->stopWork();
to << store->printStorePath(path); to << store->printStorePath(path);
@ -572,9 +593,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto path = store->parseStorePath(readString(from)); auto path = store->parseStorePath(readString(from));
logger->startWork(); logger->startWork();
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
StorePathSet paths; store->querySubstitutablePathInfos({path}, infos);
paths.insert(path.clone()); // FIXME
store->querySubstitutablePathInfos(paths, infos);
logger->stopWork(); logger->stopWork();
auto i = infos.find(path); auto i = infos.find(path);
if (i == infos.end()) if (i == infos.end())
@ -735,7 +754,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
} }
default: default:
throw Error(format("invalid operation %1%") % op); throw Error("invalid operation %1%", op);
} }
} }

View file

@ -9,13 +9,13 @@
namespace nix { namespace nix {
void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const
{ {
recursive = false; recursive = FileIngestionMethod::Flat;
string algo = hashAlgo; string algo = hashAlgo;
if (string(algo, 0, 2) == "r:") { if (string(algo, 0, 2) == "r:") {
recursive = true; recursive = FileIngestionMethod::Recursive;
algo = string(algo, 2); algo = string(algo, 2);
} }
@ -27,28 +27,6 @@ void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
} }
BasicDerivation::BasicDerivation(const BasicDerivation & other)
: platform(other.platform)
, builder(other.builder)
, args(other.args)
, env(other.env)
{
for (auto & i : other.outputs)
outputs.insert_or_assign(i.first,
DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash)));
for (auto & i : other.inputSrcs)
inputSrcs.insert(i.clone());
}
Derivation::Derivation(const Derivation & other)
: BasicDerivation(other)
{
for (auto & i : other.inputDrvs)
inputDrvs.insert_or_assign(i.first.clone(), i.second);
}
const StorePath & BasicDerivation::findOutput(const string & id) const const StorePath & BasicDerivation::findOutput(const string & id) const
{ {
auto i = outputs.find(id); auto i = outputs.find(id);
@ -65,16 +43,16 @@ bool BasicDerivation::isBuiltin() const
StorePath writeDerivation(ref<Store> store, StorePath writeDerivation(ref<Store> store,
const Derivation & drv, const string & name, RepairFlag repair) const Derivation & drv, std::string_view name, RepairFlag repair)
{ {
auto references = cloneStorePathSet(drv.inputSrcs); auto references = drv.inputSrcs;
for (auto & i : drv.inputDrvs) for (auto & i : drv.inputDrvs)
references.insert(i.first.clone()); references.insert(i.first);
/* Note that the outputs of a derivation are *not* references /* Note that the outputs of a derivation are *not* references
(that can be missing (of course) and should not necessarily be (that can be missing (of course) and should not necessarily be
held during a garbage collection). */ held during a garbage collection). */
string suffix = name + drvExtension; auto suffix = std::string(name) + drvExtension;
string contents = drv.unparse(*store, false); auto contents = drv.unparse(*store, false);
return settings.readOnlyMode return settings.readOnlyMode
? store->computeStorePathForText(suffix, contents, references) ? store->computeStorePathForText(suffix, contents, references)
: store->addTextToStore(suffix, contents, references, repair); : store->addTextToStore(suffix, contents, references, repair);
@ -87,7 +65,7 @@ static void expect(std::istream & str, const string & s)
char s2[s.size()]; char s2[s.size()];
str.read(s2, s.size()); str.read(s2, s.size());
if (string(s2, s.size()) != s) if (string(s2, s.size()) != s)
throw FormatError(format("expected string '%1%'") % s); throw FormatError("expected string '%1%'", s);
} }
@ -114,7 +92,7 @@ static Path parsePath(std::istream & str)
{ {
string s = parseString(str); string s = parseString(str);
if (s.size() == 0 || s[0] != '/') if (s.size() == 0 || s[0] != '/')
throw FormatError(format("bad path '%1%' in derivation") % s); throw FormatError("bad path '%1%' in derivation", s);
return s; return s;
} }
@ -155,7 +133,11 @@ static Derivation parseDerivation(const Store & store, const string & s)
expect(str, ","); auto hashAlgo = parseString(str); expect(str, ","); auto hashAlgo = parseString(str);
expect(str, ","); auto hash = parseString(str); expect(str, ","); auto hash = parseString(str);
expect(str, ")"); expect(str, ")");
drv.outputs.emplace(id, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash))); drv.outputs.emplace(id, DerivationOutput {
.path = std::move(path),
.hashAlgo = std::move(hashAlgo),
.hash = std::move(hash)
});
} }
/* Parse the list of input derivations. */ /* Parse the list of input derivations. */
@ -196,7 +178,7 @@ Derivation readDerivation(const Store & store, const Path & drvPath)
try { try {
return parseDerivation(store, readFile(drvPath)); return parseDerivation(store, readFile(drvPath));
} catch (FormatError & e) { } catch (FormatError & e) {
throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg());
} }
} }
@ -204,6 +186,12 @@ Derivation readDerivation(const Store & store, const Path & drvPath)
Derivation Store::derivationFromPath(const StorePath & drvPath) Derivation Store::derivationFromPath(const StorePath & drvPath)
{ {
ensurePath(drvPath); ensurePath(drvPath);
return readDerivation(drvPath);
}
Derivation Store::readDerivation(const StorePath & drvPath)
{
auto accessor = getFSAccessor(); auto accessor = getFSAccessor();
try { try {
return parseDerivation(*this, accessor->readFile(printStorePath(drvPath))); return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)));
@ -351,12 +339,10 @@ static DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drv
assert(store.isValidPath(drvPath)); assert(store.isValidPath(drvPath));
// Cache it // Cache it
h = drvHashes.insert_or_assign( h = drvHashes.insert_or_assign(
drvPath.clone(), drvPath,
hashDerivationModulo( hashDerivationModulo(
store, store,
readDerivation( store.readDerivation(drvPath),
store,
store.toRealPath(store.printStorePath(drvPath))),
false)).first; false)).first;
} }
return h->second; return h->second;
@ -437,11 +423,20 @@ StorePathSet BasicDerivation::outputPaths() const
{ {
StorePathSet paths; StorePathSet paths;
for (auto & i : outputs) for (auto & i : outputs)
paths.insert(i.second.path.clone()); paths.insert(i.second.path);
return paths; return paths;
} }
StringSet BasicDerivation::outputNames() const
{
StringSet names;
for (auto & i : outputs)
names.insert(i.first);
return names;
}
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
{ {
drv.outputs.clear(); drv.outputs.clear();
@ -451,7 +446,11 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
auto path = store.parseStorePath(readString(in)); auto path = store.parseStorePath(readString(in));
auto hashAlgo = readString(in); auto hashAlgo = readString(in);
auto hash = readString(in); auto hash = readString(in);
drv.outputs.emplace(name, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash))); drv.outputs.emplace(name, DerivationOutput {
.path = std::move(path),
.hashAlgo = std::move(hashAlgo),
.hash = std::move(hash)
});
} }
drv.inputSrcs = readStorePaths<StorePathSet>(store, in); drv.inputSrcs = readStorePaths<StorePathSet>(store, in);

View file

@ -18,12 +18,7 @@ struct DerivationOutput
StorePath path; StorePath path;
std::string hashAlgo; /* hash used for expected hash computation */ std::string hashAlgo; /* hash used for expected hash computation */
std::string hash; /* expected hash, may be null */ std::string hash; /* expected hash, may be null */
DerivationOutput(StorePath && path, std::string && hashAlgo, std::string && hash) void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const;
: path(std::move(path))
, hashAlgo(std::move(hashAlgo))
, hash(std::move(hash))
{ }
void parseHashInfo(bool & recursive, Hash & hash) const;
}; };
typedef std::map<string, DerivationOutput> DerivationOutputs; typedef std::map<string, DerivationOutput> DerivationOutputs;
@ -44,7 +39,6 @@ struct BasicDerivation
StringPairs env; StringPairs env;
BasicDerivation() { } BasicDerivation() { }
explicit BasicDerivation(const BasicDerivation & other);
virtual ~BasicDerivation() { }; virtual ~BasicDerivation() { };
/* Return the path corresponding to the output identifier `id' in /* Return the path corresponding to the output identifier `id' in
@ -59,6 +53,8 @@ struct BasicDerivation
/* Return the output paths of a derivation. */ /* Return the output paths of a derivation. */
StorePathSet outputPaths() const; StorePathSet outputPaths() const;
/* Return the output names of a derivation. */
StringSet outputNames() const;
}; };
struct Derivation : BasicDerivation struct Derivation : BasicDerivation
@ -70,8 +66,6 @@ struct Derivation : BasicDerivation
std::map<std::string, StringSet> * actualInputs = nullptr) const; std::map<std::string, StringSet> * actualInputs = nullptr) const;
Derivation() { } Derivation() { }
Derivation(Derivation && other) = default;
explicit Derivation(const Derivation & other);
}; };
@ -80,7 +74,7 @@ class Store;
/* Write a derivation to the Nix store, and return its path. */ /* Write a derivation to the Nix store, and return its path. */
StorePath writeDerivation(ref<Store> store, StorePath writeDerivation(ref<Store> store,
const Derivation & drv, const string & name, RepairFlag repair = NoRepair); const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair);
/* Read a derivation from a file. */ /* Read a derivation from a file. */
Derivation readDerivation(const Store & store, const Path & drvPath); Derivation readDerivation(const Store & store, const Path & drvPath);

View file

@ -1,3 +1,4 @@
#include "serialise.hh"
#include "store-api.hh" #include "store-api.hh"
#include "archive.hh" #include "archive.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
@ -56,7 +57,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
Hash hash = hashAndWriteSink.currentHash(); Hash hash = hashAndWriteSink.currentHash();
if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
throw Error("hash of path '%s' has changed from '%s' to '%s'!", throw Error("hash of path '%s' has changed from '%s' to '%s'!",
printStorePath(path), info->narHash.to_string(), hash.to_string()); printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
hashAndWriteSink hashAndWriteSink
<< exportMagic << exportMagic
@ -100,9 +101,11 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
if (readInt(source) == 1) if (readInt(source) == 1)
readString(source); readString(source);
addToStore(info, tee.source.data, NoRepair, checkSigs, accessor); // Can't use underlying source, which would have been exhausted
auto source = StringSource { *tee.source.data };
addToStore(info, source, NoRepair, checkSigs, accessor);
res.push_back(info.path.clone()); res.push_back(info.path);
} }
return res; return res;

View file

@ -1,14 +1,10 @@
#include "download.hh" #include "filetransfer.hh"
#include "util.hh" #include "util.hh"
#include "globals.hh" #include "globals.hh"
#include "hash.hh"
#include "store-api.hh" #include "store-api.hh"
#include "archive.hh"
#include "s3.hh" #include "s3.hh"
#include "compression.hh" #include "compression.hh"
#include "pathlocks.hh"
#include "finally.hh" #include "finally.hh"
#include "tarfile.hh"
#ifdef ENABLE_S3 #ifdef ENABLE_S3
#include <aws/core/client/ClientConfiguration.h> #include <aws/core/client/ClientConfiguration.h>
@ -31,13 +27,9 @@ using namespace std::string_literals;
namespace nix { namespace nix {
DownloadSettings downloadSettings; FileTransferSettings fileTransferSettings;
static GlobalConfig::Register r1(&downloadSettings); static GlobalConfig::Register r1(&fileTransferSettings);
CachedDownloadRequest::CachedDownloadRequest(const std::string & uri)
: uri(uri), ttl(settings.tarballTtl)
{ }
std::string resolveUri(const std::string & uri) std::string resolveUri(const std::string & uri)
{ {
@ -47,21 +39,21 @@ std::string resolveUri(const std::string & uri)
return uri; return uri;
} }
struct CurlDownloader : public Downloader struct curlFileTransfer : public FileTransfer
{ {
CURLM * curlm = 0; CURLM * curlm = 0;
std::random_device rd; std::random_device rd;
std::mt19937 mt19937; std::mt19937 mt19937;
struct DownloadItem : public std::enable_shared_from_this<DownloadItem> struct TransferItem : public std::enable_shared_from_this<TransferItem>
{ {
CurlDownloader & downloader; curlFileTransfer & fileTransfer;
DownloadRequest request; FileTransferRequest request;
DownloadResult result; FileTransferResult result;
Activity act; Activity act;
bool done = false; // whether either the success or failure function has been called bool done = false; // whether either the success or failure function has been called
Callback<DownloadResult> callback; Callback<FileTransferResult> callback;
CURL * req = 0; CURL * req = 0;
bool active = false; // whether the handle has been added to the multi object bool active = false; // whether the handle has been added to the multi object
std::string status; std::string status;
@ -80,19 +72,36 @@ struct CurlDownloader : public Downloader
curl_off_t writtenToSink = 0; curl_off_t writtenToSink = 0;
DownloadItem(CurlDownloader & downloader, /* Get the HTTP status code, or 0 for other protocols. */
const DownloadRequest & request, long getHTTPStatus()
Callback<DownloadResult> && callback) {
: downloader(downloader) long httpStatus = 0;
long protocol = 0;
curl_easy_getinfo(req, CURLINFO_PROTOCOL, &protocol);
if (protocol == CURLPROTO_HTTP || protocol == CURLPROTO_HTTPS)
curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
return httpStatus;
}
TransferItem(curlFileTransfer & fileTransfer,
const FileTransferRequest & request,
Callback<FileTransferResult> && callback)
: fileTransfer(fileTransfer)
, request(request) , request(request)
, act(*logger, lvlTalkative, actDownload, , act(*logger, lvlTalkative, actFileTransfer,
fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
{request.uri}, request.parentAct) {request.uri}, request.parentAct)
, callback(std::move(callback)) , callback(std::move(callback))
, finalSink([this](const unsigned char * data, size_t len) { , finalSink([this](const unsigned char * data, size_t len) {
if (this->request.dataCallback) { if (this->request.dataCallback) {
writtenToSink += len; auto httpStatus = getHTTPStatus();
this->request.dataCallback((char *) data, len);
/* Only write data to the sink if this is a
successful response. */
if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) {
writtenToSink += len;
this->request.dataCallback((char *) data, len);
}
} else } else
this->result.data->append((char *) data, len); this->result.data->append((char *) data, len);
}) })
@ -103,17 +112,17 @@ struct CurlDownloader : public Downloader
requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str()); requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
} }
~DownloadItem() ~TransferItem()
{ {
if (req) { if (req) {
if (active) if (active)
curl_multi_remove_handle(downloader.curlm, req); curl_multi_remove_handle(fileTransfer.curlm, req);
curl_easy_cleanup(req); curl_easy_cleanup(req);
} }
if (requestHeaders) curl_slist_free_all(requestHeaders); if (requestHeaders) curl_slist_free_all(requestHeaders);
try { try {
if (!done) if (!done)
fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri)); fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri));
} catch (...) { } catch (...) {
ignoreException(); ignoreException();
} }
@ -157,7 +166,7 @@ struct CurlDownloader : public Downloader
static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
{ {
return ((DownloadItem *) userp)->writeCallback(contents, size, nmemb); return ((TransferItem *) userp)->writeCallback(contents, size, nmemb);
} }
size_t headerCallback(void * contents, size_t size, size_t nmemb) size_t headerCallback(void * contents, size_t size, size_t nmemb)
@ -199,7 +208,7 @@ struct CurlDownloader : public Downloader
static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
{ {
return ((DownloadItem *) userp)->headerCallback(contents, size, nmemb); return ((TransferItem *) userp)->headerCallback(contents, size, nmemb);
} }
int progressCallback(double dltotal, double dlnow) int progressCallback(double dltotal, double dlnow)
@ -214,7 +223,7 @@ struct CurlDownloader : public Downloader
static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow)
{ {
return ((DownloadItem *) userp)->progressCallback(dltotal, dlnow); return ((TransferItem *) userp)->progressCallback(dltotal, dlnow);
} }
static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr)
@ -238,7 +247,7 @@ struct CurlDownloader : public Downloader
static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp) static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
{ {
return ((DownloadItem *) userp)->readCallback(buffer, size, nitems); return ((TransferItem *) userp)->readCallback(buffer, size, nitems);
} }
void init() void init()
@ -249,7 +258,7 @@ struct CurlDownloader : public Downloader
if (verbosity >= lvlVomit) { if (verbosity >= lvlVomit) {
curl_easy_setopt(req, CURLOPT_VERBOSE, 1); curl_easy_setopt(req, CURLOPT_VERBOSE, 1);
curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback); curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback);
} }
curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
@ -258,19 +267,19 @@ struct CurlDownloader : public Downloader
curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
curl_easy_setopt(req, CURLOPT_USERAGENT, curl_easy_setopt(req, CURLOPT_USERAGENT,
("curl/" LIBCURL_VERSION " Nix/" + nixVersion + ("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
(downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str()); (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : "")).c_str());
#if LIBCURL_VERSION_NUM >= 0x072b00 #if LIBCURL_VERSION_NUM >= 0x072b00
curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
#endif #endif
#if LIBCURL_VERSION_NUM >= 0x072f00 #if LIBCURL_VERSION_NUM >= 0x072f00
if (downloadSettings.enableHttp2) if (fileTransferSettings.enableHttp2)
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
else else
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
#endif #endif
curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper); curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper);
curl_easy_setopt(req, CURLOPT_WRITEDATA, this); curl_easy_setopt(req, CURLOPT_WRITEDATA, this);
curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper); curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper);
curl_easy_setopt(req, CURLOPT_HEADERDATA, this); curl_easy_setopt(req, CURLOPT_HEADERDATA, this);
curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper); curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper);
@ -298,10 +307,10 @@ struct CurlDownloader : public Downloader
curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
} }
curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get());
/* If no file exist in the specified path, curl continues to work /* If no file exist in the specified path, curl continues to work
anyway as if netrc support was disabled. */ anyway as if netrc support was disabled. */
@ -317,8 +326,7 @@ struct CurlDownloader : public Downloader
void finish(CURLcode code) void finish(CURLcode code)
{ {
long httpStatus = 0; auto httpStatus = getHTTPStatus();
curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
char * effectiveUriCStr; char * effectiveUriCStr;
curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
@ -345,7 +353,7 @@ struct CurlDownloader : public Downloader
failEx(writeException); failEx(writeException);
else if (code == CURLE_OK && else if (code == CURLE_OK &&
(httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 0 /* other protocol */))
{ {
result.cached = httpStatus == 304; result.cached = httpStatus == 304;
act.progress(result.bodySize, result.bodySize); act.progress(result.bodySize, result.bodySize);
@ -390,6 +398,7 @@ struct CurlDownloader : public Downloader
case CURLE_SSL_CACERT_BADFILE: case CURLE_SSL_CACERT_BADFILE:
case CURLE_TOO_MANY_REDIRECTS: case CURLE_TOO_MANY_REDIRECTS:
case CURLE_WRITE_ERROR: case CURLE_WRITE_ERROR:
case CURLE_UNSUPPORTED_PROTOCOL:
err = Misc; err = Misc;
break; break;
default: // Shut up warnings default: // Shut up warnings
@ -401,14 +410,14 @@ struct CurlDownloader : public Downloader
auto exc = auto exc =
code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
: httpStatus != 0 : httpStatus != 0
? DownloadError(err, ? FileTransferError(err,
fmt("unable to %s '%s': HTTP error %d", fmt("unable to %s '%s': HTTP error %d",
request.verb(), request.uri, httpStatus) request.verb(), request.uri, httpStatus)
+ (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
) )
: DownloadError(err, : FileTransferError(err,
fmt("unable to %s '%s': %s (%d)", fmt("unable to %s '%s': %s (%d)",
request.verb(), request.uri, curl_easy_strerror(code), code)); request.verb(), request.uri, curl_easy_strerror(code), code));
@ -422,13 +431,13 @@ struct CurlDownloader : public Downloader
|| writtenToSink == 0 || writtenToSink == 0
|| (acceptRanges && encoding.empty()))) || (acceptRanges && encoding.empty())))
{ {
int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937));
if (writtenToSink) if (writtenToSink)
warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
else else
warn("%s; retrying in %d ms", exc.what(), ms); warn("%s; retrying in %d ms", exc.what(), ms);
embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
downloader.enqueueItem(shared_from_this()); fileTransfer.enqueueItem(shared_from_this());
} }
else else
fail(exc); fail(exc);
@ -439,12 +448,12 @@ struct CurlDownloader : public Downloader
struct State struct State
{ {
struct EmbargoComparator { struct EmbargoComparator {
bool operator() (const std::shared_ptr<DownloadItem> & i1, const std::shared_ptr<DownloadItem> & i2) { bool operator() (const std::shared_ptr<TransferItem> & i1, const std::shared_ptr<TransferItem> & i2) {
return i1->embargo > i2->embargo; return i1->embargo > i2->embargo;
} }
}; };
bool quit = false; bool quit = false;
std::priority_queue<std::shared_ptr<DownloadItem>, std::vector<std::shared_ptr<DownloadItem>>, EmbargoComparator> incoming; std::priority_queue<std::shared_ptr<TransferItem>, std::vector<std::shared_ptr<TransferItem>>, EmbargoComparator> incoming;
}; };
Sync<State> state_; Sync<State> state_;
@ -456,7 +465,7 @@ struct CurlDownloader : public Downloader
std::thread workerThread; std::thread workerThread;
CurlDownloader() curlFileTransfer()
: mt19937(rd()) : mt19937(rd())
{ {
static std::once_flag globalInit; static std::once_flag globalInit;
@ -469,7 +478,7 @@ struct CurlDownloader : public Downloader
#endif #endif
#if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0 #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0
curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS,
downloadSettings.httpConnections.get()); fileTransferSettings.httpConnections.get());
#endif #endif
wakeupPipe.create(); wakeupPipe.create();
@ -478,7 +487,7 @@ struct CurlDownloader : public Downloader
workerThread = std::thread([&]() { workerThreadEntry(); }); workerThread = std::thread([&]() { workerThreadEntry(); });
} }
~CurlDownloader() ~curlFileTransfer()
{ {
stopWorkerThread(); stopWorkerThread();
@ -504,7 +513,7 @@ struct CurlDownloader : public Downloader
stopWorkerThread(); stopWorkerThread();
}); });
std::map<CURL *, std::shared_ptr<DownloadItem>> items; std::map<CURL *, std::shared_ptr<TransferItem>> items;
bool quit = false; bool quit = false;
@ -517,7 +526,7 @@ struct CurlDownloader : public Downloader
int running; int running;
CURLMcode mc = curl_multi_perform(curlm, &running); CURLMcode mc = curl_multi_perform(curlm, &running);
if (mc != CURLM_OK) if (mc != CURLM_OK)
throw nix::Error(format("unexpected error from curl_multi_perform(): %s") % curl_multi_strerror(mc)); throw nix::Error("unexpected error from curl_multi_perform(): %s", curl_multi_strerror(mc));
/* Set the promises of any finished requests. */ /* Set the promises of any finished requests. */
CURLMsg * msg; CURLMsg * msg;
@ -547,7 +556,7 @@ struct CurlDownloader : public Downloader
vomit("download thread waiting for %d ms", sleepTimeMs); vomit("download thread waiting for %d ms", sleepTimeMs);
mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
if (mc != CURLM_OK) if (mc != CURLM_OK)
throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % curl_multi_strerror(mc)); throw nix::Error("unexpected error from curl_multi_wait(): %s", curl_multi_strerror(mc));
nextWakeup = std::chrono::steady_clock::time_point(); nextWakeup = std::chrono::steady_clock::time_point();
@ -561,7 +570,7 @@ struct CurlDownloader : public Downloader
throw SysError("reading curl wakeup socket"); throw SysError("reading curl wakeup socket");
} }
std::vector<std::shared_ptr<DownloadItem>> incoming; std::vector<std::shared_ptr<TransferItem>> incoming;
auto now = std::chrono::steady_clock::now(); auto now = std::chrono::steady_clock::now();
{ {
@ -599,7 +608,11 @@ struct CurlDownloader : public Downloader
workerThreadMain(); workerThreadMain();
} catch (nix::Interrupted & e) { } catch (nix::Interrupted & e) {
} catch (std::exception & e) { } catch (std::exception & e) {
printError("unexpected error in download thread: %s", e.what()); logError({
.name = "File transfer",
.hint = hintfmt("unexpected error in download thread: %s",
e.what())
});
} }
{ {
@ -609,7 +622,7 @@ struct CurlDownloader : public Downloader
} }
} }
void enqueueItem(std::shared_ptr<DownloadItem> item) void enqueueItem(std::shared_ptr<TransferItem> item)
{ {
if (item->request.data if (item->request.data
&& !hasPrefix(item->request.uri, "http://") && !hasPrefix(item->request.uri, "http://")
@ -641,8 +654,8 @@ struct CurlDownloader : public Downloader
} }
#endif #endif
void enqueueDownload(const DownloadRequest & request, void enqueueFileTransfer(const FileTransferRequest & request,
Callback<DownloadResult> callback) override Callback<FileTransferResult> callback) override
{ {
/* Ugly hack to support s3:// URIs. */ /* Ugly hack to support s3:// URIs. */
if (hasPrefix(request.uri, "s3://")) { if (hasPrefix(request.uri, "s3://")) {
@ -660,9 +673,9 @@ struct CurlDownloader : public Downloader
// FIXME: implement ETag // FIXME: implement ETag
auto s3Res = s3Helper.getObject(bucketName, key); auto s3Res = s3Helper.getObject(bucketName, key);
DownloadResult res; FileTransferResult res;
if (!s3Res.data) if (!s3Res.data)
throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
res.data = s3Res.data; res.data = s3Res.data;
callback(std::move(res)); callback(std::move(res));
#else #else
@ -672,26 +685,26 @@ struct CurlDownloader : public Downloader
return; return;
} }
enqueueItem(std::make_shared<DownloadItem>(*this, request, std::move(callback))); enqueueItem(std::make_shared<TransferItem>(*this, request, std::move(callback)));
} }
}; };
ref<Downloader> getDownloader() ref<FileTransfer> getFileTransfer()
{ {
static ref<Downloader> downloader = makeDownloader(); static ref<FileTransfer> fileTransfer = makeFileTransfer();
return downloader; return fileTransfer;
} }
ref<Downloader> makeDownloader() ref<FileTransfer> makeFileTransfer()
{ {
return make_ref<CurlDownloader>(); return make_ref<curlFileTransfer>();
} }
std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & request) std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request)
{ {
auto promise = std::make_shared<std::promise<DownloadResult>>(); auto promise = std::make_shared<std::promise<FileTransferResult>>();
enqueueDownload(request, enqueueFileTransfer(request,
{[promise](std::future<DownloadResult> fut) { {[promise](std::future<FileTransferResult> fut) {
try { try {
promise->set_value(fut.get()); promise->set_value(fut.get());
} catch (...) { } catch (...) {
@ -701,15 +714,21 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest &
return promise->get_future(); return promise->get_future();
} }
DownloadResult Downloader::download(const DownloadRequest & request) FileTransferResult FileTransfer::download(const FileTransferRequest & request)
{ {
return enqueueDownload(request).get(); return enqueueFileTransfer(request).get();
} }
void Downloader::download(DownloadRequest && request, Sink & sink) FileTransferResult FileTransfer::upload(const FileTransferRequest & request)
{
/* Note: this method is the same as download, but helps in readability */
return enqueueFileTransfer(request).get();
}
void FileTransfer::download(FileTransferRequest && request, Sink & sink)
{ {
/* Note: we can't call 'sink' via request.dataCallback, because /* Note: we can't call 'sink' via request.dataCallback, because
that would cause the sink to execute on the downloader that would cause the sink to execute on the fileTransfer
thread. If 'sink' is a coroutine, this will fail. Also, if the thread. If 'sink' is a coroutine, this will fail. Also, if the
sink is expensive (e.g. one that does decompression and writing sink is expensive (e.g. one that does decompression and writing
to the Nix store), it would stall the download thread too much. to the Nix store), it would stall the download thread too much.
@ -755,8 +774,8 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
state->avail.notify_one(); state->avail.notify_one();
}; };
enqueueDownload(request, enqueueFileTransfer(request,
{[_state](std::future<DownloadResult> fut) { {[_state](std::future<FileTransferResult> fut) {
auto state(_state->lock()); auto state(_state->lock());
state->quit = true; state->quit = true;
try { try {
@ -801,140 +820,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
} }
} }
CachedDownloadResult Downloader::downloadCached(
ref<Store> store, const CachedDownloadRequest & request)
{
auto url = resolveUri(request.uri);
auto name = request.name;
if (name == "") {
auto p = url.rfind('/');
if (p != string::npos) name = string(url, p + 1);
}
std::optional<StorePath> expectedStorePath;
if (request.expectedHash) {
expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name);
if (store->isValidPath(*expectedStorePath)) {
CachedDownloadResult result;
result.storePath = store->printStorePath(*expectedStorePath);
result.path = store->toRealPath(result.storePath);
return result;
}
}
Path cacheDir = getCacheDir() + "/nix/tarballs";
createDirs(cacheDir);
string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false);
Path dataFile = cacheDir + "/" + urlHash + ".info";
Path fileLink = cacheDir + "/" + urlHash + "-file";
PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink));
std::optional<StorePath> storePath;
string expectedETag;
bool skip = false;
CachedDownloadResult result;
if (pathExists(fileLink) && pathExists(dataFile)) {
storePath = store->parseStorePath(readLink(fileLink));
// FIXME
store->addTempRoot(*storePath);
if (store->isValidPath(*storePath)) {
auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
if (ss.size() >= 3 && ss[0] == url) {
time_t lastChecked;
if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) {
skip = true;
result.effectiveUri = request.uri;
result.etag = ss[1];
} else if (!ss[1].empty()) {
debug(format("verifying previous ETag '%1%'") % ss[1]);
expectedETag = ss[1];
}
}
} else
storePath.reset();
}
if (!skip) {
try {
DownloadRequest request2(url);
request2.expectedETag = expectedETag;
auto res = download(request2);
result.effectiveUri = res.effectiveUri;
result.etag = res.etag;
if (!res.cached) {
StringSink sink;
dumpString(*res.data, sink);
Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data);
ValidPathInfo info(store->makeFixedOutputPath(false, hash, name));
info.narHash = hashString(htSHA256, *sink.s);
info.narSize = sink.s->size();
info.ca = makeFixedOutputCA(false, hash);
store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
storePath = info.path.clone();
}
assert(storePath);
replaceSymlink(store->printStorePath(*storePath), fileLink);
writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
} catch (DownloadError & e) {
if (!storePath) throw;
warn("warning: %s; using cached result", e.msg());
result.etag = expectedETag;
}
}
if (request.unpack) {
Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked";
PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink));
std::optional<StorePath> unpackedStorePath;
if (pathExists(unpackedLink)) {
unpackedStorePath = store->parseStorePath(readLink(unpackedLink));
// FIXME
store->addTempRoot(*unpackedStorePath);
if (!store->isValidPath(*unpackedStorePath))
unpackedStorePath.reset();
}
if (!unpackedStorePath) {
printInfo("unpacking '%s'...", url);
Path tmpDir = createTempDir();
AutoDelete autoDelete(tmpDir, true);
unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir);
auto members = readDirectory(tmpDir);
if (members.size() != 1)
throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
auto topDir = tmpDir + "/" + members.begin()->name;
unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair);
}
replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink);
storePath = std::move(*unpackedStorePath);
}
if (expectedStorePath && *storePath != *expectedStorePath) {
unsigned int statusCode = 102;
Hash gotHash = request.unpack
? hashPath(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))).first
: hashFile(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath)));
throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
url, request.expectedHash.to_string(), gotHash.to_string());
}
result.storePath = store->printStorePath(*storePath);
result.path = store->toRealPath(result.storePath);
return result;
}
bool isUri(const string & s) bool isUri(const string & s)
{ {
if (s.compare(0, 8, "channel:") == 0) return true; if (s.compare(0, 8, "channel:") == 0) return true;

Some files were not shown because too many files have changed in this diff Show more